diff --git a/doc/preprocessor-guide.md b/doc/preprocessor-guide.md index 82f779c..ef454ca 100644 --- a/doc/preprocessor-guide.md +++ b/doc/preprocessor-guide.md @@ -11,10 +11,11 @@ The Second Life Script Preprocessor is a comprehensive tool that supports advanc 5. [Include vs Require Behavior](#include-vs-require-behavior) 6. [Macro Definitions (Defines)](#macro-definitions-defines) 7. [Conditional Processing](#conditional-processing) -8. [Complete Examples](#complete-examples) -9. [Best Practices](#best-practices) -10. [Limitations](#limitations) -11. [Integration with VS Code Extension](#integration-with-vs-code-extension) +8. [LSL Preprocessor features](#lsl-preprocessor-features) +9. [Complete Examples](#complete-examples) +10. [Best Practices](#best-practices) +11. [Limitations](#limitations) +12. [Integration with VS Code Extension](#integration-with-vs-code-extension) ## Overview @@ -495,6 +496,53 @@ integer result = SQUARE(ADD(2, 3)); // Expands to ((2 + 3) * (2 + 3)) integer result2 = CUBE(ADD(1, 2)); // Expands to ((1 + 2) * ((1 + 2) * (1 + 2))) ``` +### Variadic Macros + +Macro function can be variadic, allowing you to pass any number of arguments (usually to handle them as a list) + +```lsl +#define LOG(type,...) llOwnerSay(llDumpList2String([type,":",__VA_ARGS__]," ")) + +// Before preproccessing +LOG("Test",1,2,3,4); + +// Result +llOwnerSay(llDumpList2String(["Test",":",1,2,3,4]," ")); +``` + +### Empty Function macros + +A common usecase for function macro's is debug statements only when a `DEBUG` flag is set, you can achieve this with a conditional function macro with one instance having an empty body + +```lsl +#ifdef DEBUG +#define debug(...) llOwnerSay(llDumpList2String([__VA_ARGS__]," ")) +#else +#define debug(...) +#endif + +default { + state_entry() { + llOwnerSay("Start"); + debug("This is a debug message:", 1, 2, 3); + } +} +``` + +Would output + +```lsl +default { + state_entry() { + llOwnerSay("Start"); + ; + } +} +``` + +Which is valid LSL and the extra `;` does not consume bytecode memory. + + ### Stringization Operator (#) The stringization operator (`#`) converts macro parameters into string literals. This is particularly useful for debugging, logging, and creating dynamic messages. @@ -780,6 +828,35 @@ Includes code if all previous conditions were false: 5. **Comparison Operations**: `==`, `!=`, `>`, `>=`, `<`, `<=` 6. **defined() Function**: `#if defined(MACRO_NAME)` +## LSL Preprocessor features + +These are features supported for LSL only, and are mostly to provide parity with common existing tooling. + +### Switch statements + +If you enable the config `slVscodeEdit.preprocessor.lsl.switchStatements` in the preprocessor section. + +The preprocessor can handle switch statements by converting them to if conditionals with jumps. + +#### Examples +```lsl +default +{ + touch_start(integer num_detected) + { + integer coin = llFloor(llFrand(2.0)); + switch(coin) { + case 1: { + llOwnerSay("Heads!"); + } + default: { + llOwnerSay("Tails!"); + } + } + } +} +``` + ## Complete Examples ### Feature Toggle System diff --git a/eslint.config.js b/eslint.config.js index 224f0d2..4fedefc 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -26,6 +26,7 @@ export default [ require: 'readonly', module: 'readonly', exports: 'readonly', + structuredClone: 'readonly', // Browser/Web globals that Node.js also has TextEncoder: 'readonly', // VS Code extension globals diff --git a/package.json b/package.json index 230e388..b7f8d16 100644 --- a/package.json +++ b/package.json @@ -139,6 +139,12 @@ "type": "boolean", "default": false, "description": "Enable predefined LSL style preproc macro constants in slua" + }, + "slVscodeEdit.preprocessor.lsl.switchStatements": { + "title": "Enable LSL switch statements", + "type": "boolean", + "default": false, + "description": "Enable LSL switch statements in the preprocessor" } } }, @@ -176,7 +182,7 @@ "lint:fix": "eslint src --fix", "test": "node ./out/test/runTest.js", "test-basic": "npx mocha \"./out/test/suite/basic.test.js\" --ui tdd --timeout 5000", - "test-unit": "npx mocha \"./out/test/suite/{basic,preprocessor,include-disk-integration}.test.js\" --ui tdd --timeout 5000", + "test-unit": "npx mocha \"./out/test/suite/{basic,preprocessor,include-disk-integration,lexingpreprocessor}.test.js\" --ui tdd --timeout 5000", "test-compile": "tsc -p . --outDir out --skipLibCheck", "precommit": "npm run lint && npm run test-compile && npm run test-unit" }, diff --git a/src/interfaces/configinterface.ts b/src/interfaces/configinterface.ts index 390a209..ef55b2d 100644 --- a/src/interfaces/configinterface.ts +++ b/src/interfaces/configinterface.ts @@ -28,6 +28,7 @@ export enum ConfigKey { PreprocessorIncludePaths = 'preprocessor.includePaths', PreprocessorMaxIncludeDepth = 'preprocessor.maxIncludeDepth', PreprocessorConstantsInSLua = 'preprocessor.constantsInSLua', + PreprocessorLSLSwitchStatements = 'preprocessor.lsl.switchStatements', LastSyntaxID = 'syntax.lastID', AskIfViewerScriptMismatchesMaster = 'sync.askIfViewerScriptMismatchesMaster', CompareHashBeforeSync = 'sync.compareHashBeforeSync', diff --git a/src/scriptsync.ts b/src/scriptsync.ts index cf80091..f90917b 100644 --- a/src/scriptsync.ts +++ b/src/scriptsync.ts @@ -29,7 +29,7 @@ import { HostInterface, normalizePath } from "./interfaces/hostinterface"; import { SynchService } from "./synchservice"; import { IncludeInfo } from "./shared/parser"; import { sha256 } from "js-sha256"; -import { LANGUAGE_CONFIGS } from "./shared/lexer"; +import { getLanguageConfig, LanguageLexerConfig } from "./shared/lexer"; //==================================================================== interface TrackedDocument { @@ -68,7 +68,7 @@ export class ScriptSync implements vscode.Disposable { // Create macro processor first this.language = language; - this.macros = new MacroProcessor(this.language); + this.macros = new MacroProcessor(); this.initializeSystemMacros(language); this.host = host ?? new VSCodeHost(); @@ -398,10 +398,12 @@ export class ScriptSync implements vscode.Disposable { console.log(`Preprocessing enabled for: ${baseName}`); this.macros.clearNonSystemMacros(); + const languageConfig = this.getLanuageConfig(); + console.error("LANG CONFIG", languageConfig); preprocessorResult = await this.preprocessor.process( originalContent, normalizePath(masterFilePath), - this.language + languageConfig, ); if (preprocessorResult.issues && preprocessorResult.issues.length > 0) { @@ -439,6 +441,14 @@ export class ScriptSync implements vscode.Disposable { return finalContent; } + private getLanuageConfig(): LanguageLexerConfig { + const config = getLanguageConfig(this.language); + if(config.name === "lsl" && this.config.getConfig(ConfigKey.PreprocessorLSLSwitchStatements, false)) { + config.directiveKeywords.push("switch"); + } + return config; + } + public async handleMasterSaved(): Promise { try { // Read the original content @@ -494,7 +504,7 @@ export class ScriptSync implements vscode.Disposable { const path = vscode.workspace.asRelativePath(this.masterDocument.uri.fsPath); - const comment = LANGUAGE_CONFIGS[this.language].lineCommentPrefix; + const comment = this.getLanuageConfig().lineCommentPrefix; meta.push(`${comment} ================ sl-vscode-plugin meta ================`); meta.push(`${comment} @file ${path}`); meta.push(`${comment} @hash ${hash}`); diff --git a/src/shared/conditionalprocessor.ts b/src/shared/conditionalprocessor.ts index 4488034..f81c8eb 100644 --- a/src/shared/conditionalprocessor.ts +++ b/src/shared/conditionalprocessor.ts @@ -7,8 +7,7 @@ * with proper nesting and state tracking. */ -import { Token, TokenType, getLanguageConfig, type LanguageLexerConfig } from './lexer'; -import { ScriptLanguage } from './languageservice'; +import { Token, TokenType, type LanguageLexerConfig } from './lexer'; import type { MacroProcessor } from './macroprocessor'; import { PreprocessorDiagnostic, DiagnosticLocation, ErrorCodes } from './diagnostics'; import { NormalizedPath } from '../interfaces/hostinterface'; @@ -66,12 +65,10 @@ export interface ConditionalResult { */ export class ConditionalProcessor { private stack: ConditionalBlock[] = []; - private language: ScriptLanguage; - private config: LanguageLexerConfig; + private language: LanguageLexerConfig; - constructor(language: ScriptLanguage) { + constructor(language: LanguageLexerConfig) { this.language = language; - this.config = getLanguageConfig(language); } //#region Public API @@ -520,7 +517,7 @@ export class ConditionalProcessor { private evaluateLogicalOr(tokens: Token[], pos: number): { value: number; pos: number } { let result = this.evaluateLogicalAnd(tokens, pos); - const orOp = this.config.logicalOperators.or; + const orOp = this.language.logicalOperators.or; while (result.pos < tokens.length) { const token = tokens[result.pos]; @@ -548,7 +545,7 @@ export class ConditionalProcessor { private evaluateLogicalAnd(tokens: Token[], pos: number): { value: number; pos: number } { let result = this.evaluateComparison(tokens, pos); - const andOp = this.config.logicalOperators.and; + const andOp = this.language.logicalOperators.and; while (result.pos < tokens.length) { const token = tokens[result.pos]; @@ -692,7 +689,7 @@ export class ConditionalProcessor { } const token = tokens[pos]; - const notOp = this.config.logicalOperators.not; + const notOp = this.language.logicalOperators.not; // Handle unary minus if (token.type === TokenType.OPERATOR && token.value === "-") { diff --git a/src/shared/diagnostics.ts b/src/shared/diagnostics.ts index 59e3d57..33bff31 100644 --- a/src/shared/diagnostics.ts +++ b/src/shared/diagnostics.ts @@ -53,6 +53,12 @@ export interface DiagnosticLocation { sourceFile: NormalizedPath; } +export class DiagnosticError extends Error { + constructor(public diagnostic: PreprocessorDiagnostic) { + super(diagnostic.message); + } +} + //#endregion //#region Diagnostic Collector diff --git a/src/shared/includeprocessor.ts b/src/shared/includeprocessor.ts index e4de3ac..8d1d6ac 100644 --- a/src/shared/includeprocessor.ts +++ b/src/shared/includeprocessor.ts @@ -11,8 +11,7 @@ */ import { NormalizedPath, HostInterface, normalizeJoinPath, normalizePath } from '../interfaces/hostinterface'; -import { ScriptLanguage } from './languageservice'; -import { Lexer, Token } from './lexer'; +import { LanguageLexerConfig, Lexer, Token } from './lexer'; import { MacroProcessor } from './macroprocessor'; import { ConditionalProcessor } from './conditionalprocessor'; import { DiagnosticCollector, DiagnosticSeverity, ErrorCodes } from './diagnostics'; @@ -63,10 +62,10 @@ export interface IncludeState { * Processor for handling include directives */ export class IncludeProcessor { - private language: ScriptLanguage; + private language: LanguageLexerConfig; private host: HostInterface; - constructor(language: ScriptLanguage, host: HostInterface) { + constructor(language: LanguageLexerConfig, host: HostInterface) { this.language = language; this.host = host; } @@ -124,7 +123,7 @@ export class IncludeProcessor { } // Resolve the include file path - const extensions = this.language === "lsl" ? ["lsl"] : ["luau", "lua"]; + const extensions = this.language.name === "lsl" ? ["lsl"] : ["luau", "lua"]; let includePaths: string[] = []; let aliased = false; @@ -176,7 +175,7 @@ export class IncludeProcessor { ); // console.error("Resolve: ", [filename, sourceFile, extensions, includePaths, aliased, allowExternal], resolvedPath); - if(!resolvedPath && this.language == "luau") { + if(!resolvedPath && this.language.name == "luau") { // Luau require supports default file in folder include mechanic 'init.luau' if(!filename.toLowerCase().endsWith(".luau") && !filename.toLocaleLowerCase().endsWith(".lua")) { filename += (filename.length ? path.sep : "") + "init"; diff --git a/src/shared/lexer.ts b/src/shared/lexer.ts index c1a7c7d..2da9b9e 100644 --- a/src/shared/lexer.ts +++ b/src/shared/lexer.ts @@ -25,7 +25,7 @@ type RegexMultiCharDelimiter = { type MultiCharDelimiter = RegexMultiCharDelimiter|[string,string]; -export interface LanguageLexerConfig { +interface BaseLanguageLexerConfig { lineCommentPrefix: string; // Line comment prefix (e.g., "//" for LSL, "--" for Luau) blockCommentDelimiters: Array; // block comment delimiters (e.g., "/*" for LSL, "--[[" for Luau) logicalOperators: { @@ -49,11 +49,21 @@ export interface LanguageLexerConfig { } } +export interface CustomLanguageLexerConfig extends BaseLanguageLexerConfig { + name: "custom"; +} + +export interface LanguageLexerConfig extends BaseLanguageLexerConfig { + name: ScriptLanguage; // Language name +} + + /** * Predefined language configurations */ -export const LANGUAGE_CONFIGS: Record = { +const LANGUAGE_CONFIGS: Record = { lsl: { + name: "lsl", lineCommentPrefix: "//", blockCommentDelimiters: [ ["/*","*/"] @@ -84,6 +94,7 @@ export const LANGUAGE_CONFIGS: Record = { stringDelimiters: ['"', "'"], // Double and single quotes }, luau: { + name: "luau", lineCommentPrefix: "--", blockCommentDelimiters: [ { @@ -140,7 +151,7 @@ export const LANGUAGE_CONFIGS: Record = { * Get language configuration for a script language */ export function getLanguageConfig(language: ScriptLanguage): LanguageLexerConfig { - return LANGUAGE_CONFIGS[language]; + return structuredClone(LANGUAGE_CONFIGS[language]); } //#endregion @@ -210,6 +221,14 @@ export class Token { return this.value; } + is(type: TokenType, value: string): boolean { + return this.isType(type) && this.value === value; + } + + isType(type: TokenType): boolean { + return this.type === type; + } + /** * Check if this token is a bracket (opening or closing) */ @@ -354,19 +373,18 @@ export class Lexer { private position: number; private context: LexerContext; private tokens: Token[]; - private config: LanguageLexerConfig; + private config: BaseLanguageLexerConfig; private sourceFile: NormalizedPath; private diagnostics: DiagnosticCollector; - constructor(source: string, language: ScriptLanguage, sourceFile?: NormalizedPath, diagnostics?: DiagnosticCollector); - constructor(source: string, config: LanguageLexerConfig, sourceFile?: NormalizedPath, diagnostics?: DiagnosticCollector); constructor( source: string, - languageOrConfig: ScriptLanguage | LanguageLexerConfig, + languageConfig: BaseLanguageLexerConfig, sourceFile?: NormalizedPath, diagnostics?: DiagnosticCollector ) { this.source = source; + this.config = languageConfig; this.position = 0; this.context = { inBlockComment: null, @@ -378,13 +396,6 @@ export class Lexer { this.tokens = []; this.sourceFile = sourceFile || ("" as NormalizedPath); this.diagnostics = diagnostics || new DiagnosticCollector(); - - // Support both language string and config object - if (typeof languageOrConfig === 'string') { - this.config = getLanguageConfig(languageOrConfig); - } else { - this.config = languageOrConfig; - } } /** diff --git a/src/shared/lexingpreprocessor.ts b/src/shared/lexingpreprocessor.ts index 704ff9f..3e48791 100644 --- a/src/shared/lexingpreprocessor.ts +++ b/src/shared/lexingpreprocessor.ts @@ -14,7 +14,7 @@ import { ScriptLanguage } from "./languageservice"; import { NormalizedPath, HostInterface } from "../interfaces/hostinterface"; import { FullConfigInterface, ConfigKey } from "../interfaces/configinterface"; -import { Lexer } from "./lexer"; +import { LanguageLexerConfig, Lexer } from "./lexer"; import { IncludeInfo, Parser } from "./parser"; import { MacroProcessor } from "./macroprocessor"; import { DiagnosticSeverity } from "./diagnostics"; @@ -36,6 +36,7 @@ export interface PreprocessorOptions { flags: OptionFlags; includePaths?: string[]; maxIncludeDepth?: number; // Maximum nesting depth for #include and require() directives + config?: { [key in ConfigKey]?: any }; } export interface PreprocessorError { @@ -75,6 +76,7 @@ export interface DirectiveImplementations { endif?: (parser: any) => void | Promise; include?: (parser: any) => void | Promise; require?: (parser: any) => void | Promise; + switch?: (parser: any) => void | Promise; } //#region Lexing Preprocessor Main Class @@ -99,7 +101,7 @@ export class LexingPreprocessor { * that should be available in all scripts of the given language. */ private initializePredefinedMacros(language: ScriptLanguage): MacroProcessor { - const macros = new MacroProcessor(language); + const macros = new MacroProcessor(); // Add standard predefined macros based on language if (language === 'lsl') { @@ -121,7 +123,7 @@ export class LexingPreprocessor { public async process( source: string, sourceFile: NormalizedPath, - language: ScriptLanguage + language: LanguageLexerConfig ): Promise { // Check if preprocessing is enabled const enabled = this.config.getConfig(ConfigKey.PreprocessorEnable) ?? true; @@ -129,7 +131,7 @@ export class LexingPreprocessor { return { content: source, success: true, - language, + language: language.name, issues: [], }; } @@ -153,7 +155,7 @@ export class LexingPreprocessor { const includePaths = this.config.getConfig(ConfigKey.PreprocessorIncludePaths) ?? ['.']; // Use provided macros or initialize predefined macros - const predefinedMacros = this.macros ?? this.initializePredefinedMacros(language); + const predefinedMacros = this.macros ?? this.initializePredefinedMacros(language.name); // Create initial parser state with predefined macros const initialState = Parser.createInitialState( @@ -161,7 +163,7 @@ export class LexingPreprocessor { this.fs, predefinedMacros, maxIncludeDepth, - language == "lsl" ? includePaths : undefined + language.name == "lsl" ? includePaths : undefined ); // Phase 2: Parsing and directive processing @@ -205,7 +207,7 @@ export class LexingPreprocessor { return { content: result.success ? result.source : source, // Return original source on error, processed source on success success: result.success, // Use parser's success determination - language, + language: language.name, lineMappings: result.mappings, issues, includes: result.includes, @@ -216,7 +218,7 @@ export class LexingPreprocessor { return { content: source, success: false, - language, + language: language.name, issues: [{ message: `Preprocessing failed: ${error instanceof Error ? error.message : String(error)}`, lineNumber: 1, diff --git a/src/shared/macroprocessor.ts b/src/shared/macroprocessor.ts index 57e9e51..3dc5943 100644 --- a/src/shared/macroprocessor.ts +++ b/src/shared/macroprocessor.ts @@ -23,7 +23,6 @@ */ import { Token, TokenType } from './lexer'; -import { ScriptLanguage } from './languageservice'; import { DiagnosticCollector, ErrorCodes, DiagnosticSeverity } from './diagnostics'; import { NormalizedPath } from '../interfaces/hostinterface'; @@ -62,12 +61,10 @@ export interface MacroDefinition { */ export class MacroProcessor { private macros: Map; - private language: ScriptLanguage; private enabled: boolean; - constructor(language: ScriptLanguage) { + constructor() { this.macros = new Map(); - this.language = language; this.enabled = true; } @@ -403,8 +400,29 @@ export class MacroProcessor { return null; } - // Get parameter names - const parameters = macro.parameters || []; + // Get parameter names -- as a new array so that we can modify it without changing the original. + const parameters = [... (macro.parameters || [])]; + + // Support for variadic function expansions. e.g. + // #define func(...) llOwnerSay((string)[__VA_ARGS__])) + const variadic = parameters[parameters.length - 1] == "..."; + if(variadic) { + parameters[parameters.length - 1] = "__VA_ARGS__"; + if(args.length > parameters.length) { + // Cut off variadic args, and combine them seperated by `, ` as a single argument + const varArgs = args.splice(parameters.length - 1); + const tokens = []; + let first = true; + for(const tokes of varArgs) { + if(!first) { + tokens.push(new Token(TokenType.PUNCTUATION, ",", 0,0,1)); + tokens.push(new Token(TokenType.WHITESPACE, " ", 0,0,1)); + } else first = false; + tokens.push(...tokes); + } + args.push(tokens); + } + } // MAC002: Validate argument count if (args.length !== parameters.length) { diff --git a/src/shared/parser.ts b/src/shared/parser.ts index a31b613..0076963 100644 --- a/src/shared/parser.ts +++ b/src/shared/parser.ts @@ -7,15 +7,14 @@ */ import * as path from 'path'; -import { Token, TokenType, getLanguageConfig } from './lexer'; -import { ScriptLanguage } from './languageservice'; +import { LanguageLexerConfig, Token, TokenType } from './lexer'; import { NormalizedPath, HostInterface } from '../interfaces/hostinterface'; import { FullConfigInterface, ConfigKey } from '../interfaces/configinterface'; import type { DirectiveImplementations } from './lexingpreprocessor'; import { MacroProcessor, MacroExpansionContext } from './macroprocessor'; import { ConditionalProcessor } from './conditionalprocessor'; import { IncludeProcessor, IncludeState } from './includeprocessor'; -import { DiagnosticCollector, PreprocessorDiagnostic, ErrorCodes } from './diagnostics'; +import { DiagnosticCollector, PreprocessorDiagnostic, ErrorCodes, DiagnosticError, DiagnosticSeverity } from './diagnostics'; //#region Parser State @@ -47,6 +46,10 @@ export interface ParserState { includeState: IncludeState; /** Require state (for SLua require() directives) - only present when require is supported */ requireState?: RequireState; + /** Set of unique identifiers to use for naming things like switch/loop jumps */ + uniqueidentifiers: Set; + /** Random number state, for deterministic random number generation */ + random: RNG; } //#endregion @@ -102,6 +105,17 @@ export interface MacroInfo { parameters?: string[]; } +class HaltParseError extends Error{} + +type CaseBlock = { + defaultCase: boolean; + condition: Token[]; + body: Token[]; + identifier: string; + commentsBefore: Token[]; + commentsAfter: Token[]; +} + //#endregion //#region Parser @@ -114,7 +128,7 @@ export class Parser { private position: number; private state: ParserState; private sourceFile: NormalizedPath; - private language: ScriptLanguage; + private language: LanguageLexerConfig; // Output accumulators private outputTokens: Token[]; @@ -152,11 +166,12 @@ export class Parser { // Flag whether external require is allowed, passed down when using .luarc private allowExternalRequires: boolean = false; + private indentationLevel: number = 0; constructor( tokens: Token[], sourceFile: NormalizedPath, - language: ScriptLanguage, + language: LanguageLexerConfig, host?: HostInterface, directives?: DirectiveImplementations, initialState?: Partial, @@ -188,16 +203,18 @@ export class Parser { // Initialize parser state this.state = { - macros: initialState?.macros || new MacroProcessor(language), - conditionals: initialState?.conditionals || new ConditionalProcessor(language), - includes: initialState?.includes || (host ? new IncludeProcessor(language, host) : undefined as any), + macros: initialState?.macros || new MacroProcessor(), + conditionals: initialState?.conditionals || new ConditionalProcessor(this.language), + includes: initialState?.includes || (host ? new IncludeProcessor(this.language, host) : undefined as any), includeState: initialState?.includeState || IncludeProcessor.createState(maxIncludeDepth, includePaths), + uniqueidentifiers: initialState?.uniqueidentifiers || new Set(), + random: initialState?.random || new RNG(), }; // Only initialize requireState for SLua (luau) files or if explicitly provided if (initialState?.requireState !== undefined) { this.state.requireState = initialState.requireState; - } else if (language === 'luau' && isTopLevel) { + } else if (this.language.name === 'luau' && isTopLevel) { this.state.requireState = Parser.createRequireState(); } @@ -233,6 +250,7 @@ export class Parser { endif: async (parser: Parser) => Parser.handleEndifDirective(parser), include: async (parser: Parser) => Parser.handleIncludeDirective(parser), require: async (parser: Parser) => Parser.handleRequireDirective(parser), + switch: async (parser: Parser) => Parser.handleSwitchDirective(parser), }; } @@ -253,17 +271,19 @@ export class Parser { * This allows the preprocessor to inject predefined macros before parsing begins */ public static createInitialState( - language: ScriptLanguage, + language: LanguageLexerConfig, host?: HostInterface, macros?: MacroProcessor, maxIncludeDepth: number = 5, includePaths: string[] = ['.'] ): ParserState { return { - macros: macros ?? new MacroProcessor(language), + macros: macros ?? new MacroProcessor(), conditionals: new ConditionalProcessor(language), includes: host ? new IncludeProcessor(language, host) : undefined as any, includeState: IncludeProcessor.createState(maxIncludeDepth, includePaths), + uniqueidentifiers: new Set(), + random: new RNG(), }; } @@ -287,18 +307,16 @@ export class Parser { while (!this.isAtEnd()) { const token = this.current(); - if (token.isDirective()) { - const positionAdvanced = await this.handleDirective(token); - if (!positionAdvanced) { - this.advance(); - } - } else if (this.shouldEmitToken()) { - const positionAdvanced = this.emitToken(token); - if (!positionAdvanced) { - this.advance(); + try { + await this.parseToken(token); + } catch(error) { + if(error instanceof HaltParseError) { + // Stop processing on HaltParseError + } else if(error instanceof DiagnosticError) { + this.diagnostics.add(error.diagnostic); + } else { + throw error; } - } else { - this.advance(); } // Stop processing immediately if we encounter any errors @@ -348,6 +366,25 @@ export class Parser { }; } + private async parseToken(token: Token): Promise { + if (token.isDirective()) { + const positionAdvanced = await this.handleDirective(token); + if (!positionAdvanced) { + this.advance(); + } + } else if (this.shouldEmitToken()) { + const positionAdvanced = this.emitToken(token); + if (!positionAdvanced) { + this.advance(); + } + } else { + this.advance(); + } + if(this.diagnostics.hasErrors()) { + throw new HaltParseError(); + } + } + //#region Directive Handling /** @@ -384,7 +421,7 @@ export class Parser { } // Consume rest of directive line (but not for require, which is inline) - if (directiveName !== 'require') { + if (directiveName !== 'require' && directiveName !== 'switch') { this.consumeDirectiveLine(); return false; // Let caller advance past the directive token } else { @@ -710,6 +747,257 @@ export class Parser { } } + /** + * Handle switch directive (LSL) + */ + private static handleSwitchDirective(parser: Parser): void { + const directiveToken = parser.current(); + parser.advance(); + + const indentation = parser.indentationLevel; + const indentationWhitespace = ' '.repeat(indentation); + parser.skipWhitespace(); + parser.consumeTokenOfType(TokenType.PAREN_OPEN, 'SWITCH directive'); + const condition :Token[] = []; + let depth = 1; + while(!parser.isAtEnd()) { + const current = parser.current(); + if(current.type === TokenType.PAREN_OPEN) { + depth++; + } else if(current.type === TokenType.PAREN_CLOSE) { + depth--; + if(depth < 1) { + parser.advance(); + break; + } + } + condition.push(current); + parser.advance(); + } + const commentsBefore = this.trimTrailingWhiteSpace(parser.consumeWhiteSpaceAndComments()); + + parser.consumeTokenOfType(TokenType.BRACE_OPEN, 'SWITCH directive'); + parser.skipWhitespace(); + + let caseBlock = this.consumeCaseBlock(parser); + let cases = []; + while(caseBlock) { + cases.push(caseBlock); + caseBlock = this.consumeCaseBlock(parser); + } + + if(cases.length < 1) { + throw new DiagnosticError({ + severity: DiagnosticSeverity.ERROR, + message: `SWITCH directive requires at least one CASE block`, + line: directiveToken.line, + column: directiveToken.column, + length: directiveToken.value.length, + sourceFile: parser.sourceFile, + }); + } + + parser.consumeTokenOfType(TokenType.BRACE_CLOSE, 'SWITCH directive'); + + if(cases.filter(c=>c.defaultCase).length > 1) { + throw new DiagnosticError({ + severity: DiagnosticSeverity.ERROR, + message: `SWITCH directive cannot have more than one DEFAULT case`, + line: directiveToken.line, + column: directiveToken.column, + length: directiveToken.value.length, + sourceFile: parser.sourceFile, + }); + } + const defaultCase = cases.find(c=>c.defaultCase); + const outJump = parser.generateUniqueIdentifier(5,"s"); + if(commentsBefore.length > 0) { + parser.emitTokens(commentsBefore); + parser.emitToken(new Token(TokenType.NEWLINE, "\n", 0,0,0)); + } + let first = true; + for(const c of cases) { + if(c.defaultCase) continue; + if(!first)parser.emitToken(new Token(TokenType.WHITESPACE, indentationWhitespace, 0,0,0)); + first = false; + this.emitCaseIfStatement(parser,condition,c); + } + + if(!first)parser.emitToken(new Token(TokenType.WHITESPACE, indentationWhitespace, 0,0,0)); + parser.emitToken(new Token(TokenType.IDENTIFIER, `jump`, 0,0,0)); + parser.emitToken(new Token(TokenType.WHITESPACE, " ", 0,0,0)); + parser.emitToken(new Token(TokenType.IDENTIFIER, defaultCase ? defaultCase.identifier : outJump, 0,0,0)); + parser.emitToken(new Token(TokenType.PUNCTUATION, ";", 0,0,0)); + if(defaultCase) parser.emitTokens(defaultCase.commentsBefore); + parser.emitToken(new Token(TokenType.NEWLINE, "\n", 0,0,0)); + + for(const c of cases) { + this.emitCaseBlock(parser,c,outJump,indentationWhitespace); + } + parser.emitToken(new Token(TokenType.WHITESPACE, indentationWhitespace, 0,0,0)); + parser.emitToken(new Token(TokenType.IDENTIFIER, `@${outJump}`, 0,0,0)); + parser.emitToken(new Token(TokenType.PUNCTUATION, ";", 0,0,0)); + // parser.emitToken(new Token(TokenType.NEWLINE, "\n", 0,0,0)); + } + + private static emitCaseBlock(parser: Parser, caseBlock: CaseBlock, outJump: string, indentationWhitespace: string): void { + parser.emitToken(new Token(TokenType.WHITESPACE, indentationWhitespace, 0,0,0)); + parser.emitToken(new Token(TokenType.IDENTIFIER, `@${caseBlock.identifier}`, 0,0,0)); + parser.emitToken(new Token(TokenType.PUNCTUATION, ";", 0,0,0)); + parser.emitTokens(caseBlock.commentsBefore); + parser.emitToken(new Token(TokenType.NEWLINE, "\n", 0,0,0)); + if(caseBlock.body.filter(t=>!t.isWhitespaceOrNewline()).length < 1) return; + parser.emitToken(new Token(TokenType.WHITESPACE, indentationWhitespace, 0,0,0)); + parser.emitToken(new Token(TokenType.BRACE_OPEN, "{", 0,0,0)); + parser.emitToken(new Token(TokenType.NEWLINE, "\n", 0,0,0)); + // parser.emitToken(new Token(TokenType.WHITESPACE, indentationWhitespace, 0,0,0)); + let indentation = 0; + let lastNewLine = true; + for(const t of caseBlock.body) { + if(t.is(TokenType.IDENTIFIER,"break")) { + parser.emitToken(new Token(TokenType.IDENTIFIER, `jump`, 0,0,0)); + parser.emitToken(new Token(TokenType.WHITESPACE, " ", 0,0,0)); + parser.emitToken(new Token(TokenType.IDENTIFIER, outJump, 0,0,0)); + continue; + } + if(lastNewLine) { + if(!t.isType(TokenType.WHITESPACE)) { + if (indentation == 0) { + parser.emitToken(new Token(TokenType.WHITESPACE, indentationWhitespace + ' ', 0,0,0)); + } + lastNewLine = false; + } else { + t.value = t.value.substring(4); + } + } + parser.emitToken(t); + if(lastNewLine && t.type == TokenType.WHITESPACE) { + indentation += t.value.length; + } + if(t.isType(TokenType.NEWLINE)) { + indentation = 0; + lastNewLine = true; + } + } + // parser.emitToken(new Token(TokenType.NEWLINE, "\n", 0,0,0)); + // parser.emitToken(new Token(TokenType.WHITESPACE, indentationWhitespace, 0,0,0)); + parser.emitToken(new Token(TokenType.BRACE_CLOSE, "}", 0,0,0)); + parser.emitTokens(caseBlock.commentsAfter); + parser.emitToken(new Token(TokenType.NEWLINE, "\n", 0,0,0)); + } + + private static emitCaseIfStatement(parser: Parser, condition: Token[], caseBlock: CaseBlock): void { + parser.emitToken(new Token(TokenType.IDENTIFIER, `if`, 0, 0, 0)); + parser.emitToken(new Token(TokenType.PAREN_OPEN, "(", 0, 0, 0)); + parser.emitToken(new Token(TokenType.PAREN_OPEN, "(", 0, 0, 0)); + + parser.emitTokens(condition); + + parser.emitToken(new Token(TokenType.PAREN_CLOSE, ")", 0, 0, 0)); + parser.emitToken(new Token(TokenType.WHITESPACE, " ", 0, 0, 0)); + parser.emitToken(new Token(TokenType.OPERATOR, "==", 0, 0, 0)); + parser.emitToken(new Token(TokenType.WHITESPACE, " ", 0, 0, 0)); + parser.emitToken(new Token(TokenType.PAREN_OPEN, "(", 0, 0, 0)); + + parser.emitTokens(caseBlock.condition); + + parser.emitToken(new Token(TokenType.PAREN_CLOSE, ")", 0, 0, 0)); + parser.emitToken(new Token(TokenType.PAREN_CLOSE, ")", 0, 0, 0)); + parser.emitToken(new Token(TokenType.WHITESPACE, " ", 0, 0, 0)); + parser.emitToken(new Token(TokenType.IDENTIFIER, "jump", 0, 0, 0)); + parser.emitToken(new Token(TokenType.WHITESPACE, " ", 0, 0, 0)); + parser.emitToken(new Token(TokenType.IDENTIFIER, caseBlock.identifier, 0, 0, 0)); + parser.emitToken(new Token(TokenType.PUNCTUATION, ";", 0, 0, 0)); + parser.emitToken(new Token(TokenType.NEWLINE, "\n", 0, 0, 0)); + } + + private static consumeCaseBlock(parser: Parser): CaseBlock | null { + if(parser.isAtEnd()) return null; + parser.skipWhitespace(); + const keyword = parser.consumeTokenIfType(TokenType.IDENTIFIER); + if(!keyword) { + return null; + } + if(keyword.value !== 'case' && keyword.value !== 'default') { + throw new DiagnosticError({ + severity: DiagnosticSeverity.ERROR, + message: `Expected 'case' or 'default' keyword in SWITCH directive`, + line: keyword.line, + column: keyword.column, + length: keyword.value.length, + sourceFile: parser.sourceFile, + }); + } + const defaultCase = keyword.value === 'default'; + parser.skipWhitespace(); + const condition :Token[] = []; + while(!parser.isAtEnd()) { + const current = parser.current(); + if(current.type == TokenType.BRACE_OPEN) break; + if(current.is(TokenType.OPERATOR,":")) break; + condition.push(current); + parser.advance(); + } + if(parser.isAtEnd()) { + throw new DiagnosticError({ + severity: DiagnosticSeverity.ERROR, + message: `Unexpected end of file while parsing CASE block in SWITCH directive`, + line: keyword.line, + column: keyword.column, + length: keyword.value.length, + sourceFile: parser.sourceFile, + }); + } + + const commentsBefore = this.trimTrailingWhiteSpace(parser.consumeWhiteSpaceAndComments()); + let current = parser.current(); + + if(current.is(TokenType.OPERATOR, ":")) { + parser.advance(); + commentsBefore.push(...this.trimTrailingWhiteSpace(parser.consumeWhiteSpaceAndComments())); + current = parser.current(); + if(current.is(TokenType.IDENTIFIER, "case") || current.is(TokenType.IDENTIFIER, "default")) { + return { + identifier: parser.generateUniqueIdentifier(5,"c"), + defaultCase, + condition, + body: [], + commentsBefore, + commentsAfter:[], + }; + } + } + parser.consumeTokenOfType(TokenType.BRACE_OPEN, 'case condition'); + parser.advance(); + + let depth = 1; + const originalOutput = parser.outputTokens; + parser.outputTokens = []; + while(!parser.isAtEnd() && depth > 0) { + const current = parser.current(); + if(current.type === TokenType.BRACE_OPEN) { + depth++; + } else if(current.type === TokenType.BRACE_CLOSE) { + depth--; + if(depth === 0) { + parser.advance(); + break; + } + } + parser.parseToken(current); + } + const body = parser.outputTokens; + parser.outputTokens = originalOutput; + return { + identifier: parser.generateUniqueIdentifier(5,"c"), + defaultCase, + condition, + body, + commentsBefore, + commentsAfter: this.trimTrailingWhiteSpace(parser.consumeWhiteSpaceAndComments()), + }; + } + /** * Handle #if directive (LSL) */ @@ -808,6 +1096,12 @@ export class Parser { //#region Token Emission + private emitTokens(tokens: Token[]): void { + for (const token of tokens) { + this.emitToken(token); + } + } + /** * Emit a token to output and track line mapping * @returns true if the position was advanced beyond the current token (e.g., for function-like macros) @@ -824,6 +1118,7 @@ export class Parser { this.lastEmittedTokenType = TokenType.NEWLINE; this.currentOutputLine++; this.lineDirectiveEmittedForCurrentLine = false; // Reset for next line + this.indentationLevel = 0; return false; } @@ -836,6 +1131,11 @@ export class Parser { token.type !== TokenType.BLOCK_COMMENT_CONTENT && token.type !== TokenType.BLOCK_COMMENT_END; + if(isLeadingWhitespace) { + // Track indentation level + this.indentationLevel += token.value.length; + } + if ((isLeadingWhitespace || isMeaningfulToken) && !this.lineDirectiveEmittedForCurrentLine) { @@ -844,8 +1144,7 @@ export class Parser { // Insert @line directive if we skipped lines (gap > 1) or changed files if ((lineSkip > 1 || fileChanged) && this.lastSourceLine > 0) { - const languageConfig = getLanguageConfig(this.language); - const lineDirectiveText = `${languageConfig.lineCommentPrefix} @line ${token.line} "${this.formatPathForLineDirective(this.sourceFile)}"`; + const lineDirectiveText = `${this.language.lineCommentPrefix} @line ${token.line} "${this.formatPathForLineDirective(this.sourceFile)}"`; const lineDirective = new Token( TokenType.LINE_COMMENT, lineDirectiveText, @@ -985,14 +1284,23 @@ export class Parser { //#region Token Stream Navigation + private peek(offset: number): Token|null { + offset += this.position; + if(offset >= this.tokens.length) { + return null; + } + return this.tokens[offset]; + } + private current(): Token { return this.tokens[this.position]; } - private advance(): Token { + private advance(steps:number = 1): Token { const token = this.current(); - if (this.position < this.tokens.length - 1) { - this.position++; + steps += this.position; + if (steps < this.tokens.length) { + this.position = steps; } return token; } @@ -1008,6 +1316,92 @@ export class Parser { } } + private static trimTrailingWhiteSpace(tokens: Token[]): Token[] { + const out = []; + let white = []; + for(const token of tokens) { + if(token.isWhitespaceOrNewline()) { + white.push(token); + } else { + out.push(...white); + white = []; + out.push(token); + } + } + return out; + } + + private consumeWhiteSpaceAndComments(): Token[] { + const comments: Token[] = []; + while(!this.isAtEnd()) { + const current = this.current(); + if(current.isComment() || current.isWhitespaceOrNewline()) { + comments.push(current); + this.advance(); + continue; + } + break; + } + return comments; + } + + private generateUniqueIdentifier(len: number = 6, prefix: string = "u"): string { + let identifier: string = ""; + while(identifier.length < 1 || this.state.uniqueidentifiers.has(identifier)) { + const rand = this.state.random.nextHex(); + identifier = prefix + rand.substring(rand.length - len); + } + this.state.uniqueidentifiers.add(identifier); + return identifier; + } + + private checkTokenOfType(type: TokenType): boolean { + if (this.isAtEnd()) { + return false; + } + return this.current().type === type; + } + + private consumeTokenIfType(type: TokenType): Token | null { + if (this.isAtEnd()) { + return null; + } + const token = this.current(); + if (token.type === type) { + this.advance(); + return token; + } + return null; + } + + private consumeTokenOfType(type: TokenType, after?: string): Token { + if (this.isAtEnd()) { + const token = this.tokens[this.tokens.length - 1]; + throw new DiagnosticError({ + severity: DiagnosticSeverity.ERROR, + message: `Expected token of type ${type}${after ? ` after ${after}` : ""}, but reached end of input`, + line: token.line, + column: token.column, + length: token.value.length, + sourceFile: this.sourceFile, + }); + } + const token = this.current(); + if (token.type === type) { + this.advance(); + return token; + } else { + throw new DiagnosticError({ + severity: DiagnosticSeverity.ERROR, + message: `Expected token of type ${type}${after ? ` after ${after}` : ""}, got ${token.type}`, + line: token.line, + column: token.column, + length: token.value.length, + sourceFile: this.sourceFile, + }); + } + } + private consumeDirectiveLine(): void { // Consume tokens until end of line let consumed = 0; @@ -1034,8 +1428,9 @@ export class Parser { this.skipWhitespace(); while (!this.isAtEnd() && this.current().type !== TokenType.PAREN_CLOSE) { - if (this.current().isIdentifier()) { - parameters.push(this.current().value); + const current = this.current(); + if (current.isIdentifier()) { + parameters.push(current.value); this.advance(); this.skipWhitespace(); @@ -1044,6 +1439,17 @@ export class Parser { this.skipWhitespace(); } } else { + // Detect ... for __VA_ARGS__ + if(current.value == ".") { + const peek1 = this.peek(1); + const peek2 = this.peek(2); + if(peek1 && peek2) { + if(peek1.value == "." && peek2.value == ".") { + this.advance(2); + parameters.push("..."); + } + } + } this.advance(); // skip unexpected token } } @@ -1055,6 +1461,33 @@ export class Parser { return parameters; } + private consumeEncapsulatedSequence(enter: TokenType, exit: TokenType) : Token[] { + const first = this.consumeTokenOfType(enter); + const tokens = [first]; + let depth = 1; + while(!this.isAtEnd()) { + const current = this.current(); + if(current.type == enter) depth++; + else if(current.type == exit) depth--; + tokens.push(current); + this.advance(); + if(depth == 0) { + return tokens; + } + } + this.diagnostics.addError( + `Unclosed sequence wrapped with ${enter} ${exit}`, + { + line: first.line, + column: first.column, + length: first.value.length, + sourceFile: this.sourceFile, + }, + ErrorCodes.INVALID_MACRO_INVOCATION + ); + return []; + } + /** * Parse argument list for macro invocation: (expr1, expr2, expr3) */ @@ -1063,7 +1496,7 @@ export class Parser { let currentArg: Token[] = []; let parenDepth = 0; - this.advance(); // consume ( + this.consumeTokenOfType(TokenType.PAREN_OPEN, "function macro call"); // consume ( while (!this.isAtEnd()) { const token = this.current(); @@ -1086,6 +1519,10 @@ export class Parser { } parenDepth--; currentArg.push(token); + } else if(token.type === TokenType.BRACKET_OPEN) { + // Consume list + currentArg.push(...this.consumeEncapsulatedSequence(TokenType.BRACKET_OPEN, TokenType.BRACKET_CLOSE)); + continue; } else if (token.value === ',' && parenDepth === 0) { // Argument separator // Trim whitespace from argument @@ -1268,8 +1705,7 @@ export class Parser { this.diagnostics.merge(includeParser.diagnostics); // Add @line directive at the start of the included file's output - const languageConfig = getLanguageConfig(this.language); - const lineDirectiveText = `${languageConfig.lineCommentPrefix} @line 1 "${this.formatPathForLineDirective(result.resolvedPath!)}"`; + const lineDirectiveText = `${this.language.lineCommentPrefix} @line 1 "${this.formatPathForLineDirective(result.resolvedPath!)}"`; const lineDirective = new Token( TokenType.LINE_COMMENT, lineDirectiveText, @@ -1434,7 +1870,6 @@ export class Parser { */ private wrapModuleInFunction(moduleTokens: Token[], resolvedPath: NormalizedPath, lineNumber: number): Token[] { const wrapped: Token[] = []; - const languageConfig = getLanguageConfig(this.language); // Opening: (function() wrapped.push(new Token(TokenType.PAREN_OPEN, '(', lineNumber, 1, 1)); @@ -1444,7 +1879,7 @@ export class Parser { wrapped.push(new Token(TokenType.NEWLINE, this.lineEnding, lineNumber, 12, 1)); // Add @line directive - const lineDirectiveText = `${languageConfig.lineCommentPrefix} @line 1 "${this.formatPathForLineDirective(resolvedPath)}"`; + const lineDirectiveText = `${this.language.lineCommentPrefix} @line 1 "${this.formatPathForLineDirective(resolvedPath)}"`; wrapped.push(new Token(TokenType.LINE_COMMENT, lineDirectiveText, lineNumber + 1, 1, lineDirectiveText.length)); wrapped.push(new Token(TokenType.NEWLINE, this.lineEnding, lineNumber + 1, lineDirectiveText.length + 1, 1)); @@ -1489,8 +1924,7 @@ export class Parser { let outputLine = 1; let currentSourceFile: NormalizedPath = this.sourceFile; let currentSourceLine = 1; - const languageConfig = getLanguageConfig(this.language); - const lineDirectivePrefix = `${languageConfig.lineCommentPrefix} @line `; + const lineDirectivePrefix = `${this.language.lineCommentPrefix} @line `; for (let i = 0; i < this.outputTokens.length; i++) { const token = this.outputTokens[i]; @@ -1550,11 +1984,10 @@ export class Parser { * // Line 2 -> main.lsl:1 * // Line 4 -> math.lsl:3 */ - public static parseLineMappingsFromContent(content: string, language: ScriptLanguage = "lsl", host: HostInterface): LineMapping[] { + public static parseLineMappingsFromContent(content: string, language: LanguageLexerConfig, host: HostInterface): LineMapping[] { const lines = content.split('\n'); const lineMappings: LineMapping[] = []; - const languageConfig = getLanguageConfig(language); - const commentPrefix = `${languageConfig.lineCommentPrefix} @line`; + const commentPrefix = `${language.lineCommentPrefix} @line`; let currentSourceFile: NormalizedPath | null = null; let currentSourceLine = 1; @@ -1621,7 +2054,6 @@ export class Parser { return; // Nothing to do if requireState doesn't exist } - const languageConfig = getLanguageConfig(this.language); const sortedIds = Array.from(this.state.requireState.wrappedModules.keys()).sort((a, b) => a - b); const moduleTokens: ([TokenType,string]|Token)[] = []; for (const moduleId of sortedIds) { @@ -1629,7 +2061,7 @@ export class Parser { const modulePath = this.state.requireState.modulePathMap.get(moduleId); const modulePathFormatted = modulePath ? ` ${this.formatPathForLineDirective(modulePath)}` : ""; // Add module comment - moduleTokens.push([TokenType.LINE_COMMENT, `${languageConfig.lineCommentPrefix} @module ${moduleId}${modulePathFormatted}`]); + moduleTokens.push([TokenType.LINE_COMMENT, `${this.language.lineCommentPrefix} @module ${moduleId}${modulePathFormatted}`]); moduleTokens.push([TokenType.NEWLINE, "\n"]); // (function() end) moduleTokens.push(...wrappedModule); @@ -1749,7 +2181,7 @@ export class Parser { [TokenType.PAREN_CLOSE,")"], [TokenType.NEWLINE,"\n"], // [TokenType.LINE_COMMENT, `${languageConfig.lineCommentPrefix}@line 1 "${this.sourceFile}"`], - [TokenType.LINE_COMMENT, `${languageConfig.lineCommentPrefix}@line 1 "${this.formatPathForLineDirective(this.sourceFile)}"`], + [TokenType.LINE_COMMENT, `${this.language.lineCommentPrefix}@line 1 "${this.formatPathForLineDirective(this.sourceFile)}"`], [TokenType.NEWLINE,"\n"], ]; @@ -1777,4 +2209,28 @@ export class Parser { //#endregion } + +class RNG { + private state: number; + constructor(seed: number = 9863369152) { + // Ensure seed is treated as a 32-bit signed integer + this.state = seed | 0; + } + public next(): number { + // Xorshift* algorithm (32-bit integer arithmetic) + let x = this.state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + // Clamp back to 32-bit signed integer + x |= 0; + this.state = x; + return x & 0x7FFFFFFF; + } + + public nextHex(): string { + return this.next().toString(16).padStart(8, '0'); + } +} + //#endregion diff --git a/src/synchservice.ts b/src/synchservice.ts index 720ad33..07adc02 100644 --- a/src/synchservice.ts +++ b/src/synchservice.ts @@ -32,7 +32,7 @@ import { import { maybe } from "./shared/sharedutils"; // TODO: migrate needed utilities from sharedutils if required import { ScriptLanguage, LanguageService } from "./shared/languageservice"; import { ScriptSync } from "./scriptsync"; -import { LANGUAGE_CONFIGS } from "./shared/lexer"; +import { getLanguageConfig } from "./shared/lexer"; import { HostInterface } from "./interfaces/hostinterface"; type ParsedTempFile = { scriptName: string; scriptId: string; extension: string, language: ScriptLanguage}; @@ -616,7 +616,7 @@ export class SynchService implements vscode.Disposable { ): Promise { // Attempt to match by file meta info if(ConfigService.getInstance().getConfig(ConfigKey.FileMetaInfoInOutput, false)) { - const cmt = LANGUAGE_CONFIGS[script.language].lineCommentPrefix; + const cmt = getLanguageConfig(script.language).lineCommentPrefix; const lineRegExp = new RegExp(`^[\\s]*${cmt}[\\s]*@file[\\s]*[A-z0-9-_/.]*[\\s]*$`,"i"); const range = new vscode.Range(0,0,10,0); const start = viewerFile.getText(range).split("\n").filter(line => line.match(lineRegExp))[0] ?? null; diff --git a/src/test/suite/conditional-diagnostics.test.ts b/src/test/suite/conditional-diagnostics.test.ts index d336772..2367b54 100644 --- a/src/test/suite/conditional-diagnostics.test.ts +++ b/src/test/suite/conditional-diagnostics.test.ts @@ -6,7 +6,7 @@ import * as assert from 'assert'; import { ConditionalProcessor } from '../../shared/conditionalprocessor'; import { MacroProcessor } from '../../shared/macroprocessor'; -import { Lexer, Token, TokenType } from '../../shared/lexer'; +import { getLanguageConfig, Lexer, Token, TokenType } from '../../shared/lexer'; import { ErrorCodes, DiagnosticSeverity } from '../../shared/diagnostics'; import { normalizePath } from '../../interfaces/hostinterface'; @@ -14,6 +14,7 @@ suite('ConditionalProcessor Diagnostics', () => { let conditionals: ConditionalProcessor; let macros: MacroProcessor; const testFile = normalizePath('test.lsl'); + const lslLanguageConfig = getLanguageConfig('lsl'); /** * Helper to create a simple numeric token @@ -23,8 +24,8 @@ suite('ConditionalProcessor Diagnostics', () => { } setup(() => { - conditionals = new ConditionalProcessor('lsl'); - macros = new MacroProcessor('lsl'); + conditionals = new ConditionalProcessor(lslLanguageConfig); + macros = new MacroProcessor(); }); suite('#elif Diagnostics', () => { diff --git a/src/test/suite/conditionalprocessor.test.ts b/src/test/suite/conditionalprocessor.test.ts index 9f0cda3..764200a 100644 --- a/src/test/suite/conditionalprocessor.test.ts +++ b/src/test/suite/conditionalprocessor.test.ts @@ -6,11 +6,14 @@ import * as assert from 'assert'; import { ConditionalProcessor } from '../../shared/conditionalprocessor'; import { MacroProcessor, MacroDefinition } from '../../shared/macroprocessor'; -import { Token, TokenType, Lexer } from '../../shared/lexer'; +import { Token, TokenType, Lexer, getLanguageConfig } from '../../shared/lexer'; +import { ScriptLanguage } from '../../shared/languageservice'; suite('ConditionalProcessor (Lexing)', () => { let processor: ConditionalProcessor; let macros: MacroProcessor; + const lslLanguageConfig = getLanguageConfig('lsl'); + const luauLanguageConfig = getLanguageConfig('luau'); /** * Helper to create a simple token @@ -48,16 +51,17 @@ suite('ConditionalProcessor (Lexing)', () => { /** * Helper to tokenize an expression for testing */ - function createTokens(expression: string, language: 'lsl' | 'luau' = 'lsl'): Token[] { - const lexer = new Lexer(expression, language); + function createTokens(expression: string, language: ScriptLanguage = 'lsl'): Token[] { + const languageConfig = getLanguageConfig(language); + const lexer = new Lexer(expression, languageConfig); const allTokens = lexer.tokenize(); // Filter out EOF token return allTokens.filter(token => token.type !== TokenType.EOF); } setup(() => { - processor = new ConditionalProcessor('lsl'); - macros = new MacroProcessor('lsl'); + processor = new ConditionalProcessor(lslLanguageConfig); + macros = new MacroProcessor(); }); //#region Basic State Management @@ -754,8 +758,8 @@ suite('ConditionalProcessor (Lexing)', () => { suite('LSL Arithmetic Operations', () => { test('should evaluate addition', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('2 + 3'); const result = conditionals.processIf(tokens, macros, 1); @@ -764,8 +768,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate subtraction', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('10 - 5'); const result = conditionals.processIf(tokens, macros, 1); @@ -774,8 +778,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate multiplication', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('3 * 4'); const result = conditionals.processIf(tokens, macros, 1); @@ -784,8 +788,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate division', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('20 / 4'); const result = conditionals.processIf(tokens, macros, 1); @@ -794,8 +798,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate modulo', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('10 % 3'); const result = conditionals.processIf(tokens, macros, 1); @@ -804,8 +808,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should respect operator precedence', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); // 2 + 3 * 4 = 2 + 12 = 14 const tokens = createTokens('2 + 3 * 4'); @@ -815,8 +819,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle parentheses', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); // (2 + 3) * 4 = 5 * 4 = 20 const tokens = createTokens('(2 + 3) * 4'); @@ -826,8 +830,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle unary minus', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('-5 + 3'); const result = conditionals.processIf(tokens, macros, 1); @@ -838,8 +842,8 @@ suite('ConditionalProcessor (Lexing)', () => { suite('LSL Comparison Operations', () => { test('should evaluate equal (==)', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('5 == 5'); const result = conditionals.processIf(tokens, macros, 1); @@ -848,8 +852,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate not equal (!=)', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('5 != 3'); const result = conditionals.processIf(tokens, macros, 1); @@ -858,8 +862,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate less than (<)', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('3 < 5'); const result = conditionals.processIf(tokens, macros, 1); @@ -868,8 +872,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate greater than (>)', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('5 > 3'); const result = conditionals.processIf(tokens, macros, 1); @@ -878,8 +882,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate less than or equal (<=)', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('5 <= 5'); const result = conditionals.processIf(tokens, macros, 1); @@ -888,8 +892,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate greater than or equal (>=)', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('5 >= 5'); const result = conditionals.processIf(tokens, macros, 1); @@ -900,8 +904,8 @@ suite('ConditionalProcessor (Lexing)', () => { suite('LSL Logical Operations', () => { test('should evaluate logical AND (&&) - both true', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('1 && 1'); const result = conditionals.processIf(tokens, macros, 1); @@ -910,8 +914,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate logical AND (&&) - one false', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('1 && 0'); const result = conditionals.processIf(tokens, macros, 1); @@ -920,8 +924,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate logical OR (||) - one true', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('1 || 0'); const result = conditionals.processIf(tokens, macros, 1); @@ -930,8 +934,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate logical OR (||) - both false', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('0 || 0'); const result = conditionals.processIf(tokens, macros, 1); @@ -940,8 +944,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should evaluate logical NOT (!)', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('!0'); const result = conditionals.processIf(tokens, macros, 1); @@ -950,8 +954,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should respect logical operator precedence', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); // 1 || 0 && 0 = 1 || (0 && 0) = 1 || 0 = 1 const tokens = createTokens('1 || 0 && 0'); @@ -963,8 +967,8 @@ suite('ConditionalProcessor (Lexing)', () => { suite('LSL Complex Expressions', () => { test('should handle combined arithmetic and comparison', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('(2 + 3) > 4'); const result = conditionals.processIf(tokens, macros, 1); @@ -973,8 +977,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle nested parentheses', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('((2 + 3) * (4 - 1)) == 15'); const result = conditionals.processIf(tokens, macros, 1); @@ -983,8 +987,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle complex logical expression', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('(5 > 3) && (2 < 4) || (1 == 0)'); const result = conditionals.processIf(tokens, macros, 1); @@ -995,8 +999,8 @@ suite('ConditionalProcessor (Lexing)', () => { suite('LSL Macro Expansion in Expressions', () => { test('should expand macros in expressions', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); macros.define({ name: 'FOO', body: [numToken('5')], isFunctionLike: false }); macros.define({ name: 'BAR', body: [numToken('3')], isFunctionLike: false }); @@ -1008,8 +1012,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle macros in comparisons', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); macros.define({ name: 'VERSION', body: [numToken('2')], isFunctionLike: false }); const tokens = createTokens('VERSION >= 2'); @@ -1020,8 +1024,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle empty macro as 1 (true)', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const emptyDef: MacroDefinition = { name: 'EMPTY', body: [], @@ -1039,8 +1043,8 @@ suite('ConditionalProcessor (Lexing)', () => { suite('Luau Logical Operators', () => { test('should use "and" operator in Luau', () => { - const conditionals = new ConditionalProcessor('luau'); - const macros = new MacroProcessor('luau'); + const conditionals = new ConditionalProcessor(luauLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('1 and 1', 'luau'); const result = conditionals.processIf(tokens, macros, 1); @@ -1049,8 +1053,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should use "or" operator in Luau', () => { - const conditionals = new ConditionalProcessor('luau'); - const macros = new MacroProcessor('luau'); + const conditionals = new ConditionalProcessor(luauLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('0 or 1', 'luau'); const result = conditionals.processIf(tokens, macros, 1); @@ -1059,8 +1063,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should use "not" operator in Luau', () => { - const conditionals = new ConditionalProcessor('luau'); - const macros = new MacroProcessor('luau'); + const conditionals = new ConditionalProcessor(luauLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('not 0', 'luau'); const result = conditionals.processIf(tokens, macros, 1); @@ -1069,8 +1073,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should use ~= for inequality in Luau', () => { - const conditionals = new ConditionalProcessor('luau'); - const macros = new MacroProcessor('luau'); + const conditionals = new ConditionalProcessor(luauLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('5 ~= 3', 'luau'); const result = conditionals.processIf(tokens, macros, 1); @@ -1079,8 +1083,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle complex Luau expression', () => { - const conditionals = new ConditionalProcessor('luau'); - const macros = new MacroProcessor('luau'); + const conditionals = new ConditionalProcessor(luauLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('(5 > 3) and (2 < 4) or (1 == 0)', 'luau'); const result = conditionals.processIf(tokens, macros, 1); @@ -1091,8 +1095,8 @@ suite('ConditionalProcessor (Lexing)', () => { suite('Expression Error Handling', () => { test('should handle division by zero gracefully', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('1 / 0'); const result = conditionals.processIf(tokens, macros, 1); @@ -1102,8 +1106,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle missing closing parenthesis', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('(1 + 2'); const result = conditionals.processIf(tokens, macros, 1); @@ -1113,8 +1117,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle empty expression', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens(''); const result = conditionals.processIf(tokens, macros, 1); @@ -1125,8 +1129,8 @@ suite('ConditionalProcessor (Lexing)', () => { suite('Expression Integration with defined()', () => { test('should combine defined() with arithmetic', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); macros.define({ name: 'DEBUG', body: [numToken('1')], isFunctionLike: false }); const tokens = createTokens('defined(DEBUG) && 1'); @@ -1137,8 +1141,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should combine defined() with comparison', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); macros.define({ name: 'VERSION', body: [numToken('2')], isFunctionLike: false }); const tokens = createTokens('defined(VERSION) && VERSION >= 2'); @@ -1149,8 +1153,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle !defined() for undefined macro', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('!defined(UNDEFINED)'); const result = conditionals.processIf(tokens, macros, 1); @@ -1160,8 +1164,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle !defined() for defined macro', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); macros.define({ name: 'DEBUG', body: [], isFunctionLike: false }); const tokens = createTokens('!defined(DEBUG)'); @@ -1172,8 +1176,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle not defined() in Luau', () => { - const conditionals = new ConditionalProcessor('luau'); - const macros = new MacroProcessor('luau'); + const conditionals = new ConditionalProcessor(luauLanguageConfig); + const macros = new MacroProcessor(); const tokens = createTokens('not defined(UNDEFINED)', 'luau'); const result = conditionals.processIf(tokens, macros, 1); @@ -1183,8 +1187,8 @@ suite('ConditionalProcessor (Lexing)', () => { }); test('should handle complex expression with !defined()', () => { - const conditionals = new ConditionalProcessor('lsl'); - const macros = new MacroProcessor('lsl'); + const conditionals = new ConditionalProcessor(lslLanguageConfig); + const macros = new MacroProcessor(); macros.define({ name: 'DEBUG', body: [], isFunctionLike: false }); const tokens = createTokens('defined(DEBUG) && !defined(RELEASE)'); diff --git a/src/test/suite/diagnostic-integration.test.ts b/src/test/suite/diagnostic-integration.test.ts index b4d3e09..00f3a84 100644 --- a/src/test/suite/diagnostic-integration.test.ts +++ b/src/test/suite/diagnostic-integration.test.ts @@ -12,6 +12,7 @@ import * as assert from 'assert'; import { LexingPreprocessor, PreprocessorOptions } from '../../shared/lexingpreprocessor'; import { HostInterface, NormalizedPath, normalizePath } from '../../interfaces/hostinterface'; import { FullConfigInterface, ConfigKey } from '../../interfaces/configinterface'; +import { getLanguageConfig } from '../../shared/lexer'; /** * Mock configuration class for testing @@ -172,6 +173,8 @@ function createMockHostWithFiles(files: Map, options?: Preproces suite("Diagnostic Integration Test Suite", () => { + const lslLanguageConfig = getLanguageConfig('lsl'); + suite("Error Propagation Through Stack", () => { test("should collect lexer errors and propagate to preprocessor result", async () => { @@ -184,7 +187,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); assert.strictEqual(result.success, false, "Should fail due to lexer error"); @@ -205,7 +208,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); assert.strictEqual(result.success, false, "Should fail due to parser error"); @@ -226,7 +229,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); assert.strictEqual(result.success, false, "Should fail due to macro error"); @@ -248,7 +251,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); assert.strictEqual(result.success, false, "Should fail due to include error"); @@ -277,7 +280,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); assert.strictEqual(result.success, false, "Should fail due to conditional error"); @@ -299,7 +302,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); const diagnostic = result.issues[0]; @@ -322,7 +325,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); assert.strictEqual(result.success, false, "Should fail due to parser error"); @@ -348,7 +351,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); assert.strictEqual(result.success, false, "Should fail due to error"); @@ -368,14 +371,14 @@ default { state_entry() {} }`; const errorResult = await preprocessor.process( `#elif`, normalizePath('/test/first.lsl'), - 'lsl' + lslLanguageConfig ); // Second run without error const successResult = await preprocessor.process( `default { state_entry() {} }`, normalizePath('/test/second.lsl'), - 'lsl' + lslLanguageConfig ); assert.ok(errorResult.issues.length > 0, "First run should have errors"); diff --git a/src/test/suite/include-diagnostics.test.ts b/src/test/suite/include-diagnostics.test.ts index 2b1a8ca..34fe558 100644 --- a/src/test/suite/include-diagnostics.test.ts +++ b/src/test/suite/include-diagnostics.test.ts @@ -11,6 +11,7 @@ import { NormalizedPath, HostInterface, normalizePath } from '../../interfaces/h import { MacroProcessor } from '../../shared/macroprocessor'; import { ConditionalProcessor } from '../../shared/conditionalprocessor'; import { IncludeInfo } from '../../shared/parser'; +import { getLanguageConfig } from '../../shared/lexer'; const quickInclude = (file:string, line:number = 1) : IncludeInfo => { return { @@ -35,10 +36,12 @@ suite('IncludeProcessor Diagnostics', () => { let macros: MacroProcessor; let conditionals: ConditionalProcessor; + const lslLanguageConfig = getLanguageConfig('lsl'); + setup(() => { diagnostics = new DiagnosticCollector(); - macros = new MacroProcessor('lsl'); - conditionals = new ConditionalProcessor('lsl'); + macros = new MacroProcessor(); + conditionals = new ConditionalProcessor(lslLanguageConfig); }); suite('INC001: File Not Found', () => { @@ -50,7 +53,7 @@ suite('IncludeProcessor Diagnostics', () => { resolveFile: async (_filename: string, _from: NormalizedPath) => null }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); // When: Trying to include a non-existent file @@ -90,7 +93,7 @@ suite('IncludeProcessor Diagnostics', () => { } }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); // When: Including an existing file @@ -120,7 +123,7 @@ suite('IncludeProcessor Diagnostics', () => { resolveFile: async (_filename: string, _from: NormalizedPath) => circularPath }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); state.includeStack.push(circularPath); // Already in stack @@ -155,7 +158,7 @@ suite('IncludeProcessor Diagnostics', () => { resolveFile: async (_filename: string, _from: NormalizedPath) => includePath }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); // When: Including a file not in the stack @@ -185,7 +188,7 @@ suite('IncludeProcessor Diagnostics', () => { normalizePath("d:/test/deep.lsl") }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const maxDepth = 3; const state = IncludeProcessor.createState(maxDepth); state.includeDepth = maxDepth; // At max depth @@ -221,7 +224,7 @@ suite('IncludeProcessor Diagnostics', () => { resolveFile: async (_filename: string, _from: NormalizedPath) => includePath }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(5); state.includeDepth = 2; // Below max @@ -252,7 +255,7 @@ suite('IncludeProcessor Diagnostics', () => { resolveFile: async (_filename: string, _from: NormalizedPath) => errorPath }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); // When: Trying to include an unreadable file @@ -285,7 +288,7 @@ suite('IncludeProcessor Diagnostics', () => { resolveFile: async (_filename: string, _from: NormalizedPath) => readablePath }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); // When: Including a readable file @@ -314,7 +317,7 @@ suite('IncludeProcessor Diagnostics', () => { resolveFile: async (_filename: string, _from: NormalizedPath) => null }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(2); // When: Multiple failed includes @@ -339,7 +342,7 @@ suite('IncludeProcessor Diagnostics', () => { resolveFile: async (_filename: string, _from: NormalizedPath) => null }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); assert.ok(processor); assert.ok(typeof processor.processInclude === 'function'); }); @@ -379,7 +382,7 @@ suite('IncludeProcessor Diagnostics', () => { } }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); // When: Processing the chain @@ -424,7 +427,7 @@ suite('IncludeProcessor Diagnostics', () => { } }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); // When: A includes B, then B tries to include A again @@ -462,7 +465,7 @@ suite('IncludeProcessor Diagnostics', () => { } }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); // When: Following the diamond pattern @@ -503,7 +506,7 @@ suite('IncludeProcessor Diagnostics', () => { resolveFile: async (filename: string) => normalizePath(`d:/test/${filename}`) }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(maxDepth); // When: Including files up to max depth @@ -550,7 +553,7 @@ suite('IncludeProcessor Diagnostics', () => { } }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(3); // When: Processing multiple branches at different depths @@ -580,7 +583,7 @@ suite('IncludeProcessor Diagnostics', () => { resolveFile: async (filename: string) => normalizePath(`d:/test/${filename}`) }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(2); // When: Multiple includes exceed depth @@ -612,7 +615,7 @@ suite('IncludeProcessor Diagnostics', () => { } }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); // When: Processing mixed includes @@ -650,7 +653,7 @@ suite('IncludeProcessor Diagnostics', () => { } }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(2); // When: Triggering different error types @@ -689,7 +692,7 @@ suite('IncludeProcessor Diagnostics', () => { resolveFile: async () => null }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); // When: Multiple includes at different line numbers @@ -726,7 +729,7 @@ suite('IncludeProcessor Diagnostics', () => { } }; - const processor = new IncludeProcessor('lsl', host as HostInterface); + const processor = new IncludeProcessor(lslLanguageConfig, host as HostInterface); const state = IncludeProcessor.createState(); // When: Outer succeeds, middle succeeds, inner fails diff --git a/src/test/suite/include-disk-integration.test.ts b/src/test/suite/include-disk-integration.test.ts index 1deebe0..a37d6a0 100644 --- a/src/test/suite/include-disk-integration.test.ts +++ b/src/test/suite/include-disk-integration.test.ts @@ -10,6 +10,7 @@ import { LexingPreprocessor, PreprocessorOptions } from '../../shared/lexingprep import { normalizePath, type NormalizedPath, type HostInterface } from '../../interfaces/hostinterface'; import type { FullConfigInterface } from '../../interfaces/configinterface'; import { ConfigKey } from '../../interfaces/configinterface'; +import { getLanguageConfig } from '../../shared/lexer'; /** * Test config implementation @@ -178,6 +179,10 @@ suite('LSL Include Directive Tests - Disk-based Integration', () => { let workspaceRoot: string; let host: DiskTestHost; + const lslLanguageConfig = getLanguageConfig('lsl'); + const lslLanguageConfigWithSwitch = getLanguageConfig('lsl'); + lslLanguageConfigWithSwitch.directiveKeywords.push('switch'); + function createDefaultOptions(): PreprocessorOptions { return { enable: true, @@ -206,7 +211,7 @@ suite('LSL Include Directive Tests - Disk-based Integration', () => { const expected = fs.readFileSync(expectedFile, 'utf-8'); const preprocessor = new LexingPreprocessor(host, host.config); - const result = await preprocessor.process(source, testFile, 'lsl'); + const result = await preprocessor.process(source, testFile, lslLanguageConfig); // Compare with expected output assert.strictEqual(result.content, expected, 'Output should match expected file'); @@ -223,7 +228,7 @@ suite('LSL Include Directive Tests - Disk-based Integration', () => { const expected = fs.readFileSync(expectedFile, 'utf-8'); const preprocessor = new LexingPreprocessor(host, host.config); - const result = await preprocessor.process(source, testFile, 'lsl'); + const result = await preprocessor.process(source, testFile, lslLanguageConfig); // Compare with expected output assert.strictEqual(result.content, expected, 'Output should match expected file'); @@ -244,7 +249,7 @@ suite('LSL Include Directive Tests - Disk-based Integration', () => { const expected = fs.readFileSync(expectedFile, 'utf-8'); const preprocessor = new LexingPreprocessor(host, host.config); - const result = await preprocessor.process(source, testFile, 'lsl'); + const result = await preprocessor.process(source, testFile, lslLanguageConfig); // Compare with expected output assert.strictEqual(result.content, expected, 'Output should match expected file'); @@ -262,7 +267,7 @@ suite('LSL Include Directive Tests - Disk-based Integration', () => { const source = fs.readFileSync(testFile, 'utf-8'); const preprocessor = new LexingPreprocessor(host, host.config); - const result = await preprocessor.process(source, testFile, 'lsl'); + const result = await preprocessor.process(source, testFile, lslLanguageConfig); const lines = result.content.split('\n'); for (let i = 0; i < lines.length; i++) { @@ -287,7 +292,7 @@ suite('LSL Include Directive Tests - Disk-based Integration', () => { const source = fs.readFileSync(testFile, 'utf-8'); const preprocessor = new LexingPreprocessor(host, host.config); - const result = await preprocessor.process(source, testFile, 'lsl'); + const result = await preprocessor.process(source, testFile, lslLanguageConfig); // Verify we have mappings assert.ok(result.lineMappings && result.lineMappings.length > 0, 'Should have line mappings'); @@ -312,7 +317,7 @@ suite('LSL Include Directive Tests - Disk-based Integration', () => { const customHost = new DiskTestHost(workspaceRoot, options); const preprocessor = new LexingPreprocessor(customHost, customHost.config); - const result = await preprocessor.process(source, testFile, 'lsl'); + const result = await preprocessor.process(source, testFile, lslLanguageConfig); // Should fail due to max depth exceeded assert.strictEqual(result.success, false, 'Should fail when max depth is exceeded'); @@ -327,4 +332,61 @@ suite('LSL Include Directive Tests - Disk-based Integration', () => { ); assert.ok(depthError, 'Should have max depth exceeded error'); }); + + test('test switch case', async () => { + const testFile = normalizePath(path.join(workspaceRoot, 'test_switch.lsl')); + const expectedFile = path.join(workspaceRoot, 'test_switch_expected.lsl'); + const source = fs.readFileSync(testFile, 'utf-8'); + const expected = fs.readFileSync(expectedFile, 'utf-8'); + const preprocessor = new LexingPreprocessor(host, host.config); + + + const result = await preprocessor.process(source, testFile, lslLanguageConfigWithSwitch); + + + // Compare with expected output + assert.strictEqual(result.content, expected, 'Output should match expected file'); + + // Verify no errors + assert.ok(result.success, 'Processing should succeed'); + assert.strictEqual(result.issues.length, 0, 'Should have no issues'); + }); + + + test('test empty defined function macro handling', async () => { + const testFile = normalizePath(path.join(workspaceRoot, 'test_define_nix_function.lsl')); + const expectedFile = path.join(workspaceRoot, 'test_define_nix_function_expected.lsl'); + const source = fs.readFileSync(testFile, 'utf-8'); + const expected = fs.readFileSync(expectedFile, 'utf-8'); + const preprocessor = new LexingPreprocessor(host, host.config); + + const result = await preprocessor.process(source, testFile, lslLanguageConfig); + + // Compare with expected output + assert.strictEqual(result.content, expected, 'Output should match expected file'); + + // Verify no errors + assert.ok(result.success, 'Processing should succeed'); + assert.strictEqual(result.issues.length, 0, 'Should have no issues'); + }); + + const variants = ["debug", "no_debug"]; + for (const variant of variants) { + test('nested if and define directives should work correctly ' + variant, async () => { + const testFile = normalizePath(path.join(workspaceRoot, `test_nested_if_define_${variant}.lsl`)); + const expectedFile = path.join(workspaceRoot, `test_nested_if_define_${variant}_expected.lsl`); + const source = fs.readFileSync(testFile, 'utf-8'); + const expected = fs.readFileSync(expectedFile, 'utf-8'); + const preprocessor = new LexingPreprocessor(host, host.config); + + const result = await preprocessor.process(source, testFile, lslLanguageConfig); + + // Compare with expected output + assert.strictEqual(result.content.trim(), expected.trim(), 'Output should match expected file'); + + // Verify no errors + assert.ok(result.success, 'Processing should succeed'); + assert.strictEqual(result.issues.length, 0, 'Should have no issues'); + }); + } }); diff --git a/src/test/suite/lexer-diagnostics.test.ts b/src/test/suite/lexer-diagnostics.test.ts index 4a7de79..fad52fc 100644 --- a/src/test/suite/lexer-diagnostics.test.ts +++ b/src/test/suite/lexer-diagnostics.test.ts @@ -5,17 +5,19 @@ */ import * as assert from 'assert'; -import { Lexer, TokenType } from '../../shared/lexer'; +import { getLanguageConfig, Lexer, TokenType } from '../../shared/lexer'; import { DiagnosticCollector, DiagnosticSeverity, ErrorCodes } from '../../shared/diagnostics'; import { NormalizedPath } from '../../interfaces/hostinterface'; suite('Lexer Diagnostics', () => { const testFile = "test.lsl" as NormalizedPath; + const lslLanguageConfig = getLanguageConfig('lsl'); + const luauLanguageConfig = getLanguageConfig('luau'); test('Unterminated string - newline', () => { const source = '"unterminated string\n'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); const tokens = lexer.tokenize(); @@ -35,7 +37,7 @@ suite('Lexer Diagnostics', () => { test('Unterminated string - EOF', () => { const source = '"unterminated string'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); const tokens = lexer.tokenize(); @@ -52,7 +54,7 @@ suite('Lexer Diagnostics', () => { test('Properly terminated string - no error', () => { const source = '"properly terminated"'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); const tokens = lexer.tokenize(); @@ -67,7 +69,7 @@ suite('Lexer Diagnostics', () => { test('Unterminated block comment', () => { const source = '/* unterminated block comment\nmore lines\n'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); const tokens = lexer.tokenize(); @@ -88,7 +90,7 @@ suite('Lexer Diagnostics', () => { test('Properly terminated block comment - no error', () => { const source = '/* properly terminated */'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); const tokens = lexer.tokenize(); @@ -104,7 +106,7 @@ suite('Lexer Diagnostics', () => { test('Multiple errors in same file', () => { const source = '"unterminated\n/* also unterminated'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); lexer.tokenize(); @@ -194,7 +196,7 @@ suite('Lexer Diagnostics', () => { test('Invalid number literal - exponent without digits', () => { const source = 'float x = 123e;'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); const tokens = lexer.tokenize(); @@ -214,7 +216,7 @@ suite('Lexer Diagnostics', () => { test('Invalid number literal - exponent with sign but no digits', () => { const source = 'float x = 1.5e+;'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); const tokens = lexer.tokenize(); @@ -233,7 +235,7 @@ suite('Lexer Diagnostics', () => { test('Valid number literals - no errors', () => { const source = 'float a = 123; float b = 1.5; float c = 1e10; float d = 1.5e-5; float e = 1.0f;'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); lexer.tokenize(); @@ -245,7 +247,7 @@ suite('Lexer Diagnostics', () => { test('Unterminated vector literal - EOF', () => { const source = 'vector v = <1.0, 2.0, 3.0'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); const tokens = lexer.tokenize(); @@ -264,7 +266,7 @@ suite('Lexer Diagnostics', () => { test('Unterminated vector literal - newline', () => { const source = 'vector v = <1.0, 2.0, 3.0\nvector v2 = <1,2,3>;'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); const tokens = lexer.tokenize(); @@ -278,7 +280,7 @@ suite('Lexer Diagnostics', () => { test('Valid vector literals - no errors', () => { const source = 'vector v = <1.0, 2.0, 3.0>; rotation r = <0.0, 0.0, 0.0, 1.0>;'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testFile, diagnostics); lexer.tokenize(); @@ -290,7 +292,7 @@ suite('Lexer Diagnostics', () => { test('Unterminated SLua string interp - newline', () => { const source = 'local str = `some string\nnew line`'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, "luau", testFile, diagnostics); + const lexer = new Lexer(source, luauLanguageConfig, testFile, diagnostics); lexer.tokenize(); assert.strictEqual(diagnostics.hasErrors(), true); assert.strictEqual(diagnostics.getCount(), 2); @@ -299,7 +301,7 @@ suite('Lexer Diagnostics', () => { test('Unterminated SLua string interp - {', () => { const source = 'local str = `some string{new line'; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, "luau", testFile, diagnostics); + const lexer = new Lexer(source, luauLanguageConfig, testFile, diagnostics); lexer.tokenize(); assert.strictEqual(diagnostics.hasErrors(), true); assert.strictEqual(diagnostics.getCount(), 1); diff --git a/src/test/suite/lexingpreprocessor.test.ts b/src/test/suite/lexingpreprocessor.test.ts index eb295ad..9c846a4 100644 --- a/src/test/suite/lexingpreprocessor.test.ts +++ b/src/test/suite/lexingpreprocessor.test.ts @@ -11,11 +11,12 @@ import { Lexer, TokenType, LanguageLexerConfig, + getLanguageConfig, + CustomLanguageLexerConfig, } from "../../shared/lexer"; -import { LexingPreprocessor, PreprocessorOptions } from "../../shared/lexingpreprocessor"; +import { LexingPreprocessor, PreprocessorOptions, PreprocessorResult } from "../../shared/lexingpreprocessor"; import { normalizePath, HostInterface, NormalizedPath } from "../../interfaces/hostinterface"; import { FullConfigInterface, ConfigKey } from "../../interfaces/configinterface"; -import { ScriptLanguage } from "../../shared/languageservice"; /** * Mock configuration class for testing @@ -75,6 +76,11 @@ class MockConfig implements FullConfigInterface { suite("Lexing Preprocessor Test Suite", () => { + const lslLanguageConfig = getLanguageConfig('lsl'); + const lslLanguageConfigWithSwitch = getLanguageConfig('lsl'); + lslLanguageConfigWithSwitch.directiveKeywords.push("switch"); + const luauLanguageConfig = getLanguageConfig('luau'); + /** * Helper to create a mock HostInterface for testing */ @@ -230,11 +236,37 @@ suite("Lexing Preprocessor Test Suite", () => { .replace(/\n{3,}/g, '\n\n'); // Collapse multiple blank lines to max 2 } + async function basicPreprocMatchExpectedAssert(source: string, expected: string) : Promise { + const options = createDefaultOptions(); + const mockHost = createMockHost(options); + const preprocessor = new LexingPreprocessor(mockHost, mockHost.config); + + const result = await preprocessor.process( + source, + normalizePath('/test/main.lsl'), + lslLanguageConfigWithSwitch + ); + + // Verify no errors + if(!result.success) { + for(const issue of result.issues) { + console.error(`Issue: ${issue.message} at line ${issue.lineNumber}`); + } + } + assert.ok(result.success, 'Processing should succeed'); + assert.strictEqual(result.issues.length, 0, 'Should have no issues'); + + // Compare with expected output + assert.strictEqual(result.content, expected, 'Output should match expected'); + + return result; + } + suite("Lexer", () => { test("should tokenize simple LSL code", () => { const source = `integer x = 42;`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); assert.strictEqual(tokens.length, 9); // integer, space, x, space, =, space, 42, ;, EOF @@ -248,7 +280,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should recognize LSL line comments", () => { const source = `// This is a comment\ninteger x;`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); assert.strictEqual(tokens[0].type, TokenType.LINE_COMMENT); @@ -258,7 +290,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should recognize SLua line comments", () => { const source = `-- This is a comment\nlocal x = 5`; - const lexer = new Lexer(source, "luau"); + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); assert.strictEqual(tokens[0].type, TokenType.LINE_COMMENT); @@ -268,7 +300,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should tokenize block comments", () => { const source = `/* Block comment */\ninteger x;`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); assert.strictEqual(tokens[0].type, TokenType.BLOCK_COMMENT_START); @@ -281,7 +313,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should tokenize multi-line block comments", () => { const source = `/* Line 1\nLine 2\nLine 3 */`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); assert.strictEqual(tokens[0].type, TokenType.BLOCK_COMMENT_START); @@ -295,7 +327,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should tokenize Lua long bracket comments with equals", () => { // Test --[=[ ... ]=] const source1 = `--[=[\nThis is a comment with ]] inside\n]=]`; - const lexer1 = new Lexer(source1, "luau"); + const lexer1 = new Lexer(source1, luauLanguageConfig); const tokens1 = lexer1.tokenize(); console.log(tokens1); @@ -309,7 +341,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test --[==[ ... ]==] const source2 = `--[==[\nDouble equals level\n]==]`; - const lexer2 = new Lexer(source2, "luau"); + const lexer2 = new Lexer(source2, luauLanguageConfig); const tokens2 = lexer2.tokenize(); assert.strictEqual(tokens2[0].type, TokenType.BLOCK_COMMENT_START); @@ -319,7 +351,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test that ]] doesn't close [=[ ... ]=] const source3 = `--[=[\nHas ]] in middle\n]=]`; - const lexer3 = new Lexer(source3, "luau"); + const lexer3 = new Lexer(source3, luauLanguageConfig); const tokens3 = lexer3.tokenize(); // Should have exactly one block comment content token with ]] inside @@ -330,7 +362,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should tokenize string literals with escapes", () => { const source = `"Hello \\"World\\""`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); assert.strictEqual(tokens[0].type, TokenType.STRING_LITERAL); @@ -340,7 +372,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should support configured string delimiters", () => { // Test LSL with both double and single quotes const lslSource = `"double" 'single'`; - const lslLexer = new Lexer(lslSource, "lsl"); + const lslLexer = new Lexer(lslSource, lslLanguageConfig); const lslTokens = lslLexer.tokenize(); const lslStrings = lslTokens.filter(t => t.type === TokenType.STRING_LITERAL); @@ -357,7 +389,7 @@ suite("Lexing Preprocessor Test Suite", () => { "[=[square equals]=]", ] const luauSource = luauSourceStrings.join(" "); - const luauLexer = new Lexer(luauSource, "luau"); + const luauLexer = new Lexer(luauSource, luauLanguageConfig); const luauTokens = luauLexer.tokenize(); assert.ok(!luauLexer.getDiagnostics().hasErrors(),"Luau Lexer has errors: " + JSON.stringify(luauLexer.getDiagnostics().getErrors(),null,2)); @@ -382,7 +414,7 @@ suite("Lexing Preprocessor Test Suite", () => { "}`" ]; const source = parts.join(''); - const luauLexer = new Lexer(source, "luau"); + const luauLexer = new Lexer(source, luauLanguageConfig); const luauTokens = luauLexer.tokenize(); assert.strictEqual(luauTokens.length, 10); for(const i in parts) { @@ -414,7 +446,7 @@ suite("Lexing Preprocessor Test Suite", () => { "}`" ]; const source = parts.join(''); - const luauLexer = new Lexer(source, "luau"); + const luauLexer = new Lexer(source, luauLanguageConfig); const luauTokens = luauLexer.tokenize(); assert.strictEqual(luauTokens.length, parts.length + 1); for(const i in parts) { @@ -422,7 +454,7 @@ suite("Lexing Preprocessor Test Suite", () => { } }); - const tokenizeAndGetStrings = (strings:string[],lang:ScriptLanguage) : string[] => { + const tokenizeAndGetStrings = (strings:string[],lang:LanguageLexerConfig) : string[] => { const lexer = new Lexer(strings.join(" "),lang); const tokens = lexer.tokenize().filter(t => t.type == TokenType.STRING_LITERAL); return tokens.map(t => t.value); @@ -431,7 +463,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should handle embedded quotes in strings", () => { // Test single quotes inside double-quoted strings const source1 = [`"This is a 'string'"`]; - const strings1 = tokenizeAndGetStrings(source1, "lsl"); + const strings1 = tokenizeAndGetStrings(source1, lslLanguageConfig); for(const i in source1) { assert.strictEqual(strings1[i], source1[i]); } @@ -439,7 +471,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test double quotes inside single-quoted strings const source2 = [`'embedded "double" quote'`]; - const strings2 = tokenizeAndGetStrings(source2, "lsl"); + const strings2 = tokenizeAndGetStrings(source2, lslLanguageConfig); for(const i in source2) { assert.strictEqual(strings2[i], source2[i]); } @@ -447,7 +479,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test mixed quotes in Luau with backticks const source3 = [`\`can contain "double" and 'single' quotes\``]; - const strings3 = tokenizeAndGetStrings(source3, "luau"); + const strings3 = tokenizeAndGetStrings(source3, luauLanguageConfig); for(const i in source3) { assert.strictEqual(strings3[i], source3[i]); } @@ -455,7 +487,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test multiple strings with different delimiters const source4 = [`"first 'has' single"`,`'second "has" double'`]; - const strings4 = tokenizeAndGetStrings(source4,"lsl"); + const strings4 = tokenizeAndGetStrings(source4,lslLanguageConfig); for(const i in source4) { assert.strictEqual(strings4[i], source4[i]); } @@ -468,7 +500,7 @@ suite("Lexing Preprocessor Test Suite", () => { `[["fourth" 'has' \`three\`]]`, `[=["fourth" 'has' \`three\` [[and one more]]]=]`, ]; - const strings5 = tokenizeAndGetStrings(source5,"luau"); + const strings5 = tokenizeAndGetStrings(source5, luauLanguageConfig); for(const i in source5) { assert.strictEqual(strings5[i], source5[i]); } @@ -478,7 +510,7 @@ suite("Lexing Preprocessor Test Suite", () => { `[===[now we[[[[[]]]]] are [==[]][[[===[]]]] just really messing [][][[][][[[]]]] around]]] ]====] ]===]`, `[===[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[===]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]===]`, ]; - const strings6 = tokenizeAndGetStrings(source6,"luau"); + const strings6 = tokenizeAndGetStrings(source6, luauLanguageConfig); for(const i in source6) { assert.strictEqual(strings6[i], source6[i]); } @@ -499,7 +531,7 @@ suite("Lexing Preprocessor Test Suite", () => { `[[\n\t\t]]`, `[[\t\t\n\t\t]]`, ]; - const strings1 = tokenizeAndGetStrings(source1,"luau"); + const strings1 = tokenizeAndGetStrings(source1, luauLanguageConfig); for(const i in source1) { assert.strictEqual(strings1[i], source1[i]); } @@ -507,7 +539,7 @@ suite("Lexing Preprocessor Test Suite", () => { // And windows style const source2 = source1.map(s => s.replaceAll("\n","\r\n")); - const strings2 = tokenizeAndGetStrings(source2,"luau"); + const strings2 = tokenizeAndGetStrings(source2, luauLanguageConfig); for(const i in source2) { assert.strictEqual(strings2[i], source2[i]); } @@ -516,7 +548,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should recognize LSL preprocessor directives", () => { const source = `#include "test.lsl"`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); assert.strictEqual(tokens[0].type, TokenType.DIRECTIVE); @@ -525,7 +557,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should recognize SLua require directive", () => { const source = `require("test.luau")`; - const lexer = new Lexer(source, "luau"); + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); assert.strictEqual(tokens[0].type, TokenType.DIRECTIVE); @@ -534,7 +566,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should tokenize numbers", () => { const source = `42 3.14 1.5e-10`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); const numberTokens = tokens.filter(t => t.type === TokenType.NUMBER_LITERAL); @@ -547,7 +579,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should tokenize multi-character operators", () => { // Test comparison operators const source1 = `if (x == y && a != b)`; - const lexer1 = new Lexer(source1, "lsl"); + const lexer1 = new Lexer(source1, lslLanguageConfig); const tokens1 = lexer1.tokenize(); const operators1 = tokens1.filter(t => t.type === TokenType.OPERATOR); @@ -557,7 +589,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test assignment operators const source2 = `x += 5; y -= 2;`; - const lexer2 = new Lexer(source2, "lsl"); + const lexer2 = new Lexer(source2, lslLanguageConfig); const tokens2 = lexer2.tokenize(); const operators2 = tokens2.filter(t => t.type === TokenType.OPERATOR); @@ -566,7 +598,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test bitwise operators const source3 = `a << 3 >> 1`; - const lexer3 = new Lexer(source3, "lsl"); + const lexer3 = new Lexer(source3, lslLanguageConfig); const tokens3 = lexer3.tokenize(); const operators3 = tokens3.filter(t => t.type === TokenType.OPERATOR); @@ -575,7 +607,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test Lua-specific operators const source4 = `if x ~= y then local s = a .. b end`; - const lexer4 = new Lexer(source4, "luau"); + const lexer4 = new Lexer(source4, luauLanguageConfig); const tokens4 = lexer4.tokenize(); const operators4 = tokens4.filter(t => t.type === TokenType.OPERATOR); @@ -584,7 +616,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test that single-character operators still work const source5 = `x + y - z * 2 / 3`; - const lexer5 = new Lexer(source5, "lsl"); + const lexer5 = new Lexer(source5, lslLanguageConfig); const tokens5 = lexer5.tokenize(); const operators5 = tokens5.filter(t => t.type === TokenType.OPERATOR); @@ -597,7 +629,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should tokenize LSL vector and rotation literals", () => { // Test 3-component vector const source1 = `vector pos = <1.0, 2.5, -3.0>;`; - const lexer1 = new Lexer(source1, "lsl"); + const lexer1 = new Lexer(source1, lslLanguageConfig); const tokens1 = lexer1.tokenize(); const vectors1 = tokens1.filter(t => t.type === TokenType.VECTOR_LITERAL); @@ -606,7 +638,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test 4-component rotation const source2 = `rotation rot = <0.0, 0.0, 0.707, 0.707>;`; - const lexer2 = new Lexer(source2, "lsl"); + const lexer2 = new Lexer(source2, lslLanguageConfig); const tokens2 = lexer2.tokenize(); const vectors2 = tokens2.filter(t => t.type === TokenType.VECTOR_LITERAL); @@ -615,7 +647,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test vector with no spaces const source3 = `<1,2,3>`; - const lexer3 = new Lexer(source3, "lsl"); + const lexer3 = new Lexer(source3, lslLanguageConfig); const tokens3 = lexer3.tokenize(); const vectors3 = tokens3.filter(t => t.type === TokenType.VECTOR_LITERAL); @@ -624,7 +656,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test that < is still an operator in non-vector context const source4 = `if (x < 5)`; - const lexer4 = new Lexer(source4, "lsl"); + const lexer4 = new Lexer(source4, lslLanguageConfig); const tokens4 = lexer4.tokenize(); const operators = tokens4.filter(t => t.type === TokenType.OPERATOR && t.value === "<"); @@ -632,7 +664,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test vector with whitespace variations const source5 = `< 1.5 , 2.5 , 3.5 >`; - const lexer5 = new Lexer(source5, "lsl"); + const lexer5 = new Lexer(source5, lslLanguageConfig); const tokens5 = lexer5.tokenize(); const vectors5 = tokens5.filter(t => t.type === TokenType.VECTOR_LITERAL); @@ -641,7 +673,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test that Luau doesn't recognize vectors const source6 = `<1, 2, 3>`; - const lexer6 = new Lexer(source6, "luau"); + const lexer6 = new Lexer(source6, luauLanguageConfig); const tokens6 = lexer6.tokenize(); const vectors6 = tokens6.filter(t => t.type === TokenType.VECTOR_LITERAL); @@ -653,7 +685,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test vectors with identifiers (variables) const source7 = `vector pos = ;`; - const lexer7 = new Lexer(source7, "lsl"); + const lexer7 = new Lexer(source7, lslLanguageConfig); const tokens7 = lexer7.tokenize(); const vectors7 = tokens7.filter(t => t.type === TokenType.VECTOR_LITERAL); @@ -662,7 +694,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test mixed literals and identifiers const source8 = `<1.0, height, 3.0>`; - const lexer8 = new Lexer(source8, "lsl"); + const lexer8 = new Lexer(source8, lslLanguageConfig); const tokens8 = lexer8.tokenize(); const vectors8 = tokens8.filter(t => t.type === TokenType.VECTOR_LITERAL); @@ -671,7 +703,7 @@ suite("Lexing Preprocessor Test Suite", () => { // Test rotation with variables const source9 = `rotation rot = <0, 0, angle, 1>;`; - const lexer9 = new Lexer(source9, "lsl"); + const lexer9 = new Lexer(source9, lslLanguageConfig); const tokens9 = lexer9.tokenize(); const vectors9 = tokens9.filter(t => t.type === TokenType.VECTOR_LITERAL); @@ -681,7 +713,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should tokenize brackets as distinct types", () => { const source = `{ ( [ ] ) }`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); const brackets = tokens.filter(t => @@ -710,7 +742,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should preserve line and column information", () => { const source = `integer x;\nfloat y;`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); // First line tokens @@ -725,7 +757,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should handle empty input", () => { const source = ``; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); assert.strictEqual(tokens.length, 1); // Just EOF @@ -734,7 +766,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should handle whitespace-only input", () => { const source = ` \t \n `; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); const nonEofTokens = tokens.filter(t => t.type !== TokenType.EOF); @@ -745,7 +777,8 @@ suite("Lexing Preprocessor Test Suite", () => { test("should accept custom language configuration", () => { // Create a custom language config with Python-style comments - const customConfig: LanguageLexerConfig = { + const customConfig: CustomLanguageLexerConfig = { + name: "custom", lineCommentPrefix: "#", // blockCommentStart: "'''", // blockCommentEnd: "'''", @@ -775,7 +808,8 @@ suite("Lexing Preprocessor Test Suite", () => { test("should support custom multi-character operators in configuration", () => { // Create a custom language config with unique operators - const customConfig: LanguageLexerConfig = { + const customConfig: CustomLanguageLexerConfig = { + name: "custom", lineCommentPrefix: "//", // blockCommentStart: "/*", // blockCommentEnd: "*/", @@ -805,7 +839,8 @@ suite("Lexing Preprocessor Test Suite", () => { test("should support custom bracket configuration", () => { // Create a custom language config with standard brackets - const customConfig: LanguageLexerConfig = { + const customConfig: CustomLanguageLexerConfig = { + name: "custom", lineCommentPrefix: "//", // blockCommentStart: "/*", // blockCommentEnd: "*/", @@ -844,7 +879,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should reconstruct simple source from tokens", () => { const source = `integer x = 42;`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); // Reconstruct using Token.emit() method @@ -858,7 +893,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should reconstruct source with whitespace", () => { const source = `integer x\t= 42;\n`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); const reconstructed = tokens @@ -871,7 +906,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should reconstruct source with comments", () => { const source = `// Line comment\ninteger x; // end comment\n/* block */`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); const reconstructed = tokens @@ -884,7 +919,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should reconstruct source with strings", () => { const source = `string msg = "Hello 'world'";`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); const reconstructed = tokens @@ -897,7 +932,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should reconstruct source with multi-character operators", () => { const source = `if (x == y && a != b || c >= d) { x += 5; }`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); const reconstructed = tokens @@ -910,7 +945,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should reconstruct source with vector literals", () => { const source = `vector pos = <1.0, 2.5, 3.0>;`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); const reconstructed = tokens @@ -923,7 +958,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should reconstruct source with Luau long bracket comments", () => { const source = `--[=[ Multi-line\ncomment ]=] x = 5`; - const lexer = new Lexer(source, "luau"); + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); const reconstructed = tokens @@ -947,7 +982,7 @@ suite("Lexing Preprocessor Test Suite", () => { string msg = "Hello"; } }`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); const reconstructed = tokens @@ -960,7 +995,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should reconstruct source with all bracket types", () => { const source = `list items = [1, 2, 3]; vector v = <1,2,3>; func(a, b);`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); const reconstructed = tokens @@ -973,7 +1008,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should reconstruct Luau source with all string delimiters", () => { const source = `local s1 = "double"; local s2 = 'single'; local s3 = \`backtick\`;`; - const lexer = new Lexer(source, "luau"); + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); const reconstructed = tokens @@ -986,7 +1021,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should use Token class methods", () => { const source = `integer x = 42; // comment`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); // Test emit() @@ -1040,7 +1075,7 @@ suite("Lexing Preprocessor Test Suite", () => { test("should use bracket checking methods", () => { const source = `{ ( [ ] ) }`; - const lexer = new Lexer(source, "lsl"); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); const braceOpen = tokens.find(t => t.value === "{"); @@ -1068,7 +1103,7 @@ suite("Lexing Preprocessor Test Suite", () => { const sourceFile = normalizePath("test.lsl"); const preprocessor = new LexingPreprocessor(mockHost, mockHost.config); - const result = await preprocessor.process(source, sourceFile, "lsl"); + const result = await preprocessor.process(source, sourceFile, lslLanguageConfig); if (!result.success) { console.log("Preprocessing failed with issues:", result.issues); @@ -1082,7 +1117,7 @@ suite("Lexing Preprocessor Test Suite", () => { const sourceFile = normalizePath("test.lsl"); const preprocessor = new LexingPreprocessor(mockHost, mockHost.config); - const result = await preprocessor.process(source, sourceFile, "lsl"); + const result = await preprocessor.process(source, sourceFile, lslLanguageConfig); assert.strictEqual(result.success, true); assert.ok(result.content.includes("// Comment")); @@ -1096,7 +1131,7 @@ suite("Lexing Preprocessor Test Suite", () => { const disabledHost = createMockHost(disabledOptions); const preprocessor = new LexingPreprocessor(disabledHost, disabledHost.config); - const result = await preprocessor.process(source, sourceFile, "lsl"); + const result = await preprocessor.process(source, sourceFile, lslLanguageConfig); assert.strictEqual(result.success, true); assert.strictEqual(result.content, source); // Unchanged @@ -1122,7 +1157,7 @@ default { const result = await preprocessor.process( source, normalizePath("test.lsl"), - "lsl" + lslLanguageConfig ); assert.strictEqual(result.success, true); @@ -1148,7 +1183,7 @@ default { const result = await preprocessor.process( source, normalizePath(sourceFile), - "lsl" + lslLanguageConfig ); // Verify successful preprocessing @@ -1219,7 +1254,7 @@ default { const result = await preprocessor.process( source, normalizePath(sourceFile), - "lsl" + lslLanguageConfig ); assert.strictEqual(result.success, true, "Preprocessing should succeed"); @@ -1293,7 +1328,7 @@ default { const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); // Should fail due to depth limit of 2 being exceeded @@ -1337,7 +1372,7 @@ default { const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); assert.strictEqual(result.success, true, "Should succeed with configured include paths"); @@ -1353,7 +1388,7 @@ default { const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); // Should work fine with defaults (maxIncludeDepth: 5, includePaths: ['.']) @@ -1377,7 +1412,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); // Should succeed but return source unchanged @@ -1400,7 +1435,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); // Should succeed and process the macros @@ -1421,7 +1456,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); // Should succeed and process (default is enabled) @@ -1452,7 +1487,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); // Should succeed with depth of 1 (allows one level of includes) @@ -1489,7 +1524,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); // Should fail because the chain exceeds depth 3 @@ -1517,7 +1552,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); // Should have warnings/errors but still produce output @@ -1540,7 +1575,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); // Should have error about unterminated conditional @@ -1570,7 +1605,7 @@ default { state_entry() {} }`; const result = await preprocessor.process( source, normalizePath('/test/main.lsl'), - 'lsl' + lslLanguageConfig ); // Should have warnings but still succeed @@ -1585,4 +1620,137 @@ default { state_entry() {} }`; assert.ok(result.content.includes('default'), "Should complete processing"); }); }); + + test('should allow switch as identifier if switch statments not enabled for lsl preproc', async () => { + const source = `integer switch = 1;`; + const expected = `integer switch = 1;`; + + const options = createDefaultOptions(); + const mockHost = createMockHost(options); + const preprocessor = new LexingPreprocessor(mockHost, mockHost.config); + + const result = await preprocessor.process( + source, + normalizePath('/test/main.lsl'), + lslLanguageConfig + ); + + // Compare with expected output + assert.strictEqual(result.content, expected, 'Output should match expected file'); + + // Verify no errors + assert.ok(result.success, 'Processing should succeed'); + assert.strictEqual(result.issues.length, 0, 'Should have no issues'); + }); + + test('should error on switch as identifier if switch statements enabled for lsl preproc', async () => { + const source = `integer switch = 1;`; + const expected = `integer switch = 1;`; + + const options = createDefaultOptions(); + const mockHost = createMockHost(options); + const preprocessor = new LexingPreprocessor(mockHost, mockHost.config); + + const result = await preprocessor.process( + source, + normalizePath('/test/main.lsl'), + lslLanguageConfigWithSwitch + ); + + // Compare with expected output + assert.strictEqual(result.content, expected, 'Output should match expected file'); + + // Verify no errors + assert.ok(!result.success, 'Processing should not succeed'); + assert.strictEqual(result.issues.length, 1, 'Should have 1 issue'); + }); + + test('should allow switch as identifier if switch statments not enabled for lsl preproc', async () => { + const source = `switch(a) { + case 1: { + print("One"); + break; + } + default: { + print("Other"); + break; + } +}`; + const expected = `if((a) == (1)) jump c860cf; +jump c7b43f; +@c860cf; +{ + print("One"); + jump sfcc97; +} +@c7b43f; +{ + // @line 7 "unittest:///test/main.lsl" + print("Other"); + jump sfcc97; +} +@sfcc97;`; + + const options = createDefaultOptions(); + const mockHost = createMockHost(options); + const preprocessor = new LexingPreprocessor(mockHost, mockHost.config); + + const result = await preprocessor.process( + source, + normalizePath('/test/main.lsl'), + lslLanguageConfigWithSwitch + ); + + // Compare with expected output + assert.strictEqual(result.content, expected, 'Output should match expected file'); + + // Verify no errors + assert.ok(result.success, 'Processing should succeed'); + assert.strictEqual(result.issues.length, 0, 'Should have no issues'); + }); + + test('empty function macro', async () => { + const source = `#define OP() llSleep(1) +#define NOOP(a,b,c,d) +OP(); +NOOP(1,2,3,4); +integer x = 42;`; + const expected = `llSleep(1); +; +integer x = 42;`; + + const options = createDefaultOptions(); + const mockHost = createMockHost(options); + const preprocessor = new LexingPreprocessor(mockHost, mockHost.config); + + const result = await preprocessor.process( + source, + normalizePath('/test/main.lsl'), + lslLanguageConfigWithSwitch + ); + + // Compare with expected output + assert.strictEqual(result.content, expected, 'Output should match expected file'); + + // Verify no errors + assert.ok(result.success, 'Processing should succeed'); + assert.strictEqual(result.issues.length, 0, 'Should have no issues'); + + }); + + test('function macro with list argument', async () => { + const source = `#define implode(sep,lst) llDumpList2String(lst, sep) +implode(",", [1, 2, 3, 4, 5]);`; + const expected = `llDumpList2String([1, 2, 3, 4, 5], ",");`; + + await basicPreprocMatchExpectedAssert(source, expected); + }); + + test('variadic function macro', async () => { + const source = `#define LOG(fmt, ...) llOwnerSay(llDumpList2String([fmt, __VA_ARGS__], " ")) +LOG("Values are:", 1, 2, 3, 4, 5);`; + const expected = `llOwnerSay(llDumpList2String(["Values are:", 1, 2, 3, 4, 5], " "));`; + await basicPreprocMatchExpectedAssert(source, expected); + }); + }); diff --git a/src/test/suite/macro-diagnostics.test.ts b/src/test/suite/macro-diagnostics.test.ts index 27e39ce..a0091d7 100644 --- a/src/test/suite/macro-diagnostics.test.ts +++ b/src/test/suite/macro-diagnostics.test.ts @@ -8,17 +8,18 @@ import * as assert from 'assert'; import { MacroProcessor, MacroExpansionContext } from '../../shared/macroprocessor'; -import { Lexer } from '../../shared/lexer'; +import { getLanguageConfig, Lexer } from '../../shared/lexer'; import { DiagnosticCollector, DiagnosticSeverity, ErrorCodes } from '../../shared/diagnostics'; import { normalizePath } from '../../interfaces/hostinterface'; suite('MacroProcessor Diagnostics', () => { let processor: MacroProcessor; let diagnostics: DiagnosticCollector; + const lslLanguageConfig = getLanguageConfig('lsl'); const testFile = normalizePath('test.lsl'); setup(() => { - processor = new MacroProcessor('lsl'); + processor = new MacroProcessor(); diagnostics = new DiagnosticCollector(); }); @@ -40,7 +41,7 @@ suite('MacroProcessor Diagnostics', () => { test('should not warn for defined macros', () => { // Given: Macro is defined - const bodyLexer = new Lexer('42', 'lsl'); + const bodyLexer = new Lexer('42', lslLanguageConfig); const bodyTokens = bodyLexer.tokenize().slice(0, -1); processor.define({ name: 'DEFINED', body: bodyTokens, isFunctionLike: false }); @@ -55,7 +56,7 @@ suite('MacroProcessor Diagnostics', () => { suite('MAC002: Argument Count Mismatch', () => { test('should error when too few arguments provided', () => { // Given: Function-like macro expecting 2 parameters - const bodyLexer = new Lexer('x + y', 'lsl'); + const bodyLexer = new Lexer('x + y', lslLanguageConfig); const bodyTokens = bodyLexer.tokenize().slice(0, -1); processor.define({ name: 'ADD', @@ -66,7 +67,7 @@ suite('MacroProcessor Diagnostics', () => { // When: Calling with only 1 argument const source = 'ADD(5)'; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig) const tokens = lexer.tokenize().slice(0, -1); // Remove EOF // Find the ADD identifier and parse the call @@ -91,7 +92,7 @@ suite('MacroProcessor Diagnostics', () => { test('should error when too many arguments provided', () => { // Given: Function-like macro expecting 1 parameter - const bodyLexer = new Lexer('x * 2', 'lsl'); + const bodyLexer = new Lexer('x * 2', lslLanguageConfig); const bodyTokens = bodyLexer.tokenize().slice(0, -1); processor.define({ name: 'DOUBLE', @@ -102,7 +103,7 @@ suite('MacroProcessor Diagnostics', () => { // When: Calling with 2 arguments const source = 'DOUBLE(5, 10)'; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig) const tokens = lexer.tokenize().slice(0, -1); // Remove EOF const context: MacroExpansionContext = { line: 1, column: 1, sourceFile: testFile }; @@ -122,7 +123,7 @@ suite('MacroProcessor Diagnostics', () => { test('should not error when correct number of arguments provided', () => { // Given: Function-like macro with parameters - const bodyLexer = new Lexer('x + y', 'lsl'); + const bodyLexer = new Lexer('x + y', lslLanguageConfig); const bodyTokens = bodyLexer.tokenize().slice(0, -1); processor.define({ name: 'ADD', @@ -133,7 +134,7 @@ suite('MacroProcessor Diagnostics', () => { // When: Calling with correct argument count const source = 'ADD(5, 10)'; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig) const tokens = lexer.tokenize().slice(0, -1); // Remove EOF const context: MacroExpansionContext = { line: 1, column: 1, sourceFile: testFile }; @@ -153,7 +154,7 @@ suite('MacroProcessor Diagnostics', () => { suite('MAC003: Recursive Expansion', () => { test('should detect direct self-recursion', () => { // Given: Macro that references itself - const bodyLexer = new Lexer('SELF + 1', 'lsl'); + const bodyLexer = new Lexer('SELF + 1', lslLanguageConfig); const bodyTokens = bodyLexer.tokenize().slice(0, -1); processor.define({ name: 'SELF', @@ -176,7 +177,7 @@ suite('MacroProcessor Diagnostics', () => { test('should detect indirect recursion', () => { // Given: Two macros that reference each other - const aBodyLexer = new Lexer('B', 'lsl'); + const aBodyLexer = new Lexer('B', lslLanguageConfig); const aBodyTokens = aBodyLexer.tokenize().slice(0, -1); processor.define({ name: 'A', @@ -184,7 +185,7 @@ suite('MacroProcessor Diagnostics', () => { isFunctionLike: false }); - const bBodyLexer = new Lexer('A', 'lsl'); + const bBodyLexer = new Lexer('A', lslLanguageConfig); const bBodyTokens = bBodyLexer.tokenize().slice(0, -1); processor.define({ name: 'B', @@ -208,7 +209,7 @@ suite('MacroProcessor Diagnostics', () => { test('should error on malformed defined() - missing parentheses', () => { // Given: defined without parentheses const source = 'defined MACRO'; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig) const tokens = lexer.tokenize(); // When: Processing defined() with diagnostics @@ -225,7 +226,7 @@ suite('MacroProcessor Diagnostics', () => { test('should error on malformed defined() - missing closing paren', () => { // Given: defined with unclosed parenthesis const source = 'defined(MACRO'; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig) const tokens = lexer.tokenize(); // When: Processing defined() with diagnostics @@ -240,7 +241,7 @@ suite('MacroProcessor Diagnostics', () => { test('should error on defined() with no identifier', () => { // Given: defined with empty parentheses const source = 'defined()'; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig) const tokens = lexer.tokenize(); // When: Processing defined() with diagnostics @@ -255,7 +256,7 @@ suite('MacroProcessor Diagnostics', () => { test('should not error on valid defined() syntax', () => { // Given: Valid defined() syntax const source = 'defined(MACRO)'; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig) const tokens = lexer.tokenize(); // When: Processing defined() with diagnostics diff --git a/src/test/suite/parser-diagnostics.test.ts b/src/test/suite/parser-diagnostics.test.ts index 765ede5..1711c82 100644 --- a/src/test/suite/parser-diagnostics.test.ts +++ b/src/test/suite/parser-diagnostics.test.ts @@ -5,12 +5,14 @@ import * as assert from 'assert'; import { Parser } from '../../shared/parser'; -import { Lexer } from '../../shared/lexer'; +import { getLanguageConfig, Lexer } from '../../shared/lexer'; import { DiagnosticCollector, DiagnosticSeverity, ErrorCodes } from '../../shared/diagnostics'; import { normalizePath, NormalizedPath } from '../../interfaces/hostinterface'; +import { get } from 'http'; suite('Parser Diagnostics Integration', () => { let sourceFile: NormalizedPath; + const lslLanguageConfig = getLanguageConfig('lsl'); setup(() => { sourceFile = normalizePath('test.lsl'); @@ -21,9 +23,9 @@ suite('Parser Diagnostics Integration', () => { const source = `#elif 1 code here`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -39,9 +41,9 @@ code here`; const source = `#else code here`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -55,9 +57,9 @@ code here`; const source = `#endif code here`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -76,9 +78,9 @@ code2 code3 #endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -98,9 +100,9 @@ code2 code3 #endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -118,9 +120,9 @@ code3 code here #endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -137,9 +139,9 @@ code2 code3 #endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -154,9 +156,9 @@ code3 #endif #endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -170,9 +172,9 @@ code3 const testSource = normalizePath('custom.lsl'); const source = `#endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', testSource, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, testSource, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testSource, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, testSource, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -187,9 +189,9 @@ code3 const source = `"unterminated #elif 1`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -203,9 +205,9 @@ code3 test('should be able to check for errors', async () => { const source = `#endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -217,9 +219,9 @@ code3 code #endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); diff --git a/src/test/suite/parser-directive-diagnostics.test.ts b/src/test/suite/parser-directive-diagnostics.test.ts index 8f84b70..e67c0b6 100644 --- a/src/test/suite/parser-directive-diagnostics.test.ts +++ b/src/test/suite/parser-directive-diagnostics.test.ts @@ -5,12 +5,13 @@ import * as assert from 'assert'; import { Parser } from '../../shared/parser'; -import { Lexer, TokenType } from '../../shared/lexer'; +import { getLanguageConfig, Lexer, TokenType } from '../../shared/lexer'; import { DiagnosticCollector, DiagnosticSeverity, ErrorCodes } from '../../shared/diagnostics'; import { normalizePath, NormalizedPath } from '../../interfaces/hostinterface'; suite('Parser Directive Validation', () => { let sourceFile: NormalizedPath; + const lslLanguageConfig = getLanguageConfig('lsl'); setup(() => { sourceFile = normalizePath('test.lsl'); @@ -21,9 +22,9 @@ suite('Parser Directive Validation', () => { const source = `#unknown code here`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -38,9 +39,9 @@ code here`; const source = `#123invalid code here`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -54,9 +55,9 @@ code here`; #ifdef X #endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -70,9 +71,9 @@ code here`; const source = `#include code here`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -89,9 +90,9 @@ code here`; code here #endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -106,9 +107,9 @@ code here code here #endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -122,9 +123,9 @@ code here const source = `#define code here`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -138,9 +139,9 @@ code here`; const source = `#undef code here`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -156,9 +157,9 @@ code here`; #endif #undef X`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -171,9 +172,9 @@ code here`; test('should error on macro name starting with digit', async () => { const source = `#define 123ABC 1`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -186,9 +187,9 @@ code here`; test('should error on duplicate parameter names', async () => { const source = `#define MACRO(a, b, a) (a + b)`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -203,9 +204,9 @@ code here`; const source = `#define PI 3.14159 #define MAX(a, b) ((a) > (b) ? (a) : (b))`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -219,9 +220,9 @@ code here`; const source = `#if 1 code here`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -236,9 +237,9 @@ code here`; const source = `#ifdef X code here`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -254,9 +255,9 @@ code here`; code here #endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -272,9 +273,9 @@ code here #endif #endif`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -288,9 +289,9 @@ code here const source = `#define MAX(a, b) ((a) > (b) ? (a) : (b)) integer x = MAX;`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -305,9 +306,9 @@ integer x = MAX;`; const source = `#define MAX(a, b) ((a) > (b) ? (a) : (b)) integer x = MAX(1, 2);`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -319,9 +320,9 @@ integer x = MAX(1, 2);`; const source = `#define PI 3.14159 float x = PI;`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -340,9 +341,9 @@ float area = PI * radius * radius; #endif #undef PI`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); @@ -357,9 +358,9 @@ float area = PI * radius * radius; #define B 2 // Comment #define C 3`; const diagnostics = new DiagnosticCollector(); - const lexer = new Lexer(source, 'lsl', sourceFile, diagnostics); + const lexer = new Lexer(source, lslLanguageConfig, sourceFile, diagnostics); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, sourceFile, 'lsl', undefined, undefined, undefined, true, undefined, diagnostics); + const parser = new Parser(tokens, sourceFile, lslLanguageConfig, undefined, undefined, undefined, true, undefined, diagnostics); await parser.parse(); diff --git a/src/test/suite/parser.test.ts b/src/test/suite/parser.test.ts index e2d640a..61ddca2 100644 --- a/src/test/suite/parser.test.ts +++ b/src/test/suite/parser.test.ts @@ -5,13 +5,16 @@ import * as assert from 'assert'; import { Parser } from '../../shared/parser'; -import { Lexer, TokenType } from '../../shared/lexer'; +import { getLanguageConfig, Lexer, TokenType } from '../../shared/lexer'; import { normalizePath, HostInterface, NormalizedPath } from '../../interfaces/hostinterface'; import { ConfigKey, FullConfigInterface } from '../../interfaces/configinterface'; suite('Parser Tests', () => { const testFile = normalizePath('/test/script.lsl'); + const lslLanguageConfig = getLanguageConfig('lsl'); + const luauLanguageConfig = getLanguageConfig('luau'); + // Create a minimal mock host for testing URI conversions function createMockHost(): HostInterface { return new class implements HostInterface { @@ -75,10 +78,10 @@ suite('Parser Tests', () => { const source = `integer x = 42; string s = "hello";`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.strictEqual(result.source, source); @@ -90,10 +93,10 @@ string s = "hello";`; const source = `#include "common.lsl" integer x = 42;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.strictEqual(result.includes.length, 1); @@ -105,10 +108,10 @@ integer x = 42;`; const source = `require("module.luau") local x = 42`; - const lexer = new Lexer(source, 'luau'); + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'luau'); + const parser = new Parser(tokens, testFile, luauLanguageConfig); const result = await parser.parse(); assert.strictEqual(result.includes.length, 1); @@ -120,10 +123,10 @@ local x = 42`; const source = `#define PI 3.14159 float area = PI * r * r;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.strictEqual(result.macros.length, 1); @@ -135,10 +138,10 @@ float area = PI * r * r;`; const source = `#define PI 3.14159 float x = PI;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Check that PI was expanded to 3.14159 @@ -153,10 +156,10 @@ llOwnerSay("Debug mode"); #endif llOwnerSay("Always");`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Both lines should be in output @@ -170,10 +173,10 @@ llOwnerSay("This should not appear"); #endif llOwnerSay("This should appear");`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Only the second line should be in output @@ -188,10 +191,10 @@ llOwnerSay("Not included"); llOwnerSay("Included"); #endif`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.ok(!result.source.includes('Not included')); @@ -206,10 +209,10 @@ llOwnerSay("Included"); llOwnerSay("Not included"); #endif`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.ok(!result.source.includes('Not included')); @@ -224,10 +227,10 @@ llOwnerSay("Not included"); #define TEST 2 #endif`; - const lexer1 = new Lexer(source1, 'lsl'); + const lexer1 = new Lexer(source1, lslLanguageConfig); const tokens1 = lexer1.tokenize(); - const parser1 = new Parser(tokens1, testFile, 'lsl'); + const parser1 = new Parser(tokens1, testFile, lslLanguageConfig); const result1 = await parser1.parse(); const macro1 = parser1.getState().macros.getMacro("TEST"); @@ -244,10 +247,10 @@ llOwnerSay("Not included"); #define TEST 2 #endif`; - const lexer2 = new Lexer(source2, 'lsl'); + const lexer2 = new Lexer(source2, lslLanguageConfig); const tokens2 = lexer2.tokenize(); - const parser2 = new Parser(tokens2, testFile, 'lsl'); + const parser2 = new Parser(tokens2, testFile, lslLanguageConfig); const result2 = await parser2.parse(); const macro2 = parser2.getState().macros.getMacro("TEST"); @@ -262,10 +265,10 @@ llOwnerSay("Not included"); line 2 line 3`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.ok(result.mappings.length > 0); @@ -277,10 +280,10 @@ line 3`; const source = `#define MAX(a, b) ((a) > (b) ? (a) : (b)) integer x = 5;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.strictEqual(result.macros.length, 1); @@ -297,10 +300,10 @@ float x = PI; #undef PI float y = PI;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // After preprocessing: @@ -327,10 +330,10 @@ float y = PI;`; test('define with line continuation', async () => { const source = `#define LONG_MACRO \\\n value1 \\\n value2 \\\n value3\nresult`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); await parser.parse(); // Macro should be defined with all three values @@ -349,10 +352,10 @@ float y = PI;`; test('define with multiple line continuations', async () => { const source = `#define MULTI \\\n a + \\\n b + \\\n c + \\\n d\nx`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); await parser.parse(); const macros = parser.getState().macros.getAllMacros(); @@ -374,10 +377,10 @@ float y = PI;`; test('define without line continuation', async () => { const source = `#define SIMPLE value\nresult`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); await parser.parse(); const macros = parser.getState().macros.getAllMacros(); @@ -389,10 +392,10 @@ float y = PI;`; test('line continuation only works at end of line', async () => { const source = `#define TEST \\ value\nresult`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); await parser.parse(); const macros = parser.getState().macros.getAllMacros(); @@ -407,10 +410,10 @@ float y = PI;`; test('function-like macro with line continuation', async () => { const source = `#define FUNC(x) \\\n ((x) * \\\n (x))\nFUNC(5)`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); await parser.parse(); const macros = parser.getState().macros.getAllMacros(); @@ -425,10 +428,10 @@ float y = PI;`; test('empty line after backslash', async () => { const source = `#define TEST \\\n\\\n value\nresult`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); await parser.parse(); const macros = parser.getState().macros.getAllMacros(); @@ -441,10 +444,10 @@ float y = PI;`; test('macro expansion with line-continued definition', async () => { const source = `#define ADD(a,b) \\\n ((a) + \\\n (b))\nADD(1,2)`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); await parser.parse(); // Verify the macro was defined with line continuation @@ -469,10 +472,10 @@ float y = PI;`; const source = `#define VECTOR <1.0, 2.0, 3.0> vector v = VECTOR;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Verify macro definition @@ -500,10 +503,10 @@ vector v = VECTOR;`; const source = `#define SQUARED_SUM (x * x) + (y * y) integer result = SQUARED_SUM;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Verify macro definition @@ -524,10 +527,10 @@ integer result = SQUARED_SUM;`; const source = `#define MESSAGE "Hello, " + "World!" string msg = MESSAGE;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Verify macro definition @@ -566,10 +569,10 @@ llOwnerSay("outer2"); #endif llOwnerSay("always");`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // All messages should be included @@ -589,10 +592,10 @@ llOwnerSay("inner"); #endif llOwnerSay("always");`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Only "always" should be included @@ -613,10 +616,10 @@ llOwnerSay("C"); llOwnerSay("none"); #endif`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Only "B" should be included @@ -636,10 +639,10 @@ integer x = VALUE; #define VALUE 20 integer y = VALUE;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // First usage should expand to 10 @@ -655,10 +658,10 @@ integer y = VALUE;`; const source = `#define FUNC() 42 integer x = FUNC();`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); await parser.parse(); // Verify macro definition @@ -674,10 +677,10 @@ integer x = FUNC();`; const source = `#define FUNC(x) x * 2 integer ptr = FUNC;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // FUNC without parentheses should remain as identifier @@ -694,10 +697,10 @@ integer ptr = FUNC;`; #define MACROB 2 * MACROA integer x = MACROB;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // MACROB should expand to "2 * 5" (with MACROA expanded) @@ -713,10 +716,10 @@ integer x = MACROB;`; #define C B + 1 integer x = C;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // C should expand to "1 + 1 + 1" through recursive expansion @@ -732,10 +735,10 @@ integer x = C;`; const source = `#define RECURSIVE RECURSIVE integer x = RECURSIVE;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // RECURSIVE should not expand to prevent infinite recursion @@ -747,10 +750,10 @@ integer x = RECURSIVE;`; #define B A integer x = A;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Should stop at the circular reference @@ -765,10 +768,10 @@ integer x = A;`; #define SIXTEEN FOUR * FOUR integer x = SIXTEEN;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // SIXTEEN should expand to "2 * 2 * 2 * 2" @@ -787,10 +790,10 @@ integer x = SIXTEEN;`; const source = `#define EMPTY integer x = 1;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); await parser.parse(); // Verify macro exists but has empty body @@ -804,10 +807,10 @@ integer x = 1;`; const source = `#define WHITESPACE integer x = 1;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); await parser.parse(); // Verify macro exists but has empty body (whitespace is not included) @@ -821,10 +824,10 @@ integer x = 1;`; const source = `#define EMPTY integer x = EMPTY 42;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // EMPTY should expand to nothing, leaving "integer x = 42;" @@ -841,10 +844,10 @@ integer x = EMPTY 42;`; llOwnerSay("enabled"); #endif`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Empty macro should still be recognized as defined for #ifdef @@ -860,10 +863,10 @@ llOwnerSay("enabled"); #endif integer x = 1;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.strictEqual(result.includes.length, 1, 'Should detect 1 include'); @@ -881,10 +884,10 @@ integer x = 1;`; integer x = 1;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.strictEqual(result.includes.length, 1, 'Should detect 1 include'); @@ -897,10 +900,10 @@ integer x = 1;`; #include "lib3.lsl" integer x = 1;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.strictEqual(result.includes.length, 3, 'Should detect 3 includes'); @@ -915,10 +918,10 @@ integer x = 1;`; #include integer x = 1;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.strictEqual(result.includes.length, 3, 'Should detect 3 includes'); @@ -934,10 +937,10 @@ integer x = 1;`; LOG("debug message"); #endif`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // LOG should be expanded within the ifdef block @@ -954,10 +957,10 @@ llOwnerSay("not defined"); llOwnerSay("should not appear"); #endif`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); assert.ok(result.source.includes('"not defined"')); @@ -970,7 +973,7 @@ llOwnerSay("should not appear"); test('should process #include directives with host interface', async () => { const includeContent = '#define PI 3.14159'; - const mainContent = `#include "lib.lsl" + const source = `#include "lib.lsl" float area = PI * r * r;`; // Create a mock host interface @@ -997,27 +1000,27 @@ float area = PI * r * r;`; }, }; - const lexer = new Lexer(mainContent, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - console.log('Main tokens:', tokens.map(t => `${t.type}:${t.value}`)); + // console.log('Main tokens:', tokens.map(t => `${t.type}:${t.value}`)); - const parser = new Parser(tokens, testFile, 'lsl', mockHost as any); + const parser = new Parser(tokens, testFile, lslLanguageConfig, mockHost as any); // Add debugging to see what's happening during parsing - console.log('Parser state before parse:', { - hasMacros: parser.getState().macros !== undefined, - hasConditionals: parser.getState().conditionals !== undefined, - hasIncludes: parser.getState().includes !== undefined - }); + // console.log('Parser state before parse:', { + // hasMacros: parser.getState().macros !== undefined, + // hasConditionals: parser.getState().conditionals !== undefined, + // hasIncludes: parser.getState().includes !== undefined + // }); const result = await parser.parse(); // Debug: Print the actual result - console.log('Result source:', JSON.stringify(result.source)); - console.log('Result source length:', result.source.length); - console.log('Result includes:', result.includes); - console.log('Result macros:', result.macros); + // console.log('Result source:', JSON.stringify(result.source)); + // console.log('Result source length:', result.source.length); + // console.log('Result includes:', result.includes); + // console.log('Result macros:', result.macros); // The included file should have been processed assert.ok(result.includes.length === 1); @@ -1058,11 +1061,11 @@ float area = PI * r * r;`; }, }; - const mainContent = '#include "a.lsl"'; - const lexer = new Lexer(mainContent, 'lsl'); + const source = '#include "a.lsl"'; + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl', mockHost as any); + const parser = new Parser(tokens, testFile, lslLanguageConfig, mockHost as any); // Parse should complete but collect circular include error const result = await parser.parse(); @@ -1104,14 +1107,14 @@ float area = PI * r * r;`; }, }; - const mainContent = `#include "lib.lsl" + const source = `#include "lib.lsl" #include "lib.lsl" integer x = 1;`; - const lexer = new Lexer(mainContent, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl', mockHost as any); + const parser = new Parser(tokens, testFile, lslLanguageConfig, mockHost as any); const result = await parser.parse(); // The file should only be read and included once due to include guards @@ -1163,11 +1166,11 @@ integer x = 1;`; }, }; - const mainContent = '#include "a.lsl"'; - const lexer = new Lexer(mainContent, 'lsl'); + const source = '#include "a.lsl"'; + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl', mockHost as any); + const parser = new Parser(tokens, testFile, lslLanguageConfig, mockHost as any); // Parse should complete but collect depth exceeded error const result = await parser.parse(); @@ -1190,7 +1193,7 @@ string s = "hello"; // @line 5 "${testFile}" float y = 3.14;`; - const mappings = Parser.parseLineMappingsFromContent(content, 'lsl', createMockHost()); + const mappings = Parser.parseLineMappingsFromContent(content, lslLanguageConfig, createMockHost()); // Should have mappings for actual code lines (not directive lines or blank lines) assert.strictEqual(mappings.length, 4); @@ -1217,7 +1220,7 @@ local s = "hello" -- @line 10 "${testFile}" local y = 3.14`; - const mappings = Parser.parseLineMappingsFromContent(content, 'luau', createMockHost()); + const mappings = Parser.parseLineMappingsFromContent(content, luauLanguageConfig, createMockHost()); assert.strictEqual(mappings.length, 4); @@ -1245,7 +1248,7 @@ float PI = 3.14159; // Back to main integer y = 2;`; - const mappings = Parser.parseLineMappingsFromContent(content, 'lsl', createMockHost()); + const mappings = Parser.parseLineMappingsFromContent(content, lslLanguageConfig, createMockHost()); // Find mapping for line 2 (integer x = 1;) const mainMapping = mappings.find(m => m.processedLine === 2); @@ -1277,7 +1280,7 @@ integer y = 2;`; string s = "hello"; float y = 3.14;`; - const mappings = Parser.parseLineMappingsFromContent(content, 'lsl', createMockHost()); + const mappings = Parser.parseLineMappingsFromContent(content, lslLanguageConfig, createMockHost()); // No directives means no mappings assert.strictEqual(mappings.length, 0); @@ -1292,7 +1295,7 @@ integer x = 42; // Another comment string s = "hello";`; - const mappings = Parser.parseLineMappingsFromContent(content, 'lsl', createMockHost()); + const mappings = Parser.parseLineMappingsFromContent(content, lslLanguageConfig, createMockHost()); // Should have mappings for all lines after the directive (including blank and comment lines) assert.ok(mappings.length > 0); @@ -1308,7 +1311,7 @@ string s = "hello";`; const content = `// @line 1 "${relativePath}" float PI = 3.14159;`; - const mappings = Parser.parseLineMappingsFromContent(content, 'lsl', createMockHost()); + const mappings = Parser.parseLineMappingsFromContent(content, lslLanguageConfig, createMockHost()); assert.strictEqual(mappings.length, 1); // Relative paths should be resolved to absolute paths @@ -1324,10 +1327,10 @@ float PI = 3.14159;`; integer x = 42; string s = "hello";`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Should have at least some mappings (one per newline in output) @@ -1348,10 +1351,10 @@ string s = "hello";`; const source = `integer x = 42; integer y = 100;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // For simple single-file code, @line directives are only inserted on line skips @@ -1369,10 +1372,10 @@ integer y = 100;`; string s = "hello";`; // Note: 2 blank lines - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Should have @line directive for the line skip @@ -1388,10 +1391,10 @@ string s = "hello";`; // Note: 2 blank lines float circumference = 2 * PI * 5.0; integer x = 42;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Should have at least one mapping (for lines with newlines in output) @@ -1410,10 +1413,10 @@ integer debugMode = 0; #endif integer x = 42;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Should have mappings for lines that made it through preprocessing @@ -1434,10 +1437,10 @@ integer x = 42;`; #endif integer y = 2;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Should have mappings for processed output @@ -1455,10 +1458,10 @@ integer y = 2; integer z = 3;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Blank lines should be preserved in output @@ -1473,10 +1476,10 @@ integer z = 3;`; integer result = SQUARE(5); integer y = 10;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Should have at least one mapping @@ -1493,10 +1496,10 @@ integer v = VERSION; #endif integer x = 42;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Should have mappings for the output @@ -1515,10 +1518,10 @@ integer x = 42;`; integer y = 2; integer z = 3;`; - const lexer = new Lexer(source, 'lsl'); + const lexer = new Lexer(source, lslLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'lsl'); + const parser = new Parser(tokens, testFile, lslLanguageConfig); const result = await parser.parse(); // Should have at least one mapping diff --git a/src/test/suite/require-table.test.ts b/src/test/suite/require-table.test.ts index 9ce7ba0..1c25a50 100644 --- a/src/test/suite/require-table.test.ts +++ b/src/test/suite/require-table.test.ts @@ -13,11 +13,13 @@ import * as assert from 'assert'; import * as path from 'path'; import { Parser } from '../../shared/parser'; -import { Lexer } from '../../shared/lexer'; +import { getLanguageConfig, Lexer } from '../../shared/lexer'; import { HostInterface, NormalizedPath, normalizePath } from '../../interfaces/hostinterface'; +import { get } from 'http'; suite('Require Table Tests', () => { const testFile = normalizePath('/test/main.luau'); + const luauLanguageConfig = getLanguageConfig('luau'); /** * Create a minimal mock host for testing with in-memory files @@ -64,10 +66,10 @@ suite('Require Table Tests', () => { const host = createMockHost(files); - const lexer = new Lexer('local result = require("module.luau")', 'luau'); + const lexer = new Lexer('local result = require("module.luau")', luauLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'luau', host); + const parser = new Parser(tokens, testFile, luauLanguageConfig, host); const result = await parser.parse(); // Should contain function wrapper @@ -83,10 +85,10 @@ suite('Require Table Tests', () => { const host = createMockHost(files); - const lexer = new Lexer('local result = require("module.luau")', 'luau'); + const lexer = new Lexer('local result = require("module.luau")', luauLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'luau', host); + const parser = new Parser(tokens, testFile, luauLanguageConfig, host); const result = await parser.parse(); // Should start with table declaration @@ -102,10 +104,10 @@ suite('Require Table Tests', () => { const host = createMockHost(files); - const lexer = new Lexer('local result = require("module.luau")', 'luau'); + const lexer = new Lexer('local result = require("module.luau")', luauLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'luau', host); + const parser = new Parser(tokens, testFile, luauLanguageConfig, host); const result = await parser.parse(); // Should have invocation in place of require @@ -121,10 +123,10 @@ suite('Require Table Tests', () => { const host = createMockHost(files); const source = 'require("module.luau")\nrequire("module.luau")'; - const lexer = new Lexer(source, 'luau'); + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'luau', host); + const parser = new Parser(tokens, testFile, luauLanguageConfig, host); const result = await parser.parse(); // Count occurrences of module in table - should be only once @@ -147,10 +149,10 @@ suite('Require Table Tests', () => { const host = createMockHost(files); const source = 'require("module1.luau")\nrequire("module2.luau")'; - const lexer = new Lexer(source, 'luau'); + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'luau', host); + const parser = new Parser(tokens, testFile, luauLanguageConfig, host); const result = await parser.parse(); // Should have both modules in table @@ -170,10 +172,11 @@ suite('Require Table Tests', () => { const host = createMockHost(files); - const lexer = new Lexer('require("module.luau")', 'luau'); + const source = 'require("module.luau")'; + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'luau', host); + const parser = new Parser(tokens, testFile, luauLanguageConfig, host); const result = await parser.parse(); // Should have @line directive in wrapped module @@ -184,10 +187,10 @@ suite('Require Table Tests', () => { test('should not emit table if no requires', async () => { const source = 'local x = 42'; - const lexer = new Lexer(source, 'luau'); + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'luau'); + const parser = new Parser(tokens, testFile, luauLanguageConfig); const result = await parser.parse(); // Should not have require table if no requires @@ -204,10 +207,11 @@ suite('Require Table Tests', () => { const host = createMockHost(files); - const lexer = new Lexer('require("module.luau")', 'luau'); + const source = 'require("module.luau")'; + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, testFile, 'luau', host); + const parser = new Parser(tokens, testFile, luauLanguageConfig, host); const result = await parser.parse(); // Should have both modules in table @@ -274,12 +278,12 @@ suite('Require Table Tests', () => { const host = createFileHost(workspaceRoot); // Read the main file - const mainContent = fs.readFileSync(mainFile, 'utf8'); - const lexer = new Lexer(mainContent, 'luau'); + const source = fs.readFileSync(mainFile, 'utf8'); + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); // Parse with the file host - const parser = new Parser(tokens, mainFile, 'luau', host); + const parser = new Parser(tokens, mainFile, luauLanguageConfig, host); const result = await parser.parse(); // Check that all modules are in the table @@ -315,12 +319,12 @@ suite('Require Table Tests', () => { const host = createFileHost(workspaceRoot); // Read the main file - const mainContent = fs.readFileSync(mainFile, 'utf8'); - const lexer = new Lexer(mainContent, 'luau'); + const source = fs.readFileSync(mainFile, 'utf8'); + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); // Parse with the file host - const parser = new Parser(tokens, mainFile, 'luau', host); + const parser = new Parser(tokens, mainFile, luauLanguageConfig, host); const result = await parser.parse(); // Check that D appears only once in the table @@ -399,10 +403,10 @@ suite('Require Table Tests', () => { files.set(fileA, 'local b = require("complex_b.luau")\n--file a\nlocal c = require("complex_c.luau")\nprint(b.getB(), c.getC())'); // Parse A - const mainContent = files.get(fileA)!; - const lexer = new Lexer(mainContent, 'luau'); + const source = files.get(fileA)!; + const lexer = new Lexer(source, luauLanguageConfig); const tokens = lexer.tokenize(); - const parser = new Parser(tokens, fileA, 'luau', memoryHost); + const parser = new Parser(tokens, fileA, luauLanguageConfig, memoryHost); const result = await parser.parse(); // // Debug: Log the actual output to see what indices are used diff --git a/src/test/workspace/set_1/include/debug.lsl b/src/test/workspace/set_1/include/debug.lsl new file mode 100644 index 0000000..81049ad --- /dev/null +++ b/src/test/workspace/set_1/include/debug.lsl @@ -0,0 +1,12 @@ +#ifndef TEST +#define TEST + +#if DEBUG +debug(string stuff) { + llOwnerSay(stuff); +} +#else +#define debug(a) +#endif + +#endif diff --git a/src/test/workspace/set_1/test_define_nix_function.lsl b/src/test/workspace/set_1/test_define_nix_function.lsl new file mode 100644 index 0000000..cac7042 --- /dev/null +++ b/src/test/workspace/set_1/test_define_nix_function.lsl @@ -0,0 +1,3 @@ +#define debug(a) +// Call debug +debug("This is a debug message."); diff --git a/src/test/workspace/set_1/test_define_nix_function_expected.lsl b/src/test/workspace/set_1/test_define_nix_function_expected.lsl new file mode 100644 index 0000000..a9a93bf --- /dev/null +++ b/src/test/workspace/set_1/test_define_nix_function_expected.lsl @@ -0,0 +1,2 @@ +// Call debug +; diff --git a/src/test/workspace/set_1/test_nested_if_define_debug.lsl b/src/test/workspace/set_1/test_nested_if_define_debug.lsl new file mode 100644 index 0000000..6ff9776 --- /dev/null +++ b/src/test/workspace/set_1/test_nested_if_define_debug.lsl @@ -0,0 +1,3 @@ +// Include with Debug +#define DEBUG 1 +#include "include/debug.lsl" diff --git a/src/test/workspace/set_1/test_nested_if_define_debug_expected.lsl b/src/test/workspace/set_1/test_nested_if_define_debug_expected.lsl new file mode 100644 index 0000000..894bb51 --- /dev/null +++ b/src/test/workspace/set_1/test_nested_if_define_debug_expected.lsl @@ -0,0 +1,6 @@ +// Include with Debug +// @line 1 "file:///test/workspace/set_1/include/debug.lsl" + +debug(string stuff) { + llOwnerSay(stuff); +} diff --git a/src/test/workspace/set_1/test_nested_if_define_no_debug.lsl b/src/test/workspace/set_1/test_nested_if_define_no_debug.lsl new file mode 100644 index 0000000..76dcd86 --- /dev/null +++ b/src/test/workspace/set_1/test_nested_if_define_no_debug.lsl @@ -0,0 +1,3 @@ +// Include without Debug + +#include "include/debug.lsl" diff --git a/src/test/workspace/set_1/test_nested_if_define_no_debug_expected.lsl b/src/test/workspace/set_1/test_nested_if_define_no_debug_expected.lsl new file mode 100644 index 0000000..b2281a8 --- /dev/null +++ b/src/test/workspace/set_1/test_nested_if_define_no_debug_expected.lsl @@ -0,0 +1,3 @@ +// Include without Debug + +// @line 1 "file:///test/workspace/set_1/include/debug.lsl" diff --git a/src/test/workspace/set_1/test_switch.lsl b/src/test/workspace/set_1/test_switch.lsl new file mode 100644 index 0000000..5be185a --- /dev/null +++ b/src/test/workspace/set_1/test_switch.lsl @@ -0,0 +1,58 @@ +default { + state_entry() + { + string test = "test"; + switch (test) + { + case "test": + { + llOwnerSay("Switch case matched 'test'"); + break; + } + default: + { + llOwnerSay("Switch case did not match"); + break; + } + } + + integer i; + switch(i) + { + case 1: + { + llOwnerSay("1"); + // fallthrough to case 2 + } + case 2: + { + llOwnerSay("1 or 2"); + // no fallthrough + break; + } + case 3: + { + llOwnerSay("3"); + // fallthrough to default + } + default: + { + llOwnerSay("3 or default"); + } + } + + switch(i) + { + case 1: // needs colon + case 2 // colon optional as curly brace opens next + { + llOwnerSay("x is 1 or 2"); + break; + } + default // colon optional as curly brace opens next + { + llOwnerSay("x is neither 1 nor 2"); + } + } + } +} diff --git a/src/test/workspace/set_1/test_switch_expected.lsl b/src/test/workspace/set_1/test_switch_expected.lsl new file mode 100644 index 0000000..dd01ca5 --- /dev/null +++ b/src/test/workspace/set_1/test_switch_expected.lsl @@ -0,0 +1,69 @@ +default { + state_entry() + { + string test = "test"; + if((test) == ("test")) jump c860cf; + jump c7b43f; + @c860cf; + { + // @line 9 "file:///test/workspace/set_1/test_switch.lsl" + llOwnerSay("Switch case matched 'test'"); + jump sfcc97; + } + @c7b43f; + { + // @line 14 "file:///test/workspace/set_1/test_switch.lsl" + llOwnerSay("Switch case did not match"); + jump sfcc97; + } + @sfcc97; + + integer i; + if((i) == (1)) jump cb2d31; + if((i) == (2)) jump c8ba57; + if((i) == (3)) jump ca5466; + jump c7cb55; + @cb2d31; + { + // @line 24 "file:///test/workspace/set_1/test_switch.lsl" + llOwnerSay("1"); + // fallthrough to case 2 + } + @c8ba57; + { + // @line 29 "file:///test/workspace/set_1/test_switch.lsl" + llOwnerSay("1 or 2"); + // no fallthrough + jump s5c77b; + } + @ca5466; + { + // @line 35 "file:///test/workspace/set_1/test_switch.lsl" + llOwnerSay("3"); + // fallthrough to default + } + @c7cb55; + { + // @line 40 "file:///test/workspace/set_1/test_switch.lsl" + llOwnerSay("3 or default"); + } + @s5c77b; + + if((i) == (1)) jump cd0f56; + if((i) == (2 // colon optional as curly brace opens next + )) jump ccc445; + jump cd702f; + @cd0f56; // needs colon + @ccc445; + { + llOwnerSay("x is 1 or 2"); + jump sc2783; + } + @cd702f; + { + // @line 54 "file:///test/workspace/set_1/test_switch.lsl" + llOwnerSay("x is neither 1 nor 2"); + } + @sc2783; + } +}