Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 19 additions & 14 deletions packages/apidom-parser-adapter-json/src/adapter-browser.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { ParseResultElement } from '@swagger-api/apidom-core';
import { Tree as WebTree } from 'web-tree-sitter';

import lexicalAnalysis from './lexical-analysis/browser.ts';
import syntacticAnalysisDirect from './syntactic-analysis/direct/index.ts';
Expand All @@ -25,15 +26,19 @@ export const detect = async (source: string): Promise<boolean> => {
return false;
}

let cst: WebTree | null = null;

try {
const cst = await lexicalAnalysis(source);
cst = await lexicalAnalysis(source);
const isError = cst.rootNode.type !== 'ERROR';

cst.delete();

return isError;
} catch {
return false;
} finally {
if (cst !== null) {
cst.delete();
}
}
};

Expand All @@ -60,16 +65,16 @@ export const parse: ParseFunction = async (
source,
{ sourceMap = false, syntacticAnalysis = 'direct' } = {},
) => {
const cst = await lexicalAnalysis(source);
let apiDOM;

if (syntacticAnalysis === 'indirect') {
apiDOM = syntacticAnalysisIndirect(cst, { sourceMap });
} else {
apiDOM = syntacticAnalysisDirect(cst, { sourceMap });
let cst: WebTree | null = null;
try {
cst = await lexicalAnalysis(source);
if (syntacticAnalysis === 'indirect') {
return syntacticAnalysisIndirect(cst, { sourceMap });
}
return syntacticAnalysisDirect(cst, { sourceMap });
} finally {
if (cst !== null) {
cst.delete();
}
}

cst.delete();

return apiDOM;
};
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,12 @@ import treeSitterJson from '../../wasm/tree-sitter-json.wasm';

let parser: Parser | null = null;
let parserInitLock: Promise<Parser> | null = null;
let currentTree: Tree | null = null;
const activeTrees: Set<Tree> = new Set();
const MAX_ACTIVE_TREES = 5;

/**
* Lexical Analysis of source string using WebTreeSitter.
* This is WebAssembly version of TreeSitters Lexical Analysis.
*
* Given JavaScript doesn't support true parallelism, this
* code should be as lazy as possible and temporal safety should be fine.
* @public
*/
const analyze = async (source: string): Promise<Tree> => {
Expand All @@ -42,11 +40,30 @@ const analyze = async (source: string): Promise<Tree> => {
);
}

currentTree = parser.parse(source);
// prevent WASM OOM during concurrency spikes by evicting oldest trees
// when the pool exceeds threshold; tree.delete() is idempotent so
// callers that still hold a reference can safely call delete() again
if (activeTrees.size >= MAX_ACTIVE_TREES) {
const treesToEvict = [...activeTrees];
activeTrees.clear();
for (const oldTree of treesToEvict) {
oldTree.delete();
}
}

const tree = parser.parse(source);
activeTrees.add(tree);

// remove from tracking when caller deletes
const originalDelete = tree.delete;
tree.delete = function deleteAndUntrack() {
activeTrees.delete(this);
originalDelete.call(this);
};

parser.reset();

return currentTree;
return tree;
};

export default analyze;
21 changes: 21 additions & 0 deletions packages/apidom-parser-adapter-json/test/adapter-browser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,27 @@ describe('adapter-browser', function () {
expect(sexprs(parseResult)).toMatchSnapshot();
});

context('lexical analysis', function () {
specify('should return independent trees across multiple calls', async function () {
const source1 = '{"key1": "value1"}';
const source2 = '{"key2": "value2"}';

const tree1 = await adapter.lexicalAnalysis(source1);
const tree2 = await adapter.lexicalAnalysis(source2);

// tree1 must still be usable after second call
assert.isNotNull(tree1.rootNode);
assert.include(tree1.rootNode.text, 'key1');

// tree2 is also valid
assert.isNotNull(tree2.rootNode);
assert.include(tree2.rootNode.text, 'key2');

tree1.delete();
tree2.delete();
});
});

context('given direct syntactic analysis', function () {
context('given zero byte empty file', function () {
specify('should return empty parse result', async function () {
Expand Down
28 changes: 18 additions & 10 deletions packages/apidom-parser-adapter-yaml-1-2/src/adapter-browser.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { ParseResultElement } from '@swagger-api/apidom-core';
import { Tree as WebTree } from 'web-tree-sitter';

import lexicalAnalysis from './lexical-analysis/browser.ts';
import syntacticAnalysis from './syntactic-analysis/indirect/index.ts';
Expand All @@ -12,15 +13,18 @@ export { lexicalAnalysis, syntacticAnalysis };
* @public
*/
export const detect = async (source: string): Promise<boolean> => {
try {
const cst = await lexicalAnalysis(source);
const isError = !cst.rootNode.isError;
let cst: WebTree | null = null;

cst.delete();
try {
cst = await lexicalAnalysis(source);

return isError;
return !cst.rootNode.isError;
} catch {
return false;
} finally {
if (cst !== null) {
cst.delete();
}
}
};

Expand All @@ -43,10 +47,14 @@ export type ParseFunction = (
* @public
*/
export const parse: ParseFunction = async (source, { sourceMap = false } = {}) => {
const cst = await lexicalAnalysis(source);
const syntacticAnalysisResult = syntacticAnalysis(cst, { sourceMap });

cst.delete();
let cst: WebTree | null = null;

return syntacticAnalysisResult;
try {
cst = await lexicalAnalysis(source);
return syntacticAnalysis(cst, { sourceMap });
} finally {
if (cst !== null) {
cst.delete();
}
}
};
10 changes: 8 additions & 2 deletions packages/apidom-parser-adapter-yaml-1-2/src/adapter-node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ export { lexicalAnalysis, syntacticAnalysis };
* @public
*/
export const detect = async (source: string): Promise<boolean> => {
const cst = await lexicalAnalysis(source);
try {
const cst = await lexicalAnalysis(source);
return !cst.rootNode.isError;
} catch {
return false;
} finally {
cst.delete();
}
};

Expand All @@ -40,5 +42,9 @@ export type ParseFunction = (
*/
export const parse: ParseFunction = async (source, { sourceMap = false } = {}) => {
const cst = await lexicalAnalysis(source);
return syntacticAnalysis(cst, { sourceMap });
try {
return syntacticAnalysis(cst, { sourceMap });
} finally {
cst.delete();
}
};
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ import { ApiDOMError } from '@swagger-api/apidom-error';

let parser: Parser | null = null;
let parserInitLock: Promise<Parser> | null = null;
let currentTree: Tree | null = null;
const activeTrees: Set<Tree> = new Set();
const MAX_ACTIVE_TREES = 5;

const createAnalyze =
(treeSitterYaml: string | Uint8Array) =>
Expand Down Expand Up @@ -33,14 +34,30 @@ const createAnalyze =
);
}

if (currentTree !== null) {
currentTree.delete();
// prevent WASM OOM during concurrency spikes by evicting oldest trees
// when the pool exceeds threshold; tree.delete() is idempotent so
// callers that still hold a reference can safely call delete() again
if (activeTrees.size >= MAX_ACTIVE_TREES) {
const treesToEvict = [...activeTrees];
activeTrees.clear();
for (const oldTree of treesToEvict) {
oldTree.delete();
}
}
currentTree = parser.parse(source);

const tree = parser.parse(source);
activeTrees.add(tree);

// remove from tracking when caller deletes
const originalDelete = tree.delete;
tree.delete = function deleteAndUntrack() {
activeTrees.delete(this);
originalDelete.call(this);
};

parser.reset();

return currentTree;
return tree;
};

export default createAnalyze;
21 changes: 21 additions & 0 deletions packages/apidom-parser-adapter-yaml-1-2/test/adapter-browser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,27 @@ describe('adapter-browser', function () {
});
});

context('lexical analysis', function () {
specify('should return independent trees across multiple calls', async function () {
const source1 = 'key1: value1';
const source2 = 'key2: value2';

const tree1 = await adapter.lexicalAnalysis(source1);
const tree2 = await adapter.lexicalAnalysis(source2);

// tree1 must still be usable after second call
assert.isNotNull(tree1.rootNode);
assert.include(tree1.rootNode.text, 'key1');

// tree2 is also valid
assert.isNotNull(tree2.rootNode);
assert.include(tree2.rootNode.text, 'key2');

tree1.delete();
tree2.delete();
});
});

context('given single-quote scalar containing only space characters', function () {
specify('should parse all space characters', async function () {
const result = await adapter.parse("' '");
Expand Down
14 changes: 14 additions & 0 deletions packages/apidom-parser-adapter-yaml-1-2/test/adapter-node/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,20 @@ describe('adapter-node', function () {
assert.isTrue(isObjectElement(parseResult.result));
assert.lengthOf(parseResult.errors, 0);
});

specify('should handle concurrency spikes', async function () {
this.timeout(30000);

Array.from({ length: 100 }).forEach(async () => {
await adapter.parse('test: 123\n'.repeat(32800));
});

const parseResult = await adapter.parse('test: 123\n'.repeat(32800));

assert.isFalse(parseResult.isEmpty);
assert.isTrue(isObjectElement(parseResult.result));
assert.lengthOf(parseResult.errors, 0);
});
});

context('given an alias', function () {
Expand Down
Loading