From 7d42a8edfb0b61826f9373677a0661fd30e8f1f9 Mon Sep 17 00:00:00 2001 From: andrewblond Date: Sun, 25 Mar 2018 18:50:26 +0300 Subject: [PATCH 1/2] perf(gulp-bem-src): cache level introspection --- packages/gulp-bem-src/lib/build-bem-graph.js | 15 ++--- packages/gulp-bem-src/lib/gulp-bem-src.js | 20 +++++-- packages/gulp-bem-src/lib/introspect-fs.js | 22 ------- .../lib/introspect-levels/index.js | 1 + .../lib/introspect-levels/introspect-level.js | 49 +++++++++++++++ .../introspect-levels/introspect-levels.js | 30 ++++++++++ .../lib/introspect-levels/introspection.js | 59 +++++++++++++++++++ packages/gulp-bem-src/package.json | 4 +- packages/gulp-bem-src/test/harvest.test.js | 13 +++- 9 files changed, 172 insertions(+), 41 deletions(-) delete mode 100644 packages/gulp-bem-src/lib/introspect-fs.js create mode 100644 packages/gulp-bem-src/lib/introspect-levels/index.js create mode 100644 packages/gulp-bem-src/lib/introspect-levels/introspect-level.js create mode 100644 packages/gulp-bem-src/lib/introspect-levels/introspect-levels.js create mode 100644 packages/gulp-bem-src/lib/introspect-levels/introspection.js diff --git a/packages/gulp-bem-src/lib/build-bem-graph.js b/packages/gulp-bem-src/lib/build-bem-graph.js index 0215afe..22451d1 100644 --- a/packages/gulp-bem-src/lib/build-bem-graph.js +++ b/packages/gulp-bem-src/lib/build-bem-graph.js @@ -6,21 +6,16 @@ const readDeps = deps.read(); const parseDeps = deps.parse(); const buildGraph = deps.buildGraph; -const getDepFiles = (introspection, levels) => { - return introspection.filter(file => file.tech === 'deps.js') - // Сортируем по уровням - .sort((f1, f2) => (levels.indexOf(f1.level) - levels.indexOf(f2.level))) -}; +const loadDeps = async (introspection) => { + const depFiles = Array.from(introspection.techFiles('deps.js')); -const loadDeps = async (introspection, levels) => { - const depFiles = getDepFiles(introspection, levels); const depFilesContents = await readDeps(depFiles); return parseDeps(depFilesContents); -} +}; -module.exports = async (introspection, levels) => { - const deps = await loadDeps(introspection, levels); +module.exports = async (introspection) => { + const deps = await loadDeps(introspection); return buildGraph(deps); }; diff --git a/packages/gulp-bem-src/lib/gulp-bem-src.js b/packages/gulp-bem-src/lib/gulp-bem-src.js index c165c2a..6e34b75 100644 --- a/packages/gulp-bem-src/lib/gulp-bem-src.js +++ b/packages/gulp-bem-src/lib/gulp-bem-src.js @@ -10,7 +10,7 @@ const bemConfig = require('@bem/sdk.config'); const File = require('vinyl'); const read = require('gulp-read'); -const introspectFs = require('./introspect-fs'); +const introspectLevels = require('./introspect-levels'); const buildBemGraph = require('./build-bem-graph'); const resolveDeps = require('./resolve-deps'); const filesToStream = require('./files-to-stream'); @@ -34,11 +34,18 @@ async function _getBundleInfo(sources, decl, tech, options) { const config = options.config || bemConfig(); // Получаем слепок файловой структуры с уровней - const introspection = await introspectFs(sources, config); - const graph = options.skipResolvingDependencies ? null : await buildBemGraph(introspection, sources) - const fulldecl = options.skipResolvingDependencies ? declToEntities(decl, tech) : resolveDeps(decl, graph, tech); + const introspection = await introspectLevels(sources, config); - return { introspection, graph, fulldecl }; + if (options.skipResolvingDependencies) { + const fulldecl = declToEntities(decl, tech); + + return { introspection, fulldecl }; + } + + const graph = await buildBemGraph(introspection); + const fulldecl = resolveDeps(decl, graph, tech); + + return { introspection, fulldecl }; } /** @@ -140,7 +147,8 @@ function harvest(opts) { }, {}); const res = [], depTechForFile = {}; - for (const file of opts.introspection) { + + for (const file of opts.introspection.files()) { if (file.tech && !fileTechToDep[file.tech] && !techMap[file.tech]) { techMap[file.tech] = [file.tech]; fileTechToDep[file.tech] = [file.tech]; diff --git a/packages/gulp-bem-src/lib/introspect-fs.js b/packages/gulp-bem-src/lib/introspect-fs.js deleted file mode 100644 index c397ab7..0000000 --- a/packages/gulp-bem-src/lib/introspect-fs.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict'; - -const assert = require('assert'); -const path = require('path'); - -const walk = require('@bem/sdk.walk'); -const streamToArray = require('stream-to-array'); - -/** - * Получаем слепок файловой структуры с уровней - * - * @param {string[]} levels - levels to use to search files - * @param {*} bemConfig - */ -module.exports = async (levels, bemConfig) => { - assert(Array.isArray(levels) && levels.length, 'Levels required to get some files'); - - const levelMap = await Promise.resolve(bemConfig.levelMap ? bemConfig.levelMap() : {}); - const introspectionStream = walk(levels, { levels: levelMap }); - - return streamToArray(introspectionStream); -}; diff --git a/packages/gulp-bem-src/lib/introspect-levels/index.js b/packages/gulp-bem-src/lib/introspect-levels/index.js new file mode 100644 index 0000000..e5e42e3 --- /dev/null +++ b/packages/gulp-bem-src/lib/introspect-levels/index.js @@ -0,0 +1 @@ +module.exports = require('./introspect-levels'); diff --git a/packages/gulp-bem-src/lib/introspect-levels/introspect-level.js b/packages/gulp-bem-src/lib/introspect-levels/introspect-level.js new file mode 100644 index 0000000..63e0661 --- /dev/null +++ b/packages/gulp-bem-src/lib/introspect-levels/introspect-level.js @@ -0,0 +1,49 @@ +'use strict'; + +const fs = require('fs'); +const { Writable } = require('stream'); + +const walk = require('@bem/sdk.walk'); +const pify = require('pify'); + +/** + * @param {string} levelPath + * @param {*} bemConfig + */ +module.exports = async (levelPath, bemConfig) => { + const levelMap = await Promise.resolve(bemConfig.levelMap ? bemConfig.levelMap() : {}); + + return new Promise((resolve, reject) => { + const entityMap = new Map(); + + walk([levelPath], levelMap) + .on('error', reject) + .pipe(new Writable({ + objectMode: true, + write(file, encoding, callback) { + tryCatch(async () => { + const id = file.entity.id; + const stats = await pify(fs.stat)(file.path); + + file.stats = stats; + + const entityFiles = entityMap.has(id) ? entityMap.get(id) : entityMap.set(id, new Set()).get(id); + entityFiles.add(file); + + callback(); + }, callback); + } + })) + .on('error', reject) + .on('finish', () => resolve(entityMap)); + }); +}; + +// try-catch optimization +function tryCatch(tryFn, catchFn) { + try { + tryFn(); + } catch (err) { + catchFn(err); + } +} diff --git a/packages/gulp-bem-src/lib/introspect-levels/introspect-levels.js b/packages/gulp-bem-src/lib/introspect-levels/introspect-levels.js new file mode 100644 index 0000000..cfd4f31 --- /dev/null +++ b/packages/gulp-bem-src/lib/introspect-levels/introspect-levels.js @@ -0,0 +1,30 @@ +'use strict'; + +const fs = require('fs'); +const { Writable } = require('stream'); + +const introspectLevel = require('./introspect-level'); +const Introspection = require('./introspection'); + +const levelCache = {}; + +/** + * @param {string[]} levelPaths + * @param {*} bemConfig + * @returns {Introspection} + */ +module.exports = async (levelPaths, bemConfig, { cache=false } = {}) => { + const levelIntrospections = await Promise.all(levelPaths.map(levelPath => { + if (cache && levelCache[levelPath]) { + return levelCache[levelPath]; + } + + const introspect = introspectLevel(levelPath, bemConfig); + + levelCache[levelPath] = introspect; + + return introspect; + })); + + return new Introspection(levelPaths, levelIntrospections); +}; diff --git a/packages/gulp-bem-src/lib/introspect-levels/introspection.js b/packages/gulp-bem-src/lib/introspect-levels/introspection.js new file mode 100644 index 0000000..b64d2c5 --- /dev/null +++ b/packages/gulp-bem-src/lib/introspect-levels/introspection.js @@ -0,0 +1,59 @@ +'use strict'; + +/** + * Contains info about files in levels for bundle. + */ +module.exports = class Introspection { + constructor(levelPaths, introspections) { + this._levelPaths = levelPaths; + this._introspections = introspections; + } + /** + * Returns all level paths. + * + * @returns {String[]} + */ + levels() { + return this._levelPaths; + } + /** + * Returns all files. + * + * @returns {Iterator} + */ + *files() { + for (const introspection of this._introspections) { + for (const files of introspection.values()) { + yield* files; + } + } + } + /** + * Returns info about files of specified entity. + * + * @param {Object} entity + * @returns {Iterator} + */ + *entityFiles(entity) { + for (const introspection of this._introspections) { + yield* introspection.get(entity.id); + } + } + /** + * Returns info about files with specified tech. + * + * @param {string} tech + * @returns {Iterator} + */ + *techFiles(tech) { + for (const introspection of this._introspections) { + for (const files of introspection.values()) { + for (const file of files) { + if (file.tech === tech) { + yield file; + } + } + } + } + } +}; diff --git a/packages/gulp-bem-src/package.json b/packages/gulp-bem-src/package.json index d3a059d..47c1bd3 100644 --- a/packages/gulp-bem-src/package.json +++ b/packages/gulp-bem-src/package.json @@ -25,11 +25,11 @@ "@bem/sdk.walk": "0.2.5", "bubble-stream-error": "1.0.0", "gulp-read": "0.0.1", - "stream-to-array": "2.3.0", "vinyl": "2.1.0" }, "devDependencies": { "@bem/sdk.naming.entity.parse": "^0.2.4", - "@bem/sdk.naming.presets": "^0.0.7" + "@bem/sdk.naming.presets": "^0.0.7", + "stream-to-array": "2.3.0" } } diff --git a/packages/gulp-bem-src/test/harvest.test.js b/packages/gulp-bem-src/test/harvest.test.js index 9b63e51..875377a 100644 --- a/packages/gulp-bem-src/test/harvest.test.js +++ b/packages/gulp-bem-src/test/harvest.test.js @@ -3,6 +3,7 @@ const path = require('path'); const parseEntity = require('@bem/sdk.naming.entity.parse')(require('@bem/sdk.naming.presets/origin')); const { assert } = require('chai'); const lib = require('..'); +const Introspection = require('../lib/introspect-levels/introspection'); describe('harvest', () => { it('should filter introspection by entity and tech', () => { @@ -96,7 +97,17 @@ describe('harvest', () => { // {entity: {block: 'button'}, tech: 'css'} function checkHarvest(opts) { - opts.introspection = opts.files.map(makeFileEntity); + const files = opts.files.map(makeFileEntity); + const entityMap = new Map(); + + for (const file of files) { + const id = file.entity.id; + const entityFiles = entityMap.has(id) ? entityMap.get(id) : entityMap.set(id, new Set()).get(id); + + entityFiles.add(file); + } + + opts.introspection = new Introspection([''], [entityMap]); opts.result = opts.result.map(makeFileEntity); opts.decl = opts.decl.map(makeEntity); From ce76b3015af3576683e825e340dc070f83f785bb Mon Sep 17 00:00:00 2001 From: Alexey Yaroshevich Date: Sun, 25 Mar 2018 21:57:34 +0300 Subject: [PATCH 2/2] fixup! perf(gulp-bem-src): cache level introspection --- packages/gulp-bem-src/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/gulp-bem-src/package.json b/packages/gulp-bem-src/package.json index 47c1bd3..127a446 100644 --- a/packages/gulp-bem-src/package.json +++ b/packages/gulp-bem-src/package.json @@ -25,6 +25,7 @@ "@bem/sdk.walk": "0.2.5", "bubble-stream-error": "1.0.0", "gulp-read": "0.0.1", + "pify": "^3.0.0", "vinyl": "2.1.0" }, "devDependencies": {