From cb8d5bddcd24115276b2f012308cced661243820 Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Fri, 8 Feb 2019 14:18:46 -0800 Subject: [PATCH 01/13] Use precompiled assets in production mode --- .gitignore | 1 + server.js | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 37464bc9dce..f92c9998ade 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ css/**/*.css yarn.lock cypress/videos/* /public/styles/octicons/octicons.html +/precompiled/ diff --git a/server.js b/server.js index ee6ae65697d..4d1af5920a2 100644 --- a/server.js +++ b/server.js @@ -43,8 +43,14 @@ app.set('view engine', 'html') app.set('views', path.join(__dirname, '/views')) app.use(compression()) app.use(helmet()) -app.use(sass()) -app.use('/scripts/index.js', browserify('scripts/index.js')) +if (process.env.NODE_ENV === 'production') { + console.log('Production app detected; serving JS and CSS from disk') + app.use(express.static(path.join(__dirname, 'precompiled'), { redirect: false })) +} else { + console.log('Dev app detected; compiling JS and CSS in memory') + app.use(sass()) + app.use('/scripts/index.js', browserify('scripts/index.js')) +} app.get('/service-worker.js', (req, res) => res.sendFile(path.resolve(__dirname, 'scripts', 'service-worker.js'))) app.use(cookieParser()) app.use(requestLanguage({ From fe52290bade732e8f4bf07b5336250528c3bfb57 Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Fri, 8 Feb 2019 14:20:36 -0800 Subject: [PATCH 02/13] Make browserify options reusable --- middleware/browserify-opts.js | 25 +++++++++++++++++++++++++ middleware/browserify.js | 22 ++-------------------- 2 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 middleware/browserify-opts.js diff --git a/middleware/browserify-opts.js b/middleware/browserify-opts.js new file mode 100644 index 00000000000..fcaa09d3ecc --- /dev/null +++ b/middleware/browserify-opts.js @@ -0,0 +1,25 @@ +const nodeModulesToAvoidBabelifying = [ + 'lodash', + 'lunr', + 'prettydate' +] + +const excludeRegex = new RegExp(`/node_modules/(${nodeModulesToAvoidBabelifying.join('|')})`) + +module.exports = function doBrowserify (browserify) { + return function (entry) { + return browserify(entry, { + transform: [ + ['babelify', { + global: true, + exclude: excludeRegex, + presets: [ + ['@babel/preset-env', { targets: '> 0.25%, not dead' }] + ] + }], + 'brfs' + ] + }) + } +} + diff --git a/middleware/browserify.js b/middleware/browserify.js index 69904ba1b56..524ccc903db 100644 --- a/middleware/browserify.js +++ b/middleware/browserify.js @@ -1,26 +1,8 @@ const browserify = require('browserify-middleware') - -const nodeModulesToAvoidBabelifying = [ - 'lodash', - 'lunr', - 'prettydate' -] - -const excludeRegex = new RegExp(`/node_modules/(${nodeModulesToAvoidBabelifying.join('|')})`) +const browserifyOptions = require('./browserify-opts') function babelifyMiddleware (entry) { - return browserify(entry, { - transform: [ - ['babelify', { - global: true, - exclude: excludeRegex, - presets: [ - ['@babel/preset-env', { targets: '> 0.25%, not dead' }] - ] - }], - 'brfs' - ] - }) + return browserifyOptions(browserify)(entry) } module.exports = babelifyMiddleware From 58718155b8e9b9044bb2a2a8197a9bb4c2f8c614 Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Fri, 8 Feb 2019 14:20:48 -0800 Subject: [PATCH 03/13] Add precomilation task --- package.json | 1 + script/precompile-assets | 12 ++++++++++++ script/precompile-js.js | 10 ++++++++++ script/precompile-sass.js | 20 ++++++++++++++++++++ 4 files changed, 43 insertions(+) create mode 100755 script/precompile-assets create mode 100644 script/precompile-js.js create mode 100644 script/precompile-sass.js diff --git a/package.json b/package.json index 5ad43792f99..cb385331837 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "release": "node script/release", "dev": "cross-env NODE_PATH=. NODE_ENV=development nodemon server.js", "prepack": "check-for-leaks", + "precompile-assets": "script/precompile-assets", "linkschecker": "NODE_PATH=. NODE_ENV=test node scripts/links-checker.js", "cypress": "cypress run", "lint": "standard --fix", diff --git a/script/precompile-assets b/script/precompile-assets new file mode 100755 index 00000000000..3cc5d6bd7f1 --- /dev/null +++ b/script/precompile-assets @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -x # print commands before execution +set -o errexit # always exit on error +set -o pipefail # honor exit codes when piping +set -o nounset # fail on unset variables + +rm -r precompiled || true +mkdir -p precompiled/scripts +mkdir -p precompiled/styles +node ./script/precompile-js +node ./script/precompile-sass diff --git a/script/precompile-js.js b/script/precompile-js.js new file mode 100644 index 00000000000..9aabf44ba42 --- /dev/null +++ b/script/precompile-js.js @@ -0,0 +1,10 @@ +const path = require('path') +const fs = require('fs') +const browserify = require('browserify') +const browserifyOptions = require('../middleware/browserify-opts') + +const entry = path.join(__dirname, '../scripts/index.js') +const destination = path.join(__dirname, '../precompiled/scripts/index.js') + +const b = browserifyOptions(browserify)(entry) +b.bundle().pipe(fs.createWriteStream(destination)) diff --git a/script/precompile-sass.js b/script/precompile-sass.js new file mode 100644 index 00000000000..6554848cbc7 --- /dev/null +++ b/script/precompile-sass.js @@ -0,0 +1,20 @@ +const path = require('path') +const fs = require('fs') +const sass = require('node-sass') + +const source = path.join(__dirname, '../public/styles/index.scss') +const destination = path.join(__dirname, '../precompiled/styles/index.css') + +sass.render({ + file: source, + includePaths: [ + path.join(__dirname, '../node_modules') + ] +}, function onSassCompiled (err, result) { + if (err) { + console.error(err) + process.exit(1) + } + + fs.writeFileSync(destination, result.css) +}) From 1bd8f607aeae2b0a75ef134d084f6d79b99ed02a Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Fri, 8 Feb 2019 14:21:00 -0800 Subject: [PATCH 04/13] Precompile assets as part of app deployment --- Procfile | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Procfile diff --git a/Procfile b/Procfile new file mode 100644 index 00000000000..f67bf72b1b0 --- /dev/null +++ b/Procfile @@ -0,0 +1,2 @@ +web: npm start +release: npm run precompile-assets From 50f39d81fc3488afcde5d054939b1ef0c9dd2de6 Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Fri, 8 Feb 2019 14:26:13 -0800 Subject: [PATCH 05/13] Compile assets as build instead of release phase --- Procfile | 2 -- package.json | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 Procfile diff --git a/Procfile b/Procfile deleted file mode 100644 index f67bf72b1b0..00000000000 --- a/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -web: npm start -release: npm run precompile-assets diff --git a/package.json b/package.json index cb385331837..a57c44a9808 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "linkschecker": "NODE_PATH=. NODE_ENV=test node scripts/links-checker.js", "cypress": "cypress run", "lint": "standard --fix", - "generate-octicons": "node ./script/generate-octicons.js" + "generate-octicons": "node ./script/generate-octicons.js", + "heroku-postbuild": "npm run precompile-assets" }, "husky": { "hooks": { From 4a4f487a4b00a412d5493d8c72c49dd0ecb0a854 Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Fri, 8 Feb 2019 14:43:33 -0800 Subject: [PATCH 06/13] :shirt: fix lint --- script/precompile-sass.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/precompile-sass.js b/script/precompile-sass.js index 6554848cbc7..9352ad45078 100644 --- a/script/precompile-sass.js +++ b/script/precompile-sass.js @@ -2,7 +2,7 @@ const path = require('path') const fs = require('fs') const sass = require('node-sass') -const source = path.join(__dirname, '../public/styles/index.scss') +const source = path.join(__dirname, '../public/styles/index.scss') const destination = path.join(__dirname, '../precompiled/styles/index.css') sass.render({ From ae78fd97bc9d9bf2979cd178c0115593678094b3 Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Fri, 8 Feb 2019 14:43:47 -0800 Subject: [PATCH 07/13] Minify precompiled JS --- package.json | 2 +- script/precompile-assets | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index a57c44a9808..80d570674de 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "release": "node script/release", "dev": "cross-env NODE_PATH=. NODE_ENV=development nodemon server.js", "prepack": "check-for-leaks", - "precompile-assets": "script/precompile-assets", + "precompile-assets": "cross-env NODE_ENV=test script/precompile-assets", "linkschecker": "NODE_PATH=. NODE_ENV=test node scripts/links-checker.js", "cypress": "cypress run", "lint": "standard --fix", diff --git a/script/precompile-assets b/script/precompile-assets index 3cc5d6bd7f1..640a0208202 100755 --- a/script/precompile-assets +++ b/script/precompile-assets @@ -9,4 +9,6 @@ rm -r precompiled || true mkdir -p precompiled/scripts mkdir -p precompiled/styles node ./script/precompile-js +./node_modules/.bin/uglifyjs --compress --mangle -o precompiled/scripts/index.min.js -- precompiled/scripts/index.js +mv precompiled/scripts/index.min.js precompiled/scripts/index.js node ./script/precompile-sass From d0b2d957602dc1d3e76b1177532932f50ba5a1cf Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Fri, 8 Feb 2019 14:46:14 -0800 Subject: [PATCH 08/13] Precompile assets before integration tests --- script/integration.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/script/integration.sh b/script/integration.sh index 158c68e5150..7e3c62a2feb 100755 --- a/script/integration.sh +++ b/script/integration.sh @@ -1,3 +1,4 @@ +npm run precompile-assets npm start & wait-on http://localhost:5000 cypress run --record --key a0cba5c6-0650-4abe-8d41-d990bb7a0a66 From 733493b261833ef9ae8ef79d9a5ed5a9c6b9663d Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Fri, 8 Feb 2019 14:53:37 -0800 Subject: [PATCH 09/13] Simplify browserify middleware --- middleware/browserify.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/middleware/browserify.js b/middleware/browserify.js index 524ccc903db..f2b04f476b6 100644 --- a/middleware/browserify.js +++ b/middleware/browserify.js @@ -1,8 +1,4 @@ const browserify = require('browserify-middleware') const browserifyOptions = require('./browserify-opts') -function babelifyMiddleware (entry) { - return browserifyOptions(browserify)(entry) -} - -module.exports = babelifyMiddleware +module.exports = browserifyOptions(browserify) From 15719da1dd5454bd781baa8eb2192f78d2e4dd6a Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Fri, 8 Feb 2019 15:45:01 -0800 Subject: [PATCH 10/13] Provide the correct NODE_ENV to asset precompilation --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 80d570674de..101629dbf4b 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "release": "node script/release", "dev": "cross-env NODE_PATH=. NODE_ENV=development nodemon server.js", "prepack": "check-for-leaks", - "precompile-assets": "cross-env NODE_ENV=test script/precompile-assets", + "precompile-assets": "cross-env NODE_ENV=production script/precompile-assets", "linkschecker": "NODE_PATH=. NODE_ENV=test node scripts/links-checker.js", "cypress": "cypress run", "lint": "standard --fix", From 07378e45ebc9c8020320c7ebcc17e43579772c26 Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Thu, 14 Feb 2019 10:51:45 -0800 Subject: [PATCH 11/13] Convert precompilation script to JS --- package-lock.json | 7 +-- package.json | 3 +- script/precompile-assets.js | 90 +++++++++++++++++++++++++++++++++++++ script/precompile-js.js | 10 ----- script/precompile-sass.js | 20 --------- 5 files changed, 94 insertions(+), 36 deletions(-) create mode 100644 script/precompile-assets.js delete mode 100644 script/precompile-js.js delete mode 100644 script/precompile-sass.js diff --git a/package-lock.json b/package-lock.json index cc63b65425b..625d6e392f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13026,7 +13026,6 @@ "version": "3.4.9", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.4.9.tgz", "integrity": "sha512-8CJsbKOtEbnJsTyv6LE6m6ZKniqMiFWmm9sRbopbkGs3gMPPfd3Fh8iIA4Ykv5MgaTbqHr4BaoGLJLZNhsrW1Q==", - "optional": true, "requires": { "commander": "~2.17.1", "source-map": "~0.6.1" @@ -13035,14 +13034,12 @@ "commander": { "version": "2.17.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz", - "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==", - "optional": true + "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==" }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "optional": true + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" } } }, diff --git a/package.json b/package.json index 89da0512eb7..ba345a4d530 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "release": "node script/release", "dev": "cross-env NODE_PATH=. NODE_ENV=development nodemon server.js", "prepack": "check-for-leaks", - "precompile-assets": "cross-env NODE_ENV=production script/precompile-assets", + "precompile-assets": "cross-env NODE_ENV=production node script/precompile-assets.js", "linkschecker": "NODE_PATH=. NODE_ENV=test node scripts/links-checker.js", "cypress": "cypress run", "lint": "standard --fix", @@ -101,6 +101,7 @@ "standard": "^12.0.1", "supertest": "^3.4.2", "supertest-session": "^3.3.0", + "uglify-js": "^3.4.9", "wait-on": "^3.1.0", "walk-sync": "^1.1.3" }, diff --git a/script/precompile-assets.js b/script/precompile-assets.js new file mode 100644 index 00000000000..ea719a10f5e --- /dev/null +++ b/script/precompile-assets.js @@ -0,0 +1,90 @@ +const path = require('path') +const stream = require('stream') +const fs = require('fs-extra') +const browserify = require('browserify') +const browserifyOptions = require('../middleware/browserify-opts') +const sass = require('node-sass') +const uglify = require('uglify-js') + +function dir (...parts) { + return path.join(__dirname, '..', ...parts) +} + +function uglifyStream () { + const buffers = [] + return new stream.Transform({ + transform (chunk, _encoding, callback) { + buffers.push(chunk) + callback() + }, + + flush (callback) { + const code = Buffer.concat(buffers).toString() + const compiled = uglify.minify(code) + this.push(compiled.code) + callback() + } + }) +} + +const PATHS = { + precompiled: dir('precompiled'), + scripts: dir('precompiled', 'scripts'), + styles: dir('precompiled', 'styles'), + nodeModules: dir('node_modules'), + + jsEntry: dir('scripts', 'index.js'), + jsDestination: dir('precompiled', 'scripts', 'index.js'), + + cssEntry: dir('public', 'styles', 'index.scss'), + cssDestination: dir('precompiled', 'styles', 'index.css') +} + +async function precompileAssets () { + try { + console.log('Creating directories...') + await fs.remove(PATHS.precompiled) + await fs.ensureDir(PATHS.scripts) + await fs.ensureDir(PATHS.styles) + console.log('Precompiling JS...') + await precompileJavaScript() + console.log('Precompiling CSS...') + await precompileCss() + } catch (err) { + console.error(err) + process.exit(1) + } +} + +function precompileJavaScript () { + return new Promise((resolve, reject) => { + const b = browserifyOptions(browserify)(PATHS.jsEntry) + const pipe = b.bundle() + .pipe(uglifyStream()) + .pipe(fs.createWriteStream(PATHS.jsDestination)) + pipe.on('error', reject) + pipe.on('finish', resolve) + }) +} + +function precompileCss () { + return new Promise((resolve, reject) => { + sass.render({ + file: PATHS.cssEntry, + includePaths: [ + PATHS.nodeModules + ] + }, async function onSassCompiled (err, result) { + if (err) { + return reject(err) + } + + await fs.writeFile(PATHS.cssDestination, result.css) + resolve() + }) + }) +} + +if (require.main === module) { + precompileAssets() +} diff --git a/script/precompile-js.js b/script/precompile-js.js deleted file mode 100644 index 9aabf44ba42..00000000000 --- a/script/precompile-js.js +++ /dev/null @@ -1,10 +0,0 @@ -const path = require('path') -const fs = require('fs') -const browserify = require('browserify') -const browserifyOptions = require('../middleware/browserify-opts') - -const entry = path.join(__dirname, '../scripts/index.js') -const destination = path.join(__dirname, '../precompiled/scripts/index.js') - -const b = browserifyOptions(browserify)(entry) -b.bundle().pipe(fs.createWriteStream(destination)) diff --git a/script/precompile-sass.js b/script/precompile-sass.js deleted file mode 100644 index 9352ad45078..00000000000 --- a/script/precompile-sass.js +++ /dev/null @@ -1,20 +0,0 @@ -const path = require('path') -const fs = require('fs') -const sass = require('node-sass') - -const source = path.join(__dirname, '../public/styles/index.scss') -const destination = path.join(__dirname, '../precompiled/styles/index.css') - -sass.render({ - file: source, - includePaths: [ - path.join(__dirname, '../node_modules') - ] -}, function onSassCompiled (err, result) { - if (err) { - console.error(err) - process.exit(1) - } - - fs.writeFileSync(destination, result.css) -}) From 5be9eea64463fa442255a437488f089c8617c7b6 Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Thu, 14 Feb 2019 10:55:05 -0800 Subject: [PATCH 12/13] Use specific lodash imports Reduces bundle size by ~15% --- lib/i18n.js | 3 ++- scripts/apply-active-class-to-active-links.js | 2 +- scripts/create-filter-list.js | 2 +- scripts/lazy-load-images.js | 2 +- test/localization.js | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/i18n.js b/lib/i18n.js index 5e0f061f00d..486db75a886 100644 --- a/lib/i18n.js +++ b/lib/i18n.js @@ -2,7 +2,8 @@ require('require-yaml') const i18n = require('electron-i18n') const flat = require('flat') -const { get, set } = require('lodash') +const get = require('lodash/get') +const set = require('lodash/set') const locales = Object.keys(i18n.locales) const websiteStrings = require('../data/locale.yml') const websiteKeys = Object.keys(flat(websiteStrings)) diff --git a/scripts/apply-active-class-to-active-links.js b/scripts/apply-active-class-to-active-links.js index 72bc40a09f0..ee3c54e97a1 100644 --- a/scripts/apply-active-class-to-active-links.js +++ b/scripts/apply-active-class-to-active-links.js @@ -1,4 +1,4 @@ -const { escapeRegExp } = require('lodash') +const escapeRegExp = require('lodash/escapeRegExp') module.exports = function applyActiveClassToActiveLinks () { const topPath = `/${location.pathname.split('/')[1]}` diff --git a/scripts/create-filter-list.js b/scripts/create-filter-list.js index 21a7889e555..19c4389d1b9 100644 --- a/scripts/create-filter-list.js +++ b/scripts/create-filter-list.js @@ -1,4 +1,4 @@ -const { debounce } = require('lodash') +const debounce = require('lodash/debounce') const lunr = require('lunr') const queryString = require('query-string') const setQueryString = require('set-query-string') diff --git a/scripts/lazy-load-images.js b/scripts/lazy-load-images.js index 9b2515d20ee..d23d0dd1cde 100644 --- a/scripts/lazy-load-images.js +++ b/scripts/lazy-load-images.js @@ -1,4 +1,4 @@ -const { throttle } = require('lodash') +const throttle = require('lodash/throttle') function inViewport (element) { const { top, right, bottom, left } = element.getBoundingClientRect() diff --git a/test/localization.js b/test/localization.js index 95a066d4f92..bb0924e89ac 100644 --- a/test/localization.js +++ b/test/localization.js @@ -8,7 +8,7 @@ const fs = require('fs') const path = require('path') const walk = require('walk-sync') const flat = require('flat') -const getProp = require('lodash').get +const getProp = require('lodash/get') const locale = require(path.join(__dirname, '../data/locale.yml')) const views = walk.entries(path.join(__dirname, '../views')) From eb7a76616055cece564d7bdfefba3a8db6d650bf Mon Sep 17 00:00:00 2001 From: Michelle Tilley Date: Thu, 14 Feb 2019 11:01:18 -0800 Subject: [PATCH 13/13] Remove old precompile assets script --- script/precompile-assets | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100755 script/precompile-assets diff --git a/script/precompile-assets b/script/precompile-assets deleted file mode 100755 index 640a0208202..00000000000 --- a/script/precompile-assets +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set -x # print commands before execution -set -o errexit # always exit on error -set -o pipefail # honor exit codes when piping -set -o nounset # fail on unset variables - -rm -r precompiled || true -mkdir -p precompiled/scripts -mkdir -p precompiled/styles -node ./script/precompile-js -./node_modules/.bin/uglifyjs --compress --mangle -o precompiled/scripts/index.min.js -- precompiled/scripts/index.js -mv precompiled/scripts/index.min.js precompiled/scripts/index.js -node ./script/precompile-sass