Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@microsoft/rush",
"comment": "Support percentage weight in operationSettings in rush-project.json file.",
"type": "none"
}
],
"packageName": "@microsoft/rush"
}
2 changes: 1 addition & 1 deletion common/config/rush/version-policies.json
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@
"policyName": "rush",
"definitionName": "lockStepVersion",
"version": "5.169.3",
"nextBump": "patch",
"nextBump": "minor",
"mainProject": "@microsoft/rush"
}
]
2 changes: 1 addition & 1 deletion common/reviews/api/rush-lib.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -676,7 +676,7 @@ export interface IOperationSettings {
outputFolderNames?: string[];
parameterNamesToIgnore?: string[];
sharding?: IRushPhaseSharding;
weight?: number;
weight?: number | `${number}%`;
}

// @internal (undocumented)
Expand Down
2 changes: 1 addition & 1 deletion libraries/rush-lib/src/api/RushProjectConfiguration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ export interface IOperationSettings {
* How many concurrency units this operation should take up during execution. The maximum concurrent units is
* determined by the -p flag.
*/
weight?: number;
weight?: number | `${number}%`;

/**
* If true, this operation can use cobuilds for orchestration without restoring build cache entries.
Expand Down
72 changes: 50 additions & 22 deletions libraries/rush-lib/src/cli/parsing/ParseParallelism.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,39 +5,67 @@ import * as os from 'node:os';

import { IS_WINDOWS } from '../../utilities/executionUtilities';

export function getNumberOfCores(): number {
return os.availableParallelism?.() ?? os.cpus().length;
}

/**
* Since the JSON value is a string, it must be a percentage like "50%",
* which we convert to a number based on the available parallelism.
* For example, if the available parallelism (not the -p flag) is 8 and the weight is "50%",
* then the resulting weight will be 4.
*
* @param weight
* @returns
*/
export function parseParallelismPercent(weight: string, numberOfCores: number = getNumberOfCores()): number {
const percentageRegExp: RegExp = /^\d+(\.\d+)?%$/;

if (!percentageRegExp.test(weight)) {
throw new Error(`Expecting a percentage string like "12%" or "34.56%".`);
}

const percentValue: number = parseFloat(weight.slice(0, -1));

if (percentValue <= 0) {
throw new Error(`Invalid percentage value of "${percentValue}": value must be greater than zero`);
}

if (percentValue > 100) {
throw new Error(`Invalid percentage value of "${percentValue}": value must not exceed 100%`);
}

// Use as much CPU as possible, so we round down the weight here
return Math.max(1, Math.floor((percentValue / 100) * numberOfCores));
}

/**
* Parses a command line specification for desired parallelism.
* Factored out to enable unit tests
*/
export function parseParallelism(
rawParallelism: string | undefined,
numberOfCores: number = os.availableParallelism?.() ?? os.cpus().length
numberOfCores: number = getNumberOfCores()
): number {
if (rawParallelism) {
rawParallelism = rawParallelism.trim();

if (rawParallelism === 'max') {
return numberOfCores;
} else {
const parallelismAsNumber: number = Number(rawParallelism);

if (typeof rawParallelism === 'string' && rawParallelism.trim().endsWith('%')) {
const parsedPercentage: number = Number(rawParallelism.trim().replace(/\%$/, ''));

if (parsedPercentage <= 0 || parsedPercentage > 100) {
throw new Error(
`Invalid percentage value of '${rawParallelism}', value cannot be less than '0%' or more than '100%'`
);
}

const workers: number = Math.floor((parsedPercentage / 100) * numberOfCores);
return Math.max(workers, 1);
} else if (!isNaN(parallelismAsNumber)) {
return Math.max(parallelismAsNumber, 1);
} else {
throw new Error(
`Invalid parallelism value of '${rawParallelism}', expected a number, a percentage, or 'max'`
);
}
}

if (rawParallelism.endsWith('%')) {
return parseParallelismPercent(rawParallelism, numberOfCores);
}

const parallelismAsNumber: number = Number(rawParallelism);
if (!isNaN(parallelismAsNumber)) {
return Math.max(parallelismAsNumber, 1);
}

throw new Error(
`Invalid parallelism value of "${rawParallelism}": expected a number, a percentage string, or "max"`
);
} else {
// If an explicit parallelism number wasn't provided, then choose a sensible
// default.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`parseParallelism throwsErrorOnInvalidParallelism 1`] = `"Invalid parallelism value of 'tequila', expected a number, a percentage, or 'max'"`;
exports[`parseParallelism throwsErrorOnInvalidParallelism 1`] = `"Invalid parallelism value of \\"tequila\\": expected a number, a percentage string, or \\"max\\""`;

exports[`parseParallelism throwsErrorOnInvalidParallelismPercentage 1`] = `"Invalid percentage value of '200%', value cannot be less than '0%' or more than '100%'"`;
exports[`parseParallelism throwsErrorOnInvalidParallelismPercentage 1`] = `"Invalid percentage value of \\"200\\": value must not exceed 100%"`;
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import type {
import type { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration';
import type { IOperationExecutionResult } from './IOperationExecutionResult';
import type { OperationExecutionRecord } from './OperationExecutionRecord';
import { parseParallelismPercent } from '../../cli/parsing/ParseParallelism';

const PLUGIN_NAME: 'WeightedOperationPlugin' = 'WeightedOperationPlugin';

Expand Down Expand Up @@ -41,8 +42,18 @@ function weightOperations(
const projectConfiguration: RushProjectConfiguration | undefined = projectConfigurations.get(project);
const operationSettings: IOperationSettings | undefined =
operation.settings ?? projectConfiguration?.operationSettingsByOperationName.get(phase.name);
Comment on lines 42 to 44

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I find myself wondering why this layer is here; operation.settings should always already be defined if projectConfigurations.get(project)?.operationSettingsByOperationName.get(phase.name) returns a truthy value.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems out of scope

if (operationSettings?.weight) {
operation.weight = operationSettings.weight;
if (operationSettings?.weight !== undefined) {
if (typeof operationSettings.weight === 'number') {
operation.weight = operationSettings.weight;
} else if (typeof operationSettings.weight === 'string') {
try {
operation.weight = parseParallelismPercent(operationSettings.weight);
} catch (error) {
throw new Error(
`${operation.name} (invalid weight: ${JSON.stringify(operationSettings.weight)}) ${(error as Error).message}`
);
}
}
}
}
Async.validateWeightedIterable(operation);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
// See LICENSE in the project root for license information.

import os from 'node:os';

import type { IPhase } from '../../../api/CommandLineConfiguration';
import type { RushConfigurationProject } from '../../../api/RushConfigurationProject';
import type { IOperationSettings, RushProjectConfiguration } from '../../../api/RushProjectConfiguration';
import {
type IExecuteOperationsContext,
PhasedCommandHooks
} from '../../../pluginFramework/PhasedCommandHooks';
import type { IOperationExecutionResult } from '../IOperationExecutionResult';
import { Operation } from '../Operation';
import { WeightedOperationPlugin } from '../WeightedOperationPlugin';
import { MockOperationRunner } from './MockOperationRunner';

const MOCK_PHASE: IPhase = {
name: '_phase:test',
allowWarningsOnSuccess: false,
associatedParameters: new Set(),
dependencies: {
self: new Set(),
upstream: new Set()
},
isSynthetic: false,
logFilenameIdentifier: '_phase_test',
missingScriptBehavior: 'silent'
};

function createProject(packageName: string): RushConfigurationProject {
return {
packageName
} as RushConfigurationProject;
}

function createOperation(options: {
project: RushConfigurationProject;
settings?: IOperationSettings;
isNoOp?: boolean;
}): Operation {
const { project, settings, isNoOp } = options;
return new Operation({
phase: MOCK_PHASE,
project,
settings,
runner: new MockOperationRunner(`${project.packageName} (${MOCK_PHASE.name})`, undefined, false, isNoOp),
logFilenameIdentifier: `${project.packageName}_phase_test`
});
}

function createExecutionRecords(operation: Operation): Map<Operation, IOperationExecutionResult> {
return new Map([
[
operation,
{
operation,
runner: operation.runner
} as unknown as IOperationExecutionResult
]
]);
}

function createContext(
projectConfigurations: ReadonlyMap<RushConfigurationProject, RushProjectConfiguration>,
parallelism: number = os.availableParallelism()
): IExecuteOperationsContext {
return {
projectConfigurations,
parallelism
} as IExecuteOperationsContext;
}

async function applyWeightPluginAsync(
operations: Map<Operation, IOperationExecutionResult>,
context: IExecuteOperationsContext
): Promise<void> {
const hooks: PhasedCommandHooks = new PhasedCommandHooks();
new WeightedOperationPlugin().apply(hooks);
await hooks.beforeExecuteOperations.promise(operations, context);
}

function mockAvailableParallelism(value: number): jest.SpyInstance<number, []> {
return jest.spyOn(os, 'availableParallelism').mockReturnValue(value);
}

describe(WeightedOperationPlugin.name, () => {
afterEach(() => {
jest.restoreAllMocks();
});

it('applies numeric weight from operation settings', async () => {
const project: RushConfigurationProject = createProject('project-number');
const operation: Operation = createOperation({
project,
settings: {
operationName: MOCK_PHASE.name,
weight: 7
}
});

await applyWeightPluginAsync(
createExecutionRecords(operation),
createContext(new Map(), /* Set parallelism to ensure -p does not affect weight calculation */ 1)
);

expect(operation.weight).toBe(7);
});

it('converts percentage weight using available parallelism', async () => {
mockAvailableParallelism(10);

const project: RushConfigurationProject = createProject('project-percent');
const operation: Operation = createOperation({
project,
settings: {
operationName: MOCK_PHASE.name,
weight: '25%'
} as IOperationSettings
});

await applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map()));

expect(operation.weight).toBe(2);
});

it('reads weight from rush-project configuration when operation settings are undefined', async () => {
mockAvailableParallelism(8);

const project: RushConfigurationProject = createProject('project-config');
const operation: Operation = createOperation({ project });
const projectConfiguration: RushProjectConfiguration = {
operationSettingsByOperationName: new Map([
[
MOCK_PHASE.name,
{
operationName: MOCK_PHASE.name,
weight: '50%'
} as IOperationSettings
]
])
} as unknown as RushProjectConfiguration;

await applyWeightPluginAsync(
createExecutionRecords(operation),
createContext(new Map([[project, projectConfiguration]]))
);

expect(operation.weight).toBe(4);
});

it('use floor when converting percentage weight to avoid zero weight', async () => {
mockAvailableParallelism(16);

const project: RushConfigurationProject = createProject('project-floor');
const operation: Operation = createOperation({
project,
settings: {
operationName: MOCK_PHASE.name,
weight: '33.3333%'
} as IOperationSettings
});

await applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map()));

expect(operation.weight).toBe(5);
});

it('forces NO-OP operation weight to zero ignore weight settings', async () => {
const project: RushConfigurationProject = createProject('project-no-op');
const operation: Operation = createOperation({
project,
isNoOp: true,
settings: {
operationName: MOCK_PHASE.name,
weight: 100
}
});

await applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map()));

expect(operation.weight).toBe(0);
});

it('throws for invalid percentage weight format', async () => {
mockAvailableParallelism(16);

const project: RushConfigurationProject = createProject('project-invalid');
const operation: Operation = createOperation({
project,
// @ts-expect-error Testing invalid input
settings: {
operationName: MOCK_PHASE.name,
weight: '12.5a%'
} as IOperationSettings
});

await expect(
applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map()))
).rejects.toThrow(/invalid weight: "12.5a%"/i);
});
});
15 changes: 12 additions & 3 deletions libraries/rush-lib/src/schemas/rush-project.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,18 @@
]
},
"weight": {
"description": "The number of concurrency units that this operation should take up. The maximum concurrency units is determined by the -p flag.",
"type": "integer",
"minimum": 0
"oneOf": [
{
"type": "string",
"pattern": "^[1-9][0-9]*(\\.\\d+)?%$",
"description": "The number of concurrency units that this operation should take up, as a percent of `os.availableParallelism()`. At runtime this value will be clamped to the range `[1, rushParallelism]`, where `rushParallelism` is the requested parallelism to the current Rush command. To have this operation consume no concurrency, use the number 0 instead of a string."
},
{
"description": "The number of concurrency units that this operation should take up. At runtime this value will be clamped to the range `[0, rushParallelism]`, where `rushParallelism` is the requested parallelism to the current Rush command.",
"type": "integer",
"minimum": 0
}
]
},
"allowCobuildWithoutCache": {
"type": "boolean",
Expand Down