Skip to content

Commit 78c4531

Browse files
merlynomsftmerlynomsft
authored
Update build pipeline to support LocalPackage build config (parameterized; not enabled by default) (#20552)
1 parent de281d0 commit 78c4531

11 files changed

+134
-75
lines changed

Diff for: ‎BuildConfigGen/GitUtil.cs

-24
Original file line numberDiff line numberDiff line change
@@ -95,30 +95,6 @@ public static void GetUntrackedFiles(string taskTarget, out IEnumerable<string>
9595
toAdd = untrackedOuput2;
9696
toRemove = untrackedOuputRemove;
9797
}
98-
internal static string GetGitRootPath(string currentDir)
99-
{
100-
const string args = "rev-parse --git-dir";
101-
string path = RunGitCommandScalar(currentDir, args);
102-
103-
path = FixupPath(path);
104-
105-
const string gitDir = ".git";
106-
if (path.EndsWith(gitDir))
107-
{
108-
path = path.Substring(0, path.Length - gitDir.Length);
109-
110-
if (path == "")
111-
{
112-
return currentDir;
113-
}
114-
115-
return path;
116-
}
117-
else
118-
{
119-
throw new Exception($"expected git {args} to return ");
120-
}
121-
}
12298

12399
internal static IEnumerable<string> GetNonIgnoredFileListFromPath(string gitRoot, string taskTarget)
124100
{

Diff for: ‎BuildConfigGen/Program.cs

+62-12
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
109109
}
110110

111111
string currentDir = Environment.CurrentDirectory;
112-
string gitRootPath = GitUtil.GetGitRootPath(currentDir);
112+
string gitRootPath = GetTasksRootPath(currentDir);
113113

114114
string globalVersionPath = Path.Combine(gitRootPath, @"globalversion.txt");
115115
TaskVersion? globalVersion = GetGlobalVersion(gitRootPath, globalVersionPath);
@@ -208,7 +208,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
208208
Console.WriteLine($"Global version: maxPatchForCurrentSprint = maxPatchForCurrentSprint + 1");
209209
}
210210

211-
Console.WriteLine($"Global version update: globalVersion = {globalVersion} maxPatchForCurrentSprint={maxPatchForCurrentSprint}" );
211+
Console.WriteLine($"Global version update: globalVersion = {globalVersion} maxPatchForCurrentSprint={maxPatchForCurrentSprint}");
212212
}
213213
else
214214
{
@@ -249,6 +249,14 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
249249
}
250250
}
251251
}
252+
else
253+
{
254+
// if we're not updating local packages, we need to ensure the global version is updated to the task major version. existing patch number is preserved
255+
if (globalVersion is not null)
256+
{
257+
globalVersion = globalVersion.CloneWithMajor(taskMajorVersion);
258+
}
259+
}
252260

253261
if (globalVersion is not null)
254262
{
@@ -265,7 +273,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
265273
ensureUpdateModeVerifier!.WriteAllText(globalVersionPath, globalVersion!.MinorPatchToString(), false);
266274
}
267275

268-
ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError("(global)", skipContentCheck: false);
276+
ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(null, skipContentCheck: false);
269277

270278
foreach (var t in tasks)
271279
{
@@ -284,6 +292,37 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
284292
}
285293
}
286294

295+
private static string GetTasksRootPath(string inputCurrentDir)
296+
{
297+
string? currentDir = inputCurrentDir;
298+
string? tasksRootPath = null;
299+
300+
do
301+
{
302+
string currentDirGit = Path.Combine(currentDir, ".git");
303+
304+
if (Directory.Exists(currentDirGit))
305+
{
306+
tasksRootPath = currentDir;
307+
}
308+
309+
currentDir = (new DirectoryInfo(currentDir)).Parent?.FullName;
310+
311+
} while (currentDir != null);
312+
313+
if (tasksRootPath == null)
314+
{
315+
throw new Exception($"could not find .git in {currentDir}");
316+
}
317+
318+
if (!File.Exists(Path.Combine(tasksRootPath, "make-options.json")))
319+
{
320+
throw new Exception($"make-options.json not found in tasksRootPath={tasksRootPath}");
321+
}
322+
323+
return tasksRootPath;
324+
}
325+
287326
private static IEnumerable<string> FilterConfigsForTask(string? configs, KeyValuePair<string, MakeOptionsReader.AgentTask> t)
288327
{
289328
var configsList = t.Value.Configs.AsEnumerable();
@@ -343,7 +382,7 @@ private static void GetVersions(string task, string configsString, out List<(str
343382

344383
string currentDir = Environment.CurrentDirectory;
345384

346-
string gitRootPath = GitUtil.GetGitRootPath(currentDir);
385+
string gitRootPath = GetTasksRootPath(currentDir);
347386

348387
string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task);
349388
if (!Directory.Exists(taskTargetPath))
@@ -419,7 +458,7 @@ private static int GetCurrentSprint()
419458
return currentSprint;
420459
}
421460

422-
private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(string task, bool skipContentCheck)
461+
private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(string? task, bool skipContentCheck)
423462
{
424463
// if !writeUpdates, error if we have written any updates
425464
var verifyErrors = ensureUpdateModeVerifier!.GetVerifyErrors(skipContentCheck).ToList();
@@ -433,7 +472,14 @@ private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferEr
433472
Console.WriteLine(s);
434473
}
435474

436-
throw new Exception($"Updates needed, please run npm make.js --task {task} ");
475+
if (task is null)
476+
{
477+
throw new Exception($"Updates needed, please run node make.js");
478+
}
479+
else
480+
{
481+
throw new Exception($"Updates needed, please run node make.js --task {task} ");
482+
}
437483
}
438484
}
439485

@@ -459,8 +505,8 @@ private static void MainUpdateTask(
459505
try
460506
{
461507
string currentDir = Environment.CurrentDirectory;
462-
string gitRootPath = GitUtil.GetGitRootPath(currentDir);
463-
string versionMapFile = GetVersionMapFile(task, gitRootPath, generatedFolder);
508+
string gitRootPath = GetTasksRootPath(currentDir);
509+
string versionMapFile = GetVersionMapFile(task, generatedFolder);
464510

465511
string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task);
466512
if (!Directory.Exists(taskTargetPath))
@@ -489,7 +535,11 @@ private static void MainUpdateTask(
489535

490536
foreach (var config in targetConfigs)
491537
{
492-
if (config.useGlobalVersion && !hasGlobalVersion)
538+
if (config.useGlobalVersion && !includeLocalPackagesBuildConfig)
539+
{
540+
Console.WriteLine($"Info: MainUpdateTask: Skipping useGlobalVersion config for task b/c --include-local-packages-build-config. not specified. hasGlobalVersion={hasGlobalVersion} config.useGlobalVersion={config.useGlobalVersion} includeLocalPackagesBuildConfig={includeLocalPackagesBuildConfig}");
541+
}
542+
else if (config.useGlobalVersion && !hasGlobalVersion)
493543
{
494544
Console.WriteLine($"Info: MainUpdateTask: Skipping useGlobalVersion config for task b/c GlobalVersion not initialized. (to opt-in and start producing LocalBuildConfig, run with --include-local-packages-build-config. hasGlobalVersion={hasGlobalVersion} config.useGlobalVersion={config.useGlobalVersion}). Note: this is not an error!");
495545
}
@@ -608,7 +658,7 @@ private static void MainUpdateTask(
608658
}
609659
}
610660

611-
private static string GetVersionMapFile(string task, string gitRootPath, string generatedFolder)
661+
private static string GetVersionMapFile(string task, string generatedFolder)
612662
{
613663
return Path.Combine(generatedFolder, @$"{task}.versionmap.txt");
614664
}
@@ -1078,7 +1128,7 @@ private static void CopyConfig(string gitRootPath, string taskTargetPathOrUnders
10781128
private static void UpdateVersionsForTask(string task, TaskStateStruct taskState, HashSet<Config.ConfigRecord> targetConfigs, int currentSprint, string globalVersionPath, TaskVersion? globalVersion, string generatedFolder)
10791129
{
10801130
string currentDir = Environment.CurrentDirectory;
1081-
string gitRootPath = GitUtil.GetGitRootPath(currentDir);
1131+
string gitRootPath = GetTasksRootPath(currentDir);
10821132
string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task);
10831133

10841134
if (!Directory.Exists(taskTargetPath))
@@ -1093,7 +1143,7 @@ private static void UpdateVersionsForTask(string task, TaskStateStruct taskState
10931143

10941144
bool defaultVersionMatchesSourceVersion;
10951145

1096-
string versionMapFile = GetVersionMapFile(task, gitRootPath, generatedFolder);
1146+
string versionMapFile = GetVersionMapFile(task, generatedFolder);
10971147

10981148
{
10991149
TaskVersion? defaultVersion = null;

Diff for: ‎BuildConfigGen/dev.sh

-14
Original file line numberDiff line numberDiff line change
@@ -36,17 +36,6 @@ function detect_platform_and_runtime_id ()
3636
fi
3737
}
3838

39-
function cmd_build ()
40-
{
41-
heading "Building"
42-
dotnet build -o bin $SOLUTION_PATH || failed build
43-
#change execution flag to allow running with sudo
44-
if [[ ("$CURRENT_PLATFORM" == "linux") || ("$CURRENT_PLATFORM" == "darwin") ]]; then
45-
chmod +x "bin/BuildConfigGen"
46-
fi
47-
48-
}
49-
5039
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5140
pushd "$SCRIPT_DIR"
5241
source "$SCRIPT_DIR/Misc/helpers.sh"
@@ -91,6 +80,3 @@ echo "Adding .NET to PATH ${DOTNETSDK_INSTALLDIR}"
9180
export PATH=${DOTNETSDK_INSTALLDIR}:$PATH
9281
echo "Path = $PATH"
9382
echo ".NET Version = $(dotnet --version)"
94-
95-
SOLUTION_PATH="$SCRIPT_DIR/BuildConfigGen.sln"
96-
cmd_build

Diff for: ‎azure-pipelines.yml

+14-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ parameters:
2929
displayName: Enable CodeQL for run
3030
type: boolean
3131
default: false
32+
- name: includeLocalPackagesBuildConfig
33+
displayName: Flag to update LocalPackages buildconfig (for testing, this will be made default later)
34+
type: boolean
35+
default: false # note: keep in sync with ci\ci-test-tasks\canary-tests-v2.yml
3236

3337
variables:
3438
- name: currentDate
@@ -41,7 +45,16 @@ variables:
4145
value: ${{ eq(parameters.enableCodeQL, true) }}
4246
- name: system.debug
4347
value: true
44-
48+
- name: includeLocalPackagesBuildConfigParameter
49+
${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}:
50+
value: '--includeLocalPackagesBuildConfig'
51+
${{ else }}:
52+
value: ''
53+
- name: IncludeLocalPackagesBuildConfigTest
54+
${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}:
55+
value: '1'
56+
${{ else }}:
57+
value: ''
4558

4659
extends:
4760
template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates

Diff for: ‎ci/build-all-steps.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ steps:
106106
displayName: Clean tasks
107107

108108
# Build Tasks
109-
- script: node make.js serverBuild --task "$(getTaskPattern.task_pattern)"
109+
- script: node make.js serverBuild --task "$(getTaskPattern.task_pattern)" $(includeLocalPackagesBuildConfigParameter)
110110
displayName: Build Tasks
111111
condition: and(succeeded(), ne(variables['numTasks'], 0))
112112

Diff for: ‎ci/build-all-tasks.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,12 @@ steps:
3030
displayName: Clean tasks
3131

3232
# Build tasks
33-
- script: node make.js serverBuild --task "$(task_pattern)"
33+
- script: node make.js serverBuild --task "$(task_pattern)" $(includeLocalPackagesBuildConfigParameter)
3434
displayName: Build tasks using pattern
3535
condition: and(succeeded(), eq('${{ parameters.deploy_all_tasks }}', false))
3636

3737
# Build all tasks
38-
- script: node make.js serverBuild
38+
- script: node make.js serverBuild $(includeLocalPackagesBuildConfigParameter)
3939
displayName: Build all tasks
4040
condition: and(succeeded(), eq('${{ parameters.deploy_all_tasks }}', true))
4141

Diff for: ‎ci/build-single-steps.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ steps:
5050
displayName: Clean tasks
5151

5252
# Build Tasks
53-
- script: node make.js serverBuild --task "$(task_pattern)"
53+
- script: node make.js serverBuild --task "$(task_pattern)" $(includeLocalPackagesBuildConfigParameter)
5454
displayName: Build Tasks
5555

5656
# Check diff for task sources

Diff for: ‎ci/ci-test-tasks/build-task.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ steps:
2222
displayName: Clean tasks
2323

2424
# Build
25-
- script: node make.js serverBuild --task "${{ parameters.task }}"
25+
- script: node make.js serverBuild --task "${{ parameters.task }}" $(includeLocalPackagesBuildConfigParameter)
2626
displayName: ${{ parameters.task }} build
2727

2828
# Check diff for task sources

Diff for: ‎ci/ci-test-tasks/canary-tests-v2.yml

+18
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,21 @@
1+
parameters:
2+
- name: includeLocalPackagesBuildConfig
3+
displayName: Flag to update LocalPackages buildconfig (for testing, this will be made default later)
4+
type: boolean
5+
default: false # Note: keep in sync with /azure-pipelines.yml
6+
7+
variables:
8+
- name: includeLocalPackagesBuildConfigParameter
9+
${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}:
10+
value: '--includeLocalPackagesBuildConfig'
11+
${{ else }}:
12+
value: ''
13+
- name: IncludeLocalPackagesBuildConfigTest
14+
${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}:
15+
value: '1'
16+
${{ else }}:
17+
value: ''
18+
119
trigger:
220
- master
321
- releases/*

Diff for: ‎make-util.js

+6-10
Original file line numberDiff line numberDiff line change
@@ -1747,28 +1747,22 @@ exports.renameCodeCoverageOutput = renameCodeCoverageOutput;
17471747
//------------------------------------------------------------------------------
17481748

17491749
/**
1750-
* Returns path to BuldConfigGenerator, build it if needed. Fail on compilation failure
1750+
* Ensure Pre-reqs for buildConfigGen (e.g. dotnet)
17511751
* @param {String} baseConfigToolPath base build config tool path
1752-
* @returns {String} Path to the executed file
17531752
*/
1754-
var getBuildConfigGenerator = function (baseConfigToolPath) {
1755-
var programPath = "";
1753+
var ensureBuildConfigGeneratorPrereqs = function (baseConfigToolPath) {
17561754
var configToolBuildUtility = "";
17571755

17581756
if (os.platform() === 'win32') {
1759-
programPath = path.join(baseConfigToolPath, 'bin', 'BuildConfigGen.exe');
17601757
configToolBuildUtility = path.join(baseConfigToolPath, "dev.cmd");
17611758
} else {
1762-
programPath = path.join(baseConfigToolPath, 'bin', 'BuildConfigGen');
17631759
configToolBuildUtility = path.join(baseConfigToolPath, "dev.sh");
17641760
}
17651761

17661762
// build configToolBuildUtility if needed. (up-to-date check will skip build if not needed)
17671763
run(configToolBuildUtility, true);
1768-
1769-
return programPath;
17701764
};
1771-
exports.getBuildConfigGenerator = getBuildConfigGenerator;
1765+
exports.ensureBuildConfigGeneratorPrereqs = ensureBuildConfigGeneratorPrereqs;
17721766

17731767
/**
17741768
* Function to validate or write generated tasks
@@ -1785,7 +1779,9 @@ var processGeneratedTasks = function(baseConfigToolPath, taskList, makeOptions,
17851779
if (sprintNumber && !Number.isInteger(sprintNumber)) fail("Sprint is not a number");
17861780

17871781
var tasks = taskList.join('|')
1788-
const programPath = getBuildConfigGenerator(baseConfigToolPath);
1782+
ensureBuildConfigGeneratorPrereqs(baseConfigToolPath);
1783+
var programPath = `dotnet run --project "${baseConfigToolPath}/BuildConfigGen.csproj" -- `
1784+
17891785
const args = [
17901786
"--task",
17911787
`"${tasks}"`

0 commit comments

Comments
 (0)