1Set-StrictMode -Version "4.0"
2
3class MatrixConfig {
4 [PSCustomObject]$displayNames
5 [Hashtable]$displayNamesLookup
6 [PSCustomObject]$matrix
7 [MatrixParameter[]]$matrixParameters
8 [Array]$include
9 [Array]$exclude
10}
11
12class MatrixParameter {
13 MatrixParameter([String]$name, [System.Object]$value) {
14 $this.Value = $value
15 $this.Name = $name
16 }
17
18 [System.Object]$Value
19 [System.Object]$Name
20
21 Set($value, [String]$keyRegex = '')
22 {
23 if ($this.Value -is [PSCustomObject]) {
24 $set = $false
25 foreach ($prop in $this.Value.PSObject.Properties) {
26 if ($prop.Name -match $keyRegex) {
27 $prop.Value = $value
28 $set = $true
29 break
30 }
31 }
32 if (!$set) {
33 throw "Property `"$keyRegex`" does not exist for MatrixParameter."
34 }
35 } else {
36 $this.Value = $value
37 }
38 }
39
40 [System.Object]Flatten()
41 {
42 if ($this.Value -is [PSCustomObject]) {
43 return $this.Value.PSObject.Properties | ForEach-Object {
44 [MatrixParameter]::new($_.Name, $_.Value)
45 }
46 } elseif ($this.Value -is [Array]) {
47 return $this.Value | ForEach-Object {
48 [MatrixParameter]::new($this.Name, $_)
49 }
50 } else {
51 return $this
52 }
53 }
54
55 [Int]Length()
56 {
57 if ($this.Value -is [PSCustomObject]) {
58 return ($this.Value.PSObject.Properties | Measure-Object).Count
59 } elseif ($this.Value -is [Array]) {
60 return $this.Value.Length
61 } else {
62 return 1
63 }
64 }
65
66 [String]CreateDisplayName([Hashtable]$displayNamesLookup)
67 {
68 if ($null -eq $this.Value) {
69 $displayName = ""
70 } elseif ($this.Value -is [PSCustomObject]) {
71 $displayName = $this.Name
72 } else {
73 $displayName = $this.Value.ToString()
74 }
75
76 if ($displayNamesLookup.ContainsKey($displayName)) {
77 $displayName = $displayNamesLookup[$displayName]
78 }
79
80 # Matrix naming restrictions:
81 # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration
82 $displayName = $displayName -replace "[^A-Za-z0-9_]", ""
83 return $displayName
84 }
85}
86
87$IMPORT_KEYWORD = '$IMPORT'
88
89function GenerateMatrix(
90 [MatrixConfig]$config,
91 [String]$selectFromMatrixType,
92 [String]$displayNameFilter = ".*",
93 [Array]$filters = @(),
94 [Array]$replace = @(),
95 [Array]$nonSparseParameters = @()
96) {
97 $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = `
98 ProcessImport $config.matrixParameters $selectFromMatrixType $nonSparseParameters $config.displayNamesLookup
99 if ($selectFromMatrixType -eq "sparse") {
100 $matrix = GenerateSparseMatrix $matrixParameters $config.displayNamesLookup $nonSparseParameters
101 } elseif ($selectFromMatrixType -eq "all") {
102 $matrix = GenerateFullMatrix $matrixParameters $config.displayNamesLookup
103 } else {
104 throw "Matrix generator not implemented for selectFromMatrixType: $($platform.selectFromMatrixType)"
105 }
106
107 # Combine with imported after matrix generation, since a sparse selection should result in a full combination of the
108 # top level and imported sparse matrices (as opposed to a sparse selection of both matrices).
109 if ($importedMatrix) {
110 $matrix = CombineMatrices $matrix $importedMatrix $combinedDisplayNameLookup
111 }
112 if ($config.exclude) {
113 $matrix = ProcessExcludes $matrix $config.exclude
114 }
115 if ($config.include) {
116 $matrix = ProcessIncludes $config $matrix $selectFromMatrixType
117 }
118
119 $matrix = FilterMatrix $matrix $filters
120 $matrix = ProcessReplace $matrix $replace $config.displayNamesLookup
121 $matrix = FilterMatrixDisplayName $matrix $displayNameFilter
122 return $matrix
123}
124
125function ProcessNonSparseParameters(
126 [MatrixParameter[]]$parameters,
127 [Array]$nonSparseParameters
128) {
129 if (!$nonSparseParameters) {
130 return $parameters, $null
131 }
132
133 $sparse = [MatrixParameter[]]@()
134 $nonSparse = [MatrixParameter[]]@()
135
136 foreach ($param in $parameters) {
137 if ($param.Name -in $nonSparseParameters) {
138 $nonSparse += $param
139 } else {
140 $sparse += $param
141 }
142 }
143
144 return $sparse, $nonSparse
145}
146
147function FilterMatrixDisplayName([array]$matrix, [string]$filter) {
148 return $matrix | Where-Object { $_ } | ForEach-Object {
149 if ($_.Name -match $filter) {
150 return $_
151 }
152 }
153}
154
155# Filters take the format of key=valueregex,key2=valueregex2
156function FilterMatrix([array]$matrix, [array]$filters) {
157 $matrix = $matrix | ForEach-Object {
158 if (MatchesFilters $_ $filters) {
159 return $_
160 }
161 }
162 return $matrix
163}
164
165function MatchesFilters([hashtable]$entry, [array]$filters) {
166 foreach ($filter in $filters) {
167 $key, $regex = ParseFilter $filter
168 # Default all regex checks to go against empty string when keys are missing.
169 # This simplifies the filter syntax/interface to be regex only.
170 $value = ""
171 if ($null -ne $entry -and $entry.parameters.Contains($key)) {
172 $value = $entry.parameters[$key]
173 }
174 if ($value -notmatch $regex) {
175 return $false
176 }
177 }
178
179 return $true
180}
181
182function ParseFilter([string]$filter) {
183 # Lazy match key in case value contains '='
184 if ($filter -match "(.+?)=(.+)") {
185 $key = $matches[1]
186 $regex = $matches[2]
187 return $key, $regex
188 } else {
189 throw "Invalid filter: `"${filter}`", expected <key>=<regex> format"
190 }
191}
192
193# Importing the JSON as PSCustomObject preserves key ordering,
194# whereas ConvertFrom-Json -AsHashtable does not
195function GetMatrixConfigFromJson([String]$jsonConfig)
196{
197 [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json
198 $config.matrixParameters = @()
199 $config.displayNamesLookup = @{}
200 $include = [MatrixParameter[]]@()
201 $exclude = [MatrixParameter[]]@()
202
203 if ($null -ne $config.displayNames) {
204 $config.displayNames.PSObject.Properties | ForEach-Object {
205 $config.displayNamesLookup.Add($_.Name, $_.Value)
206 }
207 }
208 if ($null -ne $config.matrix) {
209 $config.matrixParameters = PsObjectToMatrixParameterArray $config.matrix
210 }
211 foreach ($includeMatrix in $config.include) {
212 $include += ,@(PsObjectToMatrixParameterArray $includeMatrix)
213 }
214 foreach ($excludeMatrix in $config.exclude) {
215 $exclude += ,@(PsObjectToMatrixParameterArray $excludeMatrix)
216 }
217
218 $config.include = $include
219 $config.exclude = $exclude
220
221 return $config
222}
223
224function PsObjectToMatrixParameterArray([PSCustomObject]$obj)
225{
226 if ($obj -eq $null) {
227 return $null
228 }
229 return $obj.PSObject.Properties | ForEach-Object {
230 [MatrixParameter]::new($_.Name, $_.Value)
231 }
232}
233
234function ProcessExcludes([Array]$matrix, [Array]$excludes)
235{
236 $deleteKey = "%DELETE%"
237 $exclusionMatrix = @()
238
239 foreach ($exclusion in $excludes) {
240 $full = GenerateFullMatrix $exclusion
241 $exclusionMatrix += $full
242 }
243
244 foreach ($element in $matrix) {
245 foreach ($exclusion in $exclusionMatrix) {
246 $match = MatrixElementMatch $element.parameters $exclusion.parameters
247 if ($match) {
248 $element.parameters[$deleteKey] = $true
249 }
250 }
251 }
252
253 return $matrix | Where-Object { !$_.parameters.Contains($deleteKey) }
254}
255
256function ProcessIncludes([MatrixConfig]$config, [Array]$matrix)
257{
258 $inclusionMatrix = @()
259 foreach ($inclusion in $config.include) {
260 $full = GenerateFullMatrix $inclusion $config.displayNamesLookup
261 $inclusionMatrix += $full
262 }
263
264 return $matrix + $inclusionMatrix
265}
266
267function ParseReplacement([String]$replacement) {
268 $parsed = '', '', ''
269 $idx = 0
270 $escaped = $false
271 $operators = '=', '/'
272 $err = "Invalid replacement syntax, expecting <key>=<value>/<replace>"
273
274 foreach ($c in $replacement -split '') {
275 if ($idx -ge $parsed.Length) {
276 throw $err
277 }
278 if (!$escaped -and $c -in $operators) {
279 $idx++
280 } else {
281 $parsed[$idx] += $c
282 }
283 $escaped = $c -eq '\'
284 }
285
286 if ($idx -lt $parsed.Length - 1) {
287 throw $err
288 }
289
290 $replace = $parsed[2] -replace "\\([$($operators -join '')])", '$1'
291
292 return @{
293 "key" = '^' + $parsed[0] + '$'
294 # Force full matches only.
295 "value" = '^' + $parsed[1] + '$'
296 "replace" = $replace
297 }
298}
299
300function ProcessReplace
301{
302 param(
303 [Array]$matrix,
304 [Array]$replacements,
305 [Hashtable]$displayNamesLookup
306 )
307
308 if (!$replacements) {
309 return $matrix
310 }
311
312 $replaceMatrix = @()
313
314 foreach ($element in $matrix) {
315 $replacement = [MatrixParameter[]]@()
316
317 foreach ($perm in $element._permutation) {
318 $replace = $perm
319
320 # Iterate nested permutations or run once for singular values (int, string, bool)
321 foreach ($flattened in $perm.Flatten()) {
322 foreach ($query in $replacements) {
323 $parsed = ParseReplacement $query
324 if ($flattened.Name -match $parsed.key -and $flattened.Value -match $parsed.value) {
325 # In most cases, this will just swap one value for another, however -replace
326 # is used here in order to support replace values which may use regex capture groups
327 # e.g. 'foo-1' -replace '(foo)-1', '$1-replaced'
328 $replaceValue = $flattened.Value -replace $parsed.value, $parsed.replace
329 $perm.Set($replaceValue, $parsed.key)
330 break
331 }
332 }
333 }
334
335 $replacement += $perm
336 }
337
338 $replaceMatrix += CreateMatrixCombinationScalar $replacement $displayNamesLookup
339 }
340
341 return $replaceMatrix
342}
343
344function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$nonSparseParameters, [Hashtable]$displayNamesLookup)
345{
346 $importPath = ""
347 $matrix = $matrix | ForEach-Object {
348 if ($_.Name -ne $IMPORT_KEYWORD) {
349 return $_
350 } else {
351 $importPath = $_.Value
352 }
353 }
354 if ((!$matrix -and !$importPath) -or !$importPath) {
355 return $matrix, @()
356 }
357
358 if (!(Test-Path $importPath)) {
359 Write-Error "`$IMPORT path '$importPath' does not exist."
360 exit 1
361 }
362 $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath)
363 $importedMatrix = GenerateMatrix `
364 -config $importedMatrixConfig `
365 -selectFromMatrixType $selection `
366 -nonSparseParameters $nonSparseParameters
367
368 $combinedDisplayNameLookup = $importedMatrixConfig.displayNamesLookup
369 foreach ($lookup in $displayNamesLookup.GetEnumerator()) {
370 $combinedDisplayNameLookup[$lookup.Name] = $lookup.Value
371 }
372
373 return $matrix, $importedMatrix, $importedMatrixConfig.displayNamesLookup
374}
375
376function CombineMatrices([Array]$matrix1, [Array]$matrix2, [Hashtable]$displayNamesLookup = @{})
377{
378 $combined = @()
379 if (!$matrix1) {
380 return $matrix2
381 }
382 if (!$matrix2) {
383 return $matrix1
384 }
385
386 foreach ($entry1 in $matrix1) {
387 foreach ($entry2 in $matrix2) {
388 $combined += CreateMatrixCombinationScalar ($entry1._permutation + $entry2._permutation) $displayNamesLookup
389 }
390 }
391
392 return $combined
393}
394
395function MatrixElementMatch([System.Collections.Specialized.OrderedDictionary]$source, [System.Collections.Specialized.OrderedDictionary]$target)
396{
397 if ($target.Count -eq 0) {
398 return $false
399 }
400
401 foreach ($key in $target.Keys) {
402 if (!$source.Contains($key) -or $source[$key] -ne $target[$key]) {
403 return $false
404 }
405 }
406
407 return $true
408}
409
410function CloneOrderedDictionary([System.Collections.Specialized.OrderedDictionary]$dictionary) {
411 $newDictionary = [Ordered]@{}
412 foreach ($element in $dictionary.GetEnumerator()) {
413 $newDictionary[$element.Name] = $element.Value
414 }
415 return $newDictionary
416}
417
418function SerializePipelineMatrix([Array]$matrix)
419{
420 $pipelineMatrix = [Ordered]@{}
421 foreach ($entry in $matrix) {
422 if ($pipelineMatrix.Contains($entry.Name)) {
423 Write-Warning "Found duplicate configurations for job `"$($entry.name)`". Multiple values may have been replaced with the same value."
424 continue
425 }
426 $pipelineMatrix.Add($entry.name, [Ordered]@{})
427 foreach ($key in $entry.parameters.Keys) {
428 $pipelineMatrix[$entry.name].Add($key, $entry.parameters[$key])
429 }
430 }
431
432 return @{
433 compressed = $pipelineMatrix | ConvertTo-Json -Compress ;
434 pretty = $pipelineMatrix | ConvertTo-Json;
435 }
436}
437
438function GenerateSparseMatrix(
439 [MatrixParameter[]]$parameters,
440 [Hashtable]$displayNamesLookup,
441 [Array]$nonSparseParameters = @()
442) {
443 $parameters, $nonSparse = ProcessNonSparseParameters $parameters $nonSparseParameters
444 $dimensions = GetMatrixDimensions $parameters
445 $matrix = GenerateFullMatrix $parameters $displayNamesLookup
446
447 $sparseMatrix = @()
448 [array]$indexes = GetSparseMatrixIndexes $dimensions
449 foreach ($idx in $indexes) {
450 $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions
451 }
452
453 if ($nonSparse) {
454 $allOfMatrix = GenerateFullMatrix $nonSparse $displayNamesLookup
455 return CombineMatrices $allOfMatrix $sparseMatrix $displayNamesLookup
456 }
457
458 return $sparseMatrix
459}
460
461function GetSparseMatrixIndexes([Array]$dimensions)
462{
463 $size = ($dimensions | Measure-Object -Maximum).Maximum
464 $indexes = @()
465
466 # With full matrix, retrieve items by doing diagonal lookups across the matrix N times.
467 # For example, given a matrix with dimensions 3, 2, 2:
468 # 0, 0, 0
469 # 1, 1, 1
470 # 2, 2, 2
471 # 3, 0, 0 <- 3, 3, 3 wraps to 3, 0, 0 given the dimensions
472 for ($i = 0; $i -lt $size; $i++) {
473 $idx = @()
474 for ($j = 0; $j -lt $dimensions.Length; $j++) {
475 $idx += $i % $dimensions[$j]
476 }
477 $indexes += ,$idx
478 }
479
480 return ,$indexes
481}
482
483function GenerateFullMatrix(
484 [MatrixParameter[]] $parameters,
485 [Hashtable]$displayNamesLookup = @{}
486) {
487 # Handle when the config does not have a matrix specified (e.g. only the include field is specified)
488 if (!$parameters) {
489 return @()
490 }
491
492 $matrix = [System.Collections.ArrayList]::new()
493 InitializeMatrix $parameters $displayNamesLookup $matrix
494
495 return $matrix
496}
497
498function CreateMatrixCombinationScalar([MatrixParameter[]]$permutation, [Hashtable]$displayNamesLookup = @{})
499{
500 $names = @()
501 $flattenedParameters = [Ordered]@{}
502
503 foreach ($entry in $permutation) {
504 $nameSegment = ""
505
506 # Unwind nested permutations or run once for singular values (int, string, bool)
507 foreach ($param in $entry.Flatten()) {
508 if ($flattenedParameters.Contains($param.Name)) {
509 throw "Found duplicate parameter `"$($param.Name)`" when creating matrix combination."
510 }
511 $flattenedParameters.Add($param.Name, $param.Value)
512 }
513
514 $nameSegment = $entry.CreateDisplayName($displayNamesLookup)
515 if ($nameSegment) {
516 $names += $nameSegment
517 }
518 }
519
520 # The maximum allowed matrix name length is 100 characters
521 $name = $names -join "_"
522 if ($name -and $name[0] -match "^[0-9]") {
523 $name = "job_" + $name # Azure Pipelines only supports job names starting with letters
524 }
525 if ($name.Length -gt 100) {
526 $name = $name[0..99] -join ""
527 }
528
529 return @{
530 name = $name
531 parameters = $flattenedParameters
532 # Keep the original permutation around in case we need to re-process this entry when transforming the matrix
533 _permutation = $permutation
534 }
535}
536
537function InitializeMatrix
538{
539 param(
540 [MatrixParameter[]]$parameters,
541 [Hashtable]$displayNamesLookup,
542 [System.Collections.ArrayList]$permutations,
543 $permutation = [MatrixParameter[]]@()
544 )
545 $head, $tail = $parameters
546
547 if (!$head) {
548 $entry = CreateMatrixCombinationScalar $permutation $displayNamesLookup
549 $permutations.Add($entry) | Out-Null
550 return
551 }
552
553 # This behavior implicitly treats non-array values as single elements
554 foreach ($param in $head.Flatten()) {
555 $newPermutation = $permutation + $param
556 InitializeMatrix $tail $displayNamesLookup $permutations $newPermutation
557 }
558}
559
560function GetMatrixDimensions([MatrixParameter[]]$parameters)
561{
562 $dimensions = @()
563 foreach ($param in $parameters) {
564 $dimensions += $param.Length()
565 }
566
567 return $dimensions
568}
569
570function SetNdMatrixElement
571{
572 param(
573 $element,
574 [ValidateNotNullOrEmpty()]
575 [Array]$idx,
576 [ValidateNotNullOrEmpty()]
577 [Array]$matrix,
578 [ValidateNotNullOrEmpty()]
579 [Array]$dimensions
580 )
581
582 if ($idx.Length -ne $dimensions.Length) {
583 throw "Matrix index query $($idx.Length) must be the same length as its dimensions $($dimensions.Length)"
584 }
585
586 $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions
587 $matrix[$arrayIndex] = $element
588}
589
590function GetNdMatrixArrayIndex
591{
592 param(
593 [ValidateNotNullOrEmpty()]
594 [Array]$idx,
595 [ValidateNotNullOrEmpty()]
596 [Array]$dimensions
597 )
598
599 if ($idx.Length -ne $dimensions.Length) {
600 throw "Matrix index query length ($($idx.Length)) must be the same as dimension length ($($dimensions.Length))"
601 }
602
603 $stride = 1
604 # Commented out does lookup with wrap handling
605 # $index = $idx[$idx.Length-1] % $dimensions[$idx.Length-1]
606 $index = $idx[$idx.Length-1]
607
608 for ($i = $dimensions.Length-1; $i -ge 1; $i--) {
609 $stride *= $dimensions[$i]
610 # Commented out does lookup with wrap handling
611 # $index += ($idx[$i-1] % $dimensions[$i-1]) * $stride
612 $index += $idx[$i-1] * $stride
613 }
614
615 return $index
616}
617
618function GetNdMatrixElement
619{
620 param(
621 [ValidateNotNullOrEmpty()]
622 [Array]$idx,
623 [ValidateNotNullOrEmpty()]
624 [Array]$matrix,
625 [ValidateNotNullOrEmpty()]
626 [Array]$dimensions
627 )
628
629 $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions
630 return $matrix[$arrayIndex]
631}
632
633function GetNdMatrixIndex
634{
635 param(
636 [int]$index,
637 [ValidateNotNullOrEmpty()]
638 [Array]$dimensions
639 )
640
641 $matrixIndex = @()
642 $stride = 1
643
644 for ($i = $dimensions.Length-1; $i -ge 1; $i--) {
645 $stride *= $dimensions[$i]
646 $page = [math]::floor($index / $stride) % $dimensions[$i-1]
647 $matrixIndex = ,$page + $matrixIndex
648 }
649 $col = $index % $dimensions[$dimensions.Length-1]
650 $matrixIndex += $col
651
652 return $matrixIndex
653}
654
655# # # # # # # # # # # # # # # # # # # # # # # # # # # #
656# The below functions are non-dynamic examples that #
657# help explain the above N-dimensional algorithm #
658# # # # # # # # # # # # # # # # # # # # # # # # # # # #
659function Get4dMatrixElement([Array]$idx, [Array]$matrix, [Array]$dimensions)
660{
661 $stride1 = $idx[0] * $dimensions[1] * $dimensions[2] * $dimensions[3]
662 $stride2 = $idx[1] * $dimensions[2] * $dimensions[3]
663 $stride3 = $idx[2] * $dimensions[3]
664 $stride4 = $idx[3]
665
666 return $matrix[$stride1 + $stride2 + $stride3 + $stride4]
667}
668
669function Get4dMatrixIndex([int]$index, [Array]$dimensions)
670{
671 $stride1 = $dimensions[3]
672 $stride2 = $dimensions[2]
673 $stride3 = $dimensions[1]
674 $page1 = [math]::floor($index / $stride1) % $dimensions[2]
675 $page2 = [math]::floor($index / ($stride1 * $stride2)) % $dimensions[1]
676 $page3 = [math]::floor($index / ($stride1 * $stride2 * $stride3)) % $dimensions[0]
677 $remainder = $index % $dimensions[3]
678
679 return @($page3, $page2, $page1, $remainder)
680}
681
View as plain text