1$ErrorActionPreference = 'Stop'
2
3. $PSScriptRoot/find-all-stress-packages.ps1
4$FailedCommands = New-Object Collections.Generic.List[hashtable]
5
6. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1)
7
8# Powershell does not (at time of writing) treat exit codes from external binaries
9# as cause for stopping execution, so do this via a wrapper function.
10# See https://github.com/PowerShell/PowerShell-RFC/pull/277
11function Run()
12{
13 Write-Host "`n==> $args`n" -ForegroundColor Green
14 $command, $arguments = $args
15 & $command $arguments
16 if ($LASTEXITCODE) {
17 Write-Error "Command '$args' failed with code: $LASTEXITCODE" -ErrorAction 'Continue'
18 $FailedCommands.Add(@{ command = "$args"; code = $LASTEXITCODE })
19 }
20}
21
22function RunOrExitOnFailure()
23{
24 run @args
25 if ($LASTEXITCODE) {
26 exit $LASTEXITCODE
27 }
28}
29
30function Login([string]$subscription, [string]$clusterGroup, [switch]$pushImages)
31{
32 Write-Host "Logging in to subscription, cluster and container registry"
33 az account show *> $null
34 if ($LASTEXITCODE) {
35 RunOrExitOnFailure az login --allow-no-subscriptions
36 }
37
38 # Discover cluster name, only one cluster per group is expected
39 Write-Host "Listing AKS cluster in $subscription/$clusterGroup"
40 $cluster = RunOrExitOnFailure az aks list -g $clusterGroup --subscription $subscription -o json
41 $clusterName = ($cluster | ConvertFrom-Json).name
42
43 $kubeContext = (RunOrExitOnFailure kubectl config view -o json) | ConvertFrom-Json
44 $defaultNamespace = $kubeContext.contexts.Where({ $_.name -eq $clusterName }).context.namespace
45
46 RunOrExitOnFailure az aks get-credentials `
47 -n "$clusterName" `
48 -g "$clusterGroup" `
49 --subscription "$subscription" `
50 --overwrite-existing
51
52 if ($defaultNamespace) {
53 RunOrExitOnFailure kubectl config set-context $clusterName --namespace $defaultNamespace
54 }
55
56 if ($pushImages) {
57 $registry = RunOrExitOnFailure az acr list -g $clusterGroup --subscription $subscription -o json
58 $registryName = ($registry | ConvertFrom-Json).name
59 RunOrExitOnFailure az acr login -n $registryName
60 }
61}
62
63function DeployStressTests(
64 [string]$searchDirectory = '.',
65 [hashtable]$filters = @{},
66 [string]$environment = 'test',
67 [string]$repository = '',
68 [switch]$pushImages,
69 [string]$clusterGroup = '',
70 [string]$deployId = 'local',
71 [switch]$login,
72 [string]$subscription = '',
73 [switch]$CI,
74 [string]$Namespace
75) {
76 if ($environment -eq 'test') {
77 if ($clusterGroup -or $subscription) {
78 Write-Warning "Overriding cluster group and subscription with defaults for 'test' environment."
79 }
80 $clusterGroup = 'rg-stress-cluster-test'
81 $subscription = 'Azure SDK Developer Playground'
82 } elseif ($environment -eq 'prod') {
83 if ($clusterGroup -or $subscription) {
84 Write-Warning "Overriding cluster group and subscription with defaults for 'prod' environment."
85 }
86 $clusterGroup = 'rg-stress-cluster-prod'
87 $subscription = 'Azure SDK Test Resources'
88 }
89
90 if ($login) {
91 if (!$clusterGroup -or !$subscription) {
92 throw "clusterGroup and subscription parameters must be specified when logging into an environment that is not test or prod."
93 }
94 Login -subscription $subscription -clusterGroup $clusterGroup -pushImages:$pushImages
95 }
96
97 RunOrExitOnFailure helm repo add stress-test-charts https://stresstestcharts.blob.core.windows.net/helm/
98 Run helm repo update
99 if ($LASTEXITCODE) { return $LASTEXITCODE }
100
101 $pkgs = FindStressPackages -directory $searchDirectory -filters $filters -CI:$CI -namespaceOverride $Namespace
102 Write-Host "" "Found $($pkgs.Length) stress test packages:"
103 Write-Host $pkgs.Directory ""
104 foreach ($pkg in $pkgs) {
105 Write-Host "Deploying stress test at '$($pkg.Directory)'"
106 DeployStressPackage `
107 -pkg $pkg `
108 -deployId $deployId `
109 -environment $environment `
110 -repositoryBase $repository `
111 -pushImages:$pushImages `
112 -login:$login
113 }
114
115 Write-Host "Releases deployed by $deployId"
116 Run helm list --all-namespaces -l deployId=$deployId
117
118 if ($FailedCommands) {
119 Write-Warning "The following commands failed:"
120 foreach ($cmd in $FailedCommands) {
121 Write-Error "'$($cmd.command)' failed with code $($cmd.code)" -ErrorAction 'Continue'
122 }
123 exit 1
124 }
125
126 Write-Host "`nStress test telemetry links (dashboard, fileshare, etc.): https://aka.ms/azsdk/stress/dashboard"
127}
128
129function DeployStressPackage(
130 [object]$pkg,
131 [string]$deployId,
132 [string]$environment,
133 [string]$repositoryBase,
134 [switch]$pushImages,
135 [switch]$login
136) {
137 $registry = RunOrExitOnFailure az acr list -g $clusterGroup --subscription $subscription -o json
138 $registryName = ($registry | ConvertFrom-Json).name
139
140 Run helm dependency update $pkg.Directory
141 if ($LASTEXITCODE) { return }
142
143 if (Test-Path "$($pkg.Directory)/stress-test-resources.bicep") {
144 Run az bicep build -f "$($pkg.Directory)/stress-test-resources.bicep"
145 if ($LASTEXITCODE) { return }
146 }
147
148 $imageTag = "${registryName}.azurecr.io"
149 if ($repositoryBase) {
150 $imageTag += "/$repositoryBase"
151 }
152 $imageTag += "/$($pkg.Namespace)/$($pkg.ReleaseName):${deployId}"
153
154 $dockerFilePath = if ($pkg.Dockerfile) {
155 Join-Path $pkg.Directory $pkg.Dockerfile
156 } else {
157 "$($pkg.Directory)/Dockerfile"
158 }
159 $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath)
160
161 if ($pushImages -and (Test-Path $dockerFilePath)) {
162 Write-Host "Building and pushing stress test docker image '$imageTag'"
163 $dockerFile = Get-ChildItem $dockerFilePath
164 $dockerBuildFolder = if ($pkg.DockerBuildDir) {
165 Join-Path $pkg.Directory $pkg.DockerBuildDir
166 } else {
167 $dockerFile.DirectoryName
168 }
169 $dockerBuildFolder = [System.IO.Path]::GetFullPath($dockerBuildFolder).Trim()
170
171 Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder
172 if ($LASTEXITCODE) { return }
173
174 Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue
175 Write-Host " docker run -it $imageTag" -ForegroundColor DarkBlue
176 Write-Host " docker run -it $imageTag <shell, e.g. 'bash' 'pwsh' 'sh'>" -ForegroundColor DarkBlue
177 Write-Host "To show installed container images:" -ForegroundColor Blue
178 Write-Host " docker image ls" -ForegroundColor DarkBlue
179 Write-Host "To show running containers:" -ForegroundColor Blue
180 Write-Host " docker ps" -ForegroundColor DarkBlue
181
182 Run docker push $imageTag
183 if ($LASTEXITCODE) {
184 if ($login) {
185 Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'"
186 }
187 return
188 }
189 }
190
191 Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..."
192 kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f -
193 if ($LASTEXITCODE) {exit $LASTEXITCODE}
194
195 Write-Host "Installing or upgrading stress test $($pkg.ReleaseName) from $($pkg.Directory)"
196 Run helm upgrade $pkg.ReleaseName $pkg.Directory `
197 -n $pkg.Namespace `
198 --install `
199 --set image=$imageTag `
200 --set stress-test-addons.env=$environment
201 if ($LASTEXITCODE) {
202 # Issues like 'UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress'
203 # can be the result of cancelled `upgrade` operations (e.g. ctrl-c).
204 # See https://github.com/helm/helm/issues/4558
205 Write-Warning "The issue may be fixable by first running 'helm rollback -n $($pkg.Namespace) $($pkg.ReleaseName)'"
206 return
207 }
208
209 # Helm 3 stores release information in kubernetes secrets. The only way to add extra labels around
210 # specific releases (thereby enabling filtering on `helm list`) is to label the underlying secret resources.
211 # There is not currently support for setting these labels via the helm cli.
212 $helmReleaseConfig = kubectl get secrets `
213 -n $pkg.Namespace `
214 -l status=deployed,name=$($pkg.ReleaseName) `
215 -o jsonpath='{.items[0].metadata.name}'
216
217 Run kubectl label secret -n $pkg.Namespace --overwrite $helmReleaseConfig deployId=$deployId
218}
219
220function CheckDependencies()
221{
222 $deps = @(
223 @{
224 Command = "docker";
225 Help = "Docker must be installed: https://docs.docker.com/get-docker/";
226 }
227 @{
228 Command = "kubectl";
229 Help = "kubectl must be installed: https://kubernetes.io/docs/tasks/tools/#kubectl";
230 },
231 @{
232 Command = "helm";
233 Help = "helm must be installed: https://helm.sh/docs/intro/install/";
234 },
235 @{
236 Command = "az";
237 Help = "Azure CLI must be installed: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli";
238 }
239 )
240
241 Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
242
243 $shouldError = $false
244 foreach ($dep in $deps) {
245 if (!(Get-Command $dep.Command -ErrorAction SilentlyContinue)) {
246 $shouldError = $true
247 Write-Error $dep.Help
248 }
249 }
250
251 if ($shouldError) {
252 exit 1
253 }
254
255}
View as plain text