Skip to content

Commit 3ab49b4

Browse files
author
Florian Eiden
authored
Merge pull request #123 from Azure/bcdr-tests
Add 2 tests
2 parents c5bca66 + 1e72c0b commit 3ab49b4

File tree

6 files changed

+480
-0
lines changed

6 files changed

+480
-0
lines changed

‎BCDRTests/Test 01 - 1 input - 2 jobs - 1 output/01-provision.ps1‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
$location = "Canada Central"
2+
$runId = Get-Date -Format "yyyyMMddHHmmss"
3+
$tempFolder = ".\"
4+
5+
######################################################################
6+
# Resource Group
7+
$rgName = "rg-asa-bcdr-$runId"
8+
New-AzResourceGroup -Name $rgName -Location $location
9+
10+
######################################################################
11+
# Event hub
12+
$ehNamespace = "eh-asa-bcdr-$runId"
13+
$ehNamespaceSku = "Standard"
14+
$ehNamespaceCapacity = 1
15+
$ehName = "eh-input"
16+
$ehPartitionCount = 2
17+
$ehAuthorizationRuleName = "eh-asa-bcdr-$runId"
18+
$ehConsumerGroupName1 = "cg1"
19+
$ehConsumerGroupName2 = "cg2"
20+
21+
New-AzEventHubNamespace -ResourceGroupName $rgName -NamespaceName $ehNamespace -Location $location -SkuName $ehNamespaceSku -SkuCapacity $ehNamespaceCapacity
22+
New-AzEventHubAuthorizationRule -ResourceGroupName $rgName -NamespaceName $ehNamespace -AuthorizationRuleName $ehAuthorizationRuleName -Rights @("Listen","Send")
23+
$ehKey = Get-AzEventHubKey -ResourceGroupName $rgName -Namespace $ehNamespace -AuthorizationRuleName $ehAuthorizationRuleName
24+
25+
New-AzEventHub -ResourceGroupName $rgName -NamespaceName $ehNamespace -Name $ehName -PartitionCount $ehPartitionCount
26+
New-AzEventHubConsumerGroup -ResourceGroupName $rgName -Namespace $ehNamespace -EventHub $ehName -Name $ehConsumerGroupName1
27+
New-AzEventHubConsumerGroup -ResourceGroupName $rgName -Namespace $ehNamespace -EventHub $ehName -Name $ehConsumerGroupName2
28+
29+
######################################################################
30+
# Cosmos DB
31+
$cosmosDBAccountName = "cosmos-asa-bcdr-$runId"
32+
$cosmosDBDatabaseName = "cosmosDB-asa-bcdr-$runId"
33+
$cosmosDBContainerName = "container"
34+
$cosmosDBPartitionKey = "/deviceid"
35+
$cosmosDBPartitionKeyKind = "Hash"
36+
$cosmosDBUniqueKey = "/deviceid_minute"
37+
38+
New-AzCosmosDBAccount -ResourceGroupName $rgName -Name $cosmosDBAccountName -Location $location
39+
$cosmosDBAccountKey = Get-AzCosmosDBAccountKey -ResourceGroupName $rgName -Name $cosmosDBAccountName
40+
41+
42+
New-AzCosmosDBSqlDatabase -ResourceGroupName $rgName -AccountName $cosmosDBAccountName -Name $cosmosDBDatabaseName
43+
$cosmosDBUniqueKeyPolicy = New-AzCosmosDBSqlUniqueKeyPolicy -UniqueKey (New-AzCosmosDBSqlUniqueKey -Path $cosmosDBUniqueKey)
44+
New-AzCosmosDBSqlContainer -ResourceGroupName $rgName -AccountName $cosmosDBAccountName -DatabaseName $cosmosDBDatabaseName -Name $cosmosDBContainerName -PartitionKeyPath $cosmosDBPartitionKey -PartitionKeyKind $cosmosDBPartitionKeyKind -UniqueKeyPolicy $cosmosDBUniqueKeyPolicy
45+
46+
######################################################################
47+
# Stream Analytics Jobs
48+
$asaJobName1 = "asa-asa-bcdr-1-$runId"
49+
$asaJobName2 = "asa-asa-bcdr-2-$runId"
50+
$asaJobSKU = "Standard"
51+
$asaInputName = "ehInput"
52+
$asaOutputName = "cosmosOutput"
53+
$asaJobStreamingUnit = 3
54+
$asaJobQuery1 = "SELECT CAST(DeviceId AS NVARCHAR(MAX)) AS deviceid, COUNT(*) AS count15, SYSTEM.TIMESTAMP() AS windowend, CONCAT(CAST(DeviceId AS NVARCHAR(MAX)),'_',SUBSTRING(SYSTEM.TIMESTAMP(),1,16)) AS deviceid_minute, '$asaJobName1' AS jobname INTO $asaOutputName FROM $asaInputName GROUP BY DeviceId, HOPPING(second,60,10)"
55+
$asaJobQuery2 = "SELECT CAST(DeviceId AS NVARCHAR(MAX)) AS deviceid, COUNT(*) AS count15, SYSTEM.TIMESTAMP() AS windowend, CONCAT(CAST(DeviceId AS NVARCHAR(MAX)),'_',SUBSTRING(SYSTEM.TIMESTAMP(),1,16)) AS deviceid_minute, '$asaJobName2' AS jobname INTO $asaOutputName FROM $asaInputName GROUP BY DeviceId, HOPPING(second,60,10)"
56+
57+
58+
New-AzStreamAnalyticsJob -ResourceGroupName $rgName -Name $asaJobName1 -Location $location -SkuName $asaJobSKU
59+
New-AzStreamAnalyticsJob -ResourceGroupName $rgName -Name $asaJobName2 -Location $location -SkuName $asaJobSKU
60+
61+
$asaInputProperties1 = @{
62+
properties = @{
63+
type = "Stream"
64+
serialization = @{
65+
type = "Json"
66+
properties = @{
67+
encoding = "UTF8"
68+
}
69+
}
70+
compression = @{
71+
type = "None"
72+
}
73+
datasource = @{
74+
type = "Microsoft.EventHub/EventHub"
75+
properties = @{
76+
serviceBusNamespace = $ehNamespace
77+
sharedAccessPolicyName = $ehAuthorizationRuleName
78+
sharedAccessPolicyKey = $ehKey.PrimaryKey
79+
authenticationMode = "ConnectionString"
80+
eventHubName = $ehName
81+
consumerGroupName = $ehConsumerGroupName1
82+
}
83+
}
84+
}
85+
}
86+
$asaInputProperties1 | ConvertTo-JSON -depth 4 | Out-File -Path "$($tempFolder)tempInput1.json"
87+
New-AzStreamAnalyticsInput -ResourceGroupName $rgName -JobName $asaJobName1 -Name $asaInputName -File "$($tempFolder)tempInput1.json"
88+
Remove-Item -Path "$($tempFolder)tempInput1.json"
89+
90+
$asaInputProperties2 = @{
91+
properties = @{
92+
type = "Stream"
93+
serialization = @{
94+
type = "Json"
95+
properties = @{
96+
encoding = "UTF8"
97+
}
98+
}
99+
compression = @{
100+
type = "None"
101+
}
102+
datasource = @{
103+
type = "Microsoft.EventHub/EventHub"
104+
properties = @{
105+
serviceBusNamespace = $ehNamespace
106+
sharedAccessPolicyName = $ehAuthorizationRuleName
107+
sharedAccessPolicyKey = $ehKey.PrimaryKey
108+
authenticationMode = "ConnectionString"
109+
eventHubName = $ehName
110+
consumerGroupName = $ehConsumerGroupName2
111+
}
112+
}
113+
}
114+
}
115+
$asaInputProperties2 | ConvertTo-JSON -depth 4 | Out-File -Path "$($tempFolder)tempInput2.json"
116+
New-AzStreamAnalyticsInput -ResourceGroupName $rgName -JobName $asaJobName2 -Name $asaInputName -File "$($tempFolder)tempInput2.json"
117+
Remove-Item -Path "$($tempFolder)tempInput2.json"
118+
119+
120+
$asaOutputProperties = @{
121+
properties = @{
122+
datasource = @{
123+
type = "Microsoft.Storage/DocumentDB"
124+
properties = @{
125+
accountId = $cosmosDBAccountName
126+
accountKey = $cosmosDBAccountKey.PrimaryMasterKey
127+
collectionNamePattern = $cosmosDBContainerName
128+
database = $cosmosDBDatabaseName
129+
documentId = $cosmosDBUniqueKey.replace("/","")
130+
partitionKey = $cosmosDBPartitionKey.replace("/","")
131+
}
132+
}
133+
}
134+
}
135+
136+
$asaOutputProperties | ConvertTo-JSON -depth 4 | Out-File -Path "$($tempFolder)tempOutput.json"
137+
138+
New-AzStreamAnalyticsOutput -ResourceGroupName $rgName -JobName $asaJobName1 -Name $asaOutputName -File "$($tempFolder)tempOutput.json"
139+
New-AzStreamAnalyticsOutput -ResourceGroupName $rgName -JobName $asaJobName2 -Name $asaOutputName -File "$($tempFolder)tempOutput.json"
140+
Remove-Item -Path "$($tempFolder)tempOutput.json"
141+
142+
New-AzStreamAnalyticsTransformation -ResourceGroupName $rgName -JobName $asaJobName1 -Name $asaJobName1 -StreamingUnit $asaJobStreamingUnit -Query $asaJobQuery1
143+
New-AzStreamAnalyticsTransformation -ResourceGroupName $rgName -JobName $asaJobName2 -Name $asaJobName2 -StreamingUnit $asaJobStreamingUnit -Query $asaJobQuery2
144+
145+
######################################################################
146+
#Gather necessary variables, paste results into next session
147+
148+
Write-Host "`
149+
`$rgName = `"$rgName`"`
150+
`$asaJobName1 = `"$asaJobName1`"`
151+
`$asaJobName2 = `"$asaJobName2`"`
152+
`$ehNamespace = `"$ehNamespace`"`
153+
`$ehName = `"$ehName`"`
154+
`$ehAuthorizationRuleName = `"$ehAuthorizationRuleName`"`
155+
`$ehKey = `"$($ehKey.PrimaryKey)`"`
156+
`$cosmosDBAccountKey = `"$($cosmosDBAccountKey.PrimaryMasterKey)`"`
157+
`$cosmosDBAccountName = `"$cosmosDBAccountName`"`
158+
`$cosmosDBDatabaseName = `"$cosmosDBDatabaseName`"`
159+
`$cosmosDBContainerName = `"$cosmosDBContainerName`""

‎BCDRTests/Test 01 - 1 input - 2 jobs - 1 output/02-run.ps1‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
######################################################################
2+
#Gather necessary variables from previous terminal if necessary, and paste them into a new window
3+
4+
Write-Host "`
5+
`$rgName = `"$rgName`"`
6+
`$asaJobName1 = `"$asaJobName1`"`
7+
`$asaJobName2 = `"$asaJobName2`"`
8+
`$ehNamespace = `"$ehNamespace`"`
9+
`$ehName = `"$ehName`"`
10+
`$ehAuthorizationRuleName = `"$ehAuthorizationRuleName`"`
11+
`$ehKey = `"$($ehKey.PrimaryKey)`"`
12+
`$cosmosDBAccountKey = `"$($cosmosDBAccountKey.PrimaryMasterKey)`"`
13+
`$cosmosDBAccountName = `"$cosmosDBAccountName`"`
14+
`$cosmosDBDatabaseName = `"$cosmosDBDatabaseName`"`
15+
`$cosmosDBContainerName = `"$cosmosDBContainerName`""
16+
17+
18+
######################################################################
19+
#Starting jobs
20+
Start-AzStreamAnalyticsJob -ResourceGroupName $rgName -Name $asaJobName1
21+
Start-AzStreamAnalyticsJob -ResourceGroupName $rgName -Name $asaJobName2
22+
23+
######################################################################
24+
#Emitting records
25+
26+
##this will block the current terminal
27+
##if necessary, install https://www.powershellgallery.com/packages/Azure.EventHub/
28+
29+
$s = Get-AzureEHSASToken `
30+
-URI "$ehNamespace.servicebus.windows.net/$ehName" `
31+
-AccessPolicyName $ehAuthorizationRuleName `
32+
-AccessPolicyKey $ehKey
33+
34+
while ($True) {
35+
Send-AzureEHDatagram `
36+
-URI "$ehNamespace.servicebus.windows.net/$ehName" `
37+
-SASToken $s `
38+
-Datagram ('{"DeviceId": '+(Get-Random -Maximum 8)+',"readingTimestamp": "'+(Get-Date -Format o)+'", "readingNum":'+(Get-Random -Maximum 1024)+'}')
39+
}
40+
41+
######################################################################
42+
#Observing records in Cosmos DB
43+
44+
##in a new terminal
45+
##if necessary, install https://www.powershellgallery.com/packages/CosmosDB/
46+
47+
$primaryKey = ConvertTo-SecureString -String $cosmosDBAccountKey -AsPlainText -Force
48+
$cosmosDbContext = New-CosmosDbContext -Account $cosmosDBAccountName -Database $cosmosDBDatabaseName -Key $primaryKey
49+
50+
$query = "SELECT * FROM customers c WHERE c.deviceid = '0' ORDER BY c.windowend DESC"
51+
(Get-CosmosDbDocument -Context $cosmosDbContext -CollectionId $cosmosDBContainerName -Query $query) | Select-Object {$_.deviceid_minute,$_.jobname}
52+
53+
## We should obvserve a random alternance of jobnames over time
54+
55+
######################################################################
56+
#Stopping one job
57+
58+
Stop-AzStreamAnalyticsJob -ResourceGroupName $rgName -Name $asaJobName1
59+
60+
## From that point forward, new records will only be associated to the remaining ASA job
61+
62+
$query = "SELECT * FROM customers c WHERE c.deviceid = '0' ORDER BY c.windowend DESC"
63+
(Get-CosmosDbDocument -Context $cosmosDbContext -CollectionId $cosmosDBContainerName -Query $query) | Select-Object {$_.deviceid_minute,$_.jobname}
64+

‎BCDRTests/Test 01 - 1 input - 2 jobs - 1 output/03-cleanup.ps1‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
######################################################################
2+
#Gather necessary variables from previous terminal if necessary, and paste them into a new window
3+
4+
Write-Host "`
5+
`$rgName = `"$rgName`""
6+
7+
######################################################################
8+
#Delete the resource group
9+
10+
Remove-AzResourceGroup -Name $rgName -Force

0 commit comments

Comments
 (0)