From cf1dc3cb5a1cd5618538bf19a83c99f64289e459 Mon Sep 17 00:00:00 2001 From: Ibrahim Abedalghafer Date: Thu, 12 Feb 2026 19:49:42 +0300 Subject: [PATCH] CMK private standard agent setup --- .../README.md | 45 ++ .../main.bicep | 449 ++++++++++++++++++ .../main.bicepparam | 77 +++ .../add-project-capability-host.bicep | 34 ++ .../ai-account-encryption.bicep | 83 ++++ .../ai-account-identity.bicep | 63 +++ .../ai-project-identity-unique.bicep | 106 +++++ .../ai-project-identity.bicep | 103 ++++ .../ai-search-role-assignments.bicep | 43 ++ ...zure-storage-account-role-assignment.bicep | 24 + ...ge-container-role-assignments-unique.bicep | 38 ++ ...b-storage-container-role-assignments.bicep | 36 ++ .../cosmos-container-role-assignments.bicep | 32 ++ .../cosmosdb-account-role-assignment.bicep | 27 ++ .../existing-vnet.bicep | 89 ++++ .../format-project-workspace-id.bicep | 12 + .../network-agent-vnet.bicep | 67 +++ .../private-endpoint-and-dns.bicep | 405 ++++++++++++++++ .../standard-dependent-resources.bicep | 148 ++++++ .../modules-network-secured/subnet.bicep | 22 + .../validate-existing-resources.bicep | 94 ++++ .../modules-network-secured/vnet.bicep | 83 ++++ 22 files changed, 2080 insertions(+) create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/README.md create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/main.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/main.bicepparam create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/add-project-capability-host.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-account-encryption.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-account-identity.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-project-identity-unique.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-project-identity.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-search-role-assignments.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/azure-storage-account-role-assignment.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/blob-storage-container-role-assignments-unique.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/blob-storage-container-role-assignments.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/cosmos-container-role-assignments.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/cosmosdb-account-role-assignment.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/existing-vnet.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/format-project-workspace-id.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/network-agent-vnet.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/private-endpoint-and-dns.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/standard-dependent-resources.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/subnet.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/validate-existing-resources.bicep create mode 100644 infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/vnet.bicep diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/README.md b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/README.md new file mode 100644 index 000000000..eead301c7 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/README.md @@ -0,0 +1,45 @@ +--- +description: This template deploys an Azure AI Foundry account, project, and model deployment while using your key for encryption (Customer Managed Key) in full private setup. +page_type: sample +products: +- azure +- azure-resource-manager +urlFragment: aifoundry-cmk +languages: +- bicep +- json +--- +# Set up Azure AI Foundry using Customer Managed Keys for encryption + +This Azure AI Foundry template demonstrates how to deploy AI Foundry with Agents private network standard setup and customer-managed keys for encryption. + +## Prerequisites + +* An existing Azure Key Vault resource. This sample template does not create it. +* You must enable both the Soft Delete and Do Not Purge properties on the existing Azure Key Vault instance. +* If you use the Key Vault firewall, you must allow trusted Microsoft services to access the Azure Key Vault. +* The template uses RBAC roles for keyvault and assign the identity of the AI Foundry account and global cosmos DB account "Key Vault Crypto Service Encryption User" permission on keyvault +* Only RSA and RSA-HSM keys of size 2048 are supported. For more information about keys, see Key Vault keys in + +## Features +This template provides same features in template `15-private-network-standard-agent-setup` for selecting existing resources, different subscription dns zones and all other features + +## Run the Bicep deployment commands + +Steps: + ```bash + az deployment group create --resource-group --template-file main.bicep --parameters main.bicepparam + ``` + + +## Learn more +If you are new to Azure AI Foundry, see: + +- [Azure AI Foundry](https://learn.microsoft.com/azure/ai-foundry/) + +If you are new to template deployment, see: + +- [Azure Resource Manager documentation](https://learn.microsoft.com/azure/azure-resource-manager/) +- [Azure AI services quickstart article](https://learn.microsoft.com/azure/cognitive-services/resource-manager-template) + +`Tags: Microsoft.CognitiveServices/accounts/projects` diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/main.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/main.bicep new file mode 100644 index 000000000..29ed76b03 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/main.bicep @@ -0,0 +1,449 @@ +/* +Standard Setup Network Secured Steps for main.bicep +----------------------------------- +*/ +@description('Location for all resources.') +@allowed([ + 'westus' + 'eastus' + 'eastus2' + 'japaneast' + 'francecentral' + 'spaincentral' + 'uaenorth' + 'southcentralus' + 'italynorth' + 'germanywestcentral' + 'brazilsouth' + 'southafricanorth' + 'australiaeast' + 'swedencentral' + 'canadaeast' + 'westeurope' + 'westus3' + 'uksouth' + 'southindia' + + //only class B and C + 'koreacentral' + 'polandcentral' + 'switzerlandnorth' + 'norwayeast' +]) +param location string = 'eastus' + +@description('Name for your AI Services resource.') +param aiServices string = 'aiservices' + +// Model deployment parameters +@description('The name of the model you want to deploy') +param modelName string = 'gpt-4.1' +@description('The provider of your model') +param modelFormat string = 'OpenAI' +@description('The version of your model') +param modelVersion string = '2025-04-14' +@description('The sku of your model deployment') +param modelSkuName string = 'GlobalStandard' +@description('The tokens per minute (TPM) of your model deployment') +param modelCapacity int = 1 + +// Create a short, unique suffix, that will be unique to each resource group +param deploymentTimestamp string = utcNow('yyyyMMddHHmmss') +var uniqueSuffix = substring(uniqueString('${resourceGroup().id}-${deploymentTimestamp}'), 0, 4) +var accountName = toLower('${aiServices}${uniqueSuffix}') + +@description('Name for your project resource.') +param firstProjectName string = 'project' + +@description('This project will be a sub-resource of your account') +param projectDescription string = 'A project for the AI Foundry account with network secured deployed Agent' + +@description('The display name of the project') +param displayName string = 'network secured agent project' + +// Existing Virtual Network parameters +@description('Virtual Network name for the Agent to create new or existing virtual network') +param vnetName string = 'agent-vnet-test' + +@description('The name of Agents Subnet to create new or existing subnet for agents') +param agentSubnetName string = 'agent-subnet' + +@description('The name of Private Endpoint subnet to create new or existing subnet for private endpoints') +param peSubnetName string = 'pe-subnet' + +//Existing standard Agent required resources +@description('Existing Virtual Network name Resource ID') +param existingVnetResourceId string = '' + +@description('Address space for the VNet (only used for new VNet)') +param vnetAddressPrefix string = '' + +@description('Address prefix for the agent subnet. The default value is 192.168.0.0/24 but you can choose any size /26 or any class like 10.0.0.0 or 172.168.0.0') +param agentSubnetPrefix string = '' + +@description('Address prefix for the private endpoint subnet') +param peSubnetPrefix string = '' + +@description('The AI Search Service full ARM Resource ID. This is an optional field, and if not provided, the resource will be created.') +param aiSearchResourceId string = '' +@description('The AI Storage Account full ARM Resource ID. This is an optional field, and if not provided, the resource will be created.') +param azureStorageAccountResourceId string = '' +@description('The Cosmos DB Account full ARM Resource ID. This is an optional field, and if not provided, the resource will be created.') +param azureCosmosDBAccountResourceId string = '' + +//New Param for resource group of Private DNS zones +//@description('Optional: Resource group containing existing private DNS zones. If specified, DNS zones will not be created.') +//param existingDnsZonesResourceGroup string = '' + +@description('Subscription ID where existing private DNS zones are located. Leave empty to use current subscription.') +param dnsZonesSubscriptionId string = '' + +@description('Object mapping DNS zone names to their resource group, or empty string to indicate creation') +param existingDnsZones object = { + 'privatelink.services.ai.azure.com': '' + 'privatelink.openai.azure.com': '' + 'privatelink.cognitiveservices.azure.com': '' + 'privatelink.search.windows.net': '' + 'privatelink.blob.core.windows.net': '' + 'privatelink.documents.azure.com': '' +} + +@description('Zone Names for Validation of existing Private Dns Zones') +param dnsZoneNames array = [ + 'privatelink.services.ai.azure.com' + 'privatelink.openai.azure.com' + 'privatelink.cognitiveservices.azure.com' + 'privatelink.search.windows.net' + 'privatelink.blob.core.windows.net' + 'privatelink.documents.azure.com' +] + +@description('Name of the Azure Key Vault target') +param keyVaultName string = '' + +@description('Name of the Azure Key Vault key') +param keyName string = '<>YOUR KEY NAME>' + +@description('Version of the Azure Key Vault key') +param keyVersion string = '' + +var keyVaultUri = 'https://${keyVaultName}.vault.azure.net/' +var keyVaultKeyUri = '${keyVaultUri}keys/${keyName}' + +var projectName = toLower('${firstProjectName}${uniqueSuffix}') +var cosmosDBName = toLower('${aiServices}${uniqueSuffix}cosmosdb') +var aiSearchName = toLower('${aiServices}${uniqueSuffix}search') +var azureStorageName = toLower('${aiServices}${uniqueSuffix}storage') + +// Check if existing resources have been passed in +var storagePassedIn = azureStorageAccountResourceId != '' +var searchPassedIn = aiSearchResourceId != '' +var cosmosPassedIn = azureCosmosDBAccountResourceId != '' +var existingVnetPassedIn = existingVnetResourceId != '' + + +var acsParts = split(aiSearchResourceId, '/') +var aiSearchServiceSubscriptionId = searchPassedIn ? acsParts[2] : subscription().subscriptionId +var aiSearchServiceResourceGroupName = searchPassedIn ? acsParts[4] : resourceGroup().name + +var cosmosParts = split(azureCosmosDBAccountResourceId, '/') +var cosmosDBSubscriptionId = cosmosPassedIn ? cosmosParts[2] : subscription().subscriptionId +var cosmosDBResourceGroupName = cosmosPassedIn ? cosmosParts[4] : resourceGroup().name + +var storageParts = split(azureStorageAccountResourceId, '/') +var azureStorageSubscriptionId = storagePassedIn ? storageParts[2] : subscription().subscriptionId +var azureStorageResourceGroupName = storagePassedIn ? storageParts[4] : resourceGroup().name + +var vnetParts = split(existingVnetResourceId, '/') +var vnetSubscriptionId = existingVnetPassedIn ? vnetParts[2] : subscription().subscriptionId +var vnetResourceGroupName = existingVnetPassedIn ? vnetParts[4] : resourceGroup().name +var existingVnetName = existingVnetPassedIn ? last(vnetParts) : vnetName +var trimVnetName = trim(existingVnetName) + +// Resolve DNS zones subscription ID - use current subscription if not specified +var resolvedDnsZonesSubscriptionId = empty(dnsZonesSubscriptionId) ? subscription().subscriptionId : dnsZonesSubscriptionId + +@description('The name of the project capability host to be created') +param projectCapHost string = 'caphostproj' + +// Create Virtual Network and Subnets +module vnet 'modules-network-secured/network-agent-vnet.bicep' = { + name: 'vnet-${trimVnetName}-${uniqueSuffix}-deployment' + params: { + location: location + vnetName: trimVnetName + useExistingVnet: existingVnetPassedIn + existingVnetResourceGroupName: vnetResourceGroupName + agentSubnetName: agentSubnetName + peSubnetName: peSubnetName + vnetAddressPrefix: vnetAddressPrefix + agentSubnetPrefix: agentSubnetPrefix + peSubnetPrefix: peSubnetPrefix + existingVnetSubscriptionId: vnetSubscriptionId + } +} + +/* + Create the AI Services account and gpt-4o model deployment +*/ +module aiAccount 'modules-network-secured/ai-account-identity.bicep' = { + name: '${accountName}-${uniqueSuffix}-deployment' + params: { + // workspace organization + accountName: accountName + location: location + modelName: modelName + modelFormat: modelFormat + modelVersion: modelVersion + modelSkuName: modelSkuName + modelCapacity: modelCapacity + agentSubnetId: vnet.outputs.agentSubnetId + } +} +/* + Validate existing resources + This module will check if the AI Search Service, Storage Account, and Cosmos DB Account already exist. + If they do, it will set the corresponding output to true. If they do not exist, it will set the output to false. +*/ +module validateExistingResources 'modules-network-secured/validate-existing-resources.bicep' = { + name: 'validate-existing-resources-${uniqueSuffix}-deployment' + params: { + aiSearchResourceId: aiSearchResourceId + azureStorageAccountResourceId: azureStorageAccountResourceId + azureCosmosDBAccountResourceId: azureCosmosDBAccountResourceId + existingDnsZones: existingDnsZones + dnsZoneNames: dnsZoneNames + dnsZonesSubscriptionId: resolvedDnsZonesSubscriptionId + } +} + +// This module will create new agent dependent resources +// A Cosmos DB account, an AI Search Service, and a Storage Account are created if they do not already exist +module aiDependencies 'modules-network-secured/standard-dependent-resources.bicep' = { + name: 'dependencies-${uniqueSuffix}-deployment' + params: { + location: location + azureStorageName: azureStorageName + aiSearchName: aiSearchName + cosmosDBName: cosmosDBName + + // AI Search Service parameters + aiSearchResourceId: aiSearchResourceId + aiSearchExists: validateExistingResources.outputs.aiSearchExists + + // Storage Account + azureStorageAccountResourceId: azureStorageAccountResourceId + azureStorageExists: validateExistingResources.outputs.azureStorageExists + + // Cosmos DB Account + cosmosDBResourceId: azureCosmosDBAccountResourceId + cosmosDBExists: validateExistingResources.outputs.cosmosDBExists + } +} + +resource storage 'Microsoft.Storage/storageAccounts@2022-05-01' existing = { + name: aiDependencies.outputs.azureStorageName + scope: resourceGroup(azureStorageSubscriptionId, azureStorageResourceGroupName) +} + + +resource aiSearch 'Microsoft.Search/searchServices@2023-11-01' existing = { + name: aiDependencies.outputs.aiSearchName + scope: resourceGroup(aiDependencies.outputs.aiSearchServiceSubscriptionId, aiDependencies.outputs.aiSearchServiceResourceGroupName) +} + +resource cosmosDB 'Microsoft.DocumentDB/databaseAccounts@2024-11-15' existing = { + name: aiDependencies.outputs.cosmosDBName + scope: resourceGroup(cosmosDBSubscriptionId, cosmosDBResourceGroupName) +} + +// Private Endpoint and DNS Configuration +// This module sets up private network access for all Azure services: +// 1. Creates private endpoints in the specified subnet +// 2. Sets up private DNS zones for each service +// 3. Links private DNS zones to the VNet for name resolution +// 4. Configures network policies to restrict access to private endpoints only +module privateEndpointAndDNS 'modules-network-secured/private-endpoint-and-dns.bicep' = { + name: '${uniqueSuffix}-private-endpoint' + params: { + aiAccountName: aiAccount.outputs.accountName // AI Services to secure + aiSearchName: aiDependencies.outputs.aiSearchName // AI Search to secure + storageName: aiDependencies.outputs.azureStorageName // Storage to secure + cosmosDBName:aiDependencies.outputs.cosmosDBName + vnetName: vnet.outputs.virtualNetworkName // VNet containing subnets + peSubnetName: vnet.outputs.peSubnetName // Subnet for private endpoints + suffix: uniqueSuffix // Unique identifier + vnetResourceGroupName: vnet.outputs.virtualNetworkResourceGroup + vnetSubscriptionId: vnet.outputs.virtualNetworkSubscriptionId // Subscription ID for the VNet + cosmosDBSubscriptionId: cosmosDBSubscriptionId // Subscription ID for Cosmos DB + cosmosDBResourceGroupName: cosmosDBResourceGroupName // Resource Group for Cosmos DB + aiSearchSubscriptionId: aiSearchServiceSubscriptionId // Subscription ID for AI Search Service + aiSearchResourceGroupName: aiSearchServiceResourceGroupName // Resource Group for AI Search Service + storageAccountResourceGroupName: azureStorageResourceGroupName // Resource Group for Storage Account + storageAccountSubscriptionId: azureStorageSubscriptionId // Subscription ID for Storage Account + existingDnsZones: existingDnsZones + dnsZonesSubscriptionId: resolvedDnsZonesSubscriptionId + } + dependsOn: [ + aiSearch // Ensure AI Search exists + storage // Ensure Storage exists + cosmosDB // Ensure Cosmos DB exists + ] + } + +// Set up customer-managed key encryption once managed identity has been created +module encryptionUpdate 'modules-network-secured/ai-account-encryption.bicep' = { + name: 'updateEncryption' + params: { + aiFoundryName: aiAccount.outputs.accountName + aiFoundryPrincipal: aiAccount.outputs.accountPrincipalId + keyVaultName: keyVaultName + location: location + keyVaultUri: keyVaultUri + keyName: keyName + keyVersion: keyVersion + } + dependsOn: [ + aiDependencies + ] +} + + + +/* + Creates a new project (sub-resource of the AI Services account) +*/ +module aiProject 'modules-network-secured/ai-project-identity.bicep' = { + name: '${projectName}-${uniqueSuffix}-deployment' + params: { + // workspace organization + projectName: projectName + projectDescription: projectDescription + displayName: displayName + location: location + + aiSearchName: aiDependencies.outputs.aiSearchName + aiSearchServiceResourceGroupName: aiDependencies.outputs.aiSearchServiceResourceGroupName + aiSearchServiceSubscriptionId: aiDependencies.outputs.aiSearchServiceSubscriptionId + + cosmosDBName: aiDependencies.outputs.cosmosDBName + cosmosDBSubscriptionId: aiDependencies.outputs.cosmosDBSubscriptionId + cosmosDBResourceGroupName: aiDependencies.outputs.cosmosDBResourceGroupName + + azureStorageName: aiDependencies.outputs.azureStorageName + azureStorageSubscriptionId: aiDependencies.outputs.azureStorageSubscriptionId + azureStorageResourceGroupName: aiDependencies.outputs.azureStorageResourceGroupName + // dependent resources + accountName: aiAccount.outputs.accountName + } + dependsOn: [ + encryptionUpdate + privateEndpointAndDNS + cosmosDB + aiSearch + storage + ] +} + +module formatProjectWorkspaceId 'modules-network-secured/format-project-workspace-id.bicep' = { + name: 'format-project-workspace-id-${uniqueSuffix}-deployment' + params: { + projectWorkspaceId: aiProject.outputs.projectWorkspaceId + } +} + +/* + Assigns the project SMI the storage blob data contributor role on the storage account +*/ +module storageAccountRoleAssignment 'modules-network-secured/azure-storage-account-role-assignment.bicep' = { + name: 'storage-${azureStorageName}-${uniqueSuffix}-deployment' + scope: resourceGroup(azureStorageSubscriptionId, azureStorageResourceGroupName) + params: { + azureStorageName: aiDependencies.outputs.azureStorageName + projectPrincipalId: aiProject.outputs.projectPrincipalId + } + dependsOn: [ + storage + privateEndpointAndDNS + ] +} + +// The Comos DB Operator role must be assigned before the caphost is created +module cosmosAccountRoleAssignments 'modules-network-secured/cosmosdb-account-role-assignment.bicep' = { + name: 'cosmos-account-ra-${uniqueSuffix}-deployment' + scope: resourceGroup(cosmosDBSubscriptionId, cosmosDBResourceGroupName) + params: { + cosmosDBName: aiDependencies.outputs.cosmosDBName + projectPrincipalId: aiProject.outputs.projectPrincipalId + } + dependsOn: [ + cosmosDB + privateEndpointAndDNS + ] +} + +// This role can be assigned before or after the caphost is created +module aiSearchRoleAssignments 'modules-network-secured/ai-search-role-assignments.bicep' = { + name: 'ai-search-ra-${uniqueSuffix}-deployment' + scope: resourceGroup(aiSearchServiceSubscriptionId, aiSearchServiceResourceGroupName) + params: { + aiSearchName: aiDependencies.outputs.aiSearchName + projectPrincipalId: aiProject.outputs.projectPrincipalId + } + dependsOn: [ + aiSearch + privateEndpointAndDNS + ] +} + +// This module creates the capability host for the project and account +module addProjectCapabilityHost 'modules-network-secured/add-project-capability-host.bicep' = { + name: 'capabilityHost-configuration-${uniqueSuffix}-deployment' + params: { + accountName: aiAccount.outputs.accountName + projectName: aiProject.outputs.projectName + cosmosDBConnection: aiProject.outputs.cosmosDBConnection + azureStorageConnection: aiProject.outputs.azureStorageConnection + aiSearchConnection: aiProject.outputs.aiSearchConnection + projectCapHost: projectCapHost + } + dependsOn: [ + aiSearch // Ensure AI Search exists + storage // Ensure Storage exists + cosmosDB + privateEndpointAndDNS + cosmosAccountRoleAssignments + storageAccountRoleAssignment + aiSearchRoleAssignments + ] +} + +// The Storage Blob Data Owner role must be assigned after the caphost is created +module storageContainersRoleAssignment 'modules-network-secured/blob-storage-container-role-assignments.bicep' = { + name: 'storage-containers-ra-${uniqueSuffix}-deployment' + scope: resourceGroup(azureStorageSubscriptionId, azureStorageResourceGroupName) + params: { + aiProjectPrincipalId: aiProject.outputs.projectPrincipalId + storageName: aiDependencies.outputs.azureStorageName + workspaceId: formatProjectWorkspaceId.outputs.projectWorkspaceIdGuid + } + dependsOn: [ + addProjectCapabilityHost + ] +} + +// The Cosmos Built-In Data Contributor role must be assigned after the caphost is created +module cosmosContainerRoleAssignments 'modules-network-secured/cosmos-container-role-assignments.bicep' = { + name: 'cosmos-containers-ra-${uniqueSuffix}-deployment' + scope: resourceGroup(cosmosDBSubscriptionId, cosmosDBResourceGroupName) + params: { + cosmosAccountName: aiDependencies.outputs.cosmosDBName + projectWorkspaceId: formatProjectWorkspaceId.outputs.projectWorkspaceIdGuid + projectPrincipalId: aiProject.outputs.projectPrincipalId + + } +dependsOn: [ + addProjectCapabilityHost + storageContainersRoleAssignment + ] +} diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/main.bicepparam b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/main.bicepparam new file mode 100644 index 000000000..15f9a41df --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/main.bicepparam @@ -0,0 +1,77 @@ +using './main.bicep' + +param location = 'westus' +param aiServices = 'foundry' +param modelName = 'gpt-4.1' +param modelFormat = 'OpenAI' +param modelVersion = '2025-04-14' +param modelSkuName = 'GlobalStandard' +param modelCapacity = 1 +param firstProjectName = 'project' +param projectDescription = 'A project for the AI Foundry account with network secured deployed Agent' +param displayName = 'project' +param peSubnetName = 'pesubnet' + +// Resource IDs for existing resources +// If you provide these, the deployment will use the existing resources instead of creating new ones +param existingVnetResourceId = '' +param vnetName = '' +param agentSubnetName = 'agentsubnet' +param aiSearchResourceId = '' +param azureStorageAccountResourceId = '' +param azureCosmosDBAccountResourceId = '' + +// Subscription ID where DNS zones are located (leave empty to use deployment subscription) +// ⚠️ If set to a different subscription, ALL zones below MUST have resource groups specified +param dnsZonesSubscriptionId = '' + +// DNS zone map: provide resource group name to use existing zone, or leave empty to create new +// Note: Empty values only allowed when dnsZonesSubscriptionId is empty or matches current subscription +param existingDnsZones = { + 'privatelink.services.ai.azure.com': '' + 'privatelink.openai.azure.com': '' + 'privatelink.cognitiveservices.azure.com': '' + 'privatelink.search.windows.net': '' + 'privatelink.blob.core.windows.net': '' + 'privatelink.documents.azure.com': '' +} + +//DNSZones names for validating if they exist +param dnsZoneNames = [ + 'privatelink.services.ai.azure.com' + 'privatelink.openai.azure.com' + 'privatelink.cognitiveservices.azure.com' + 'privatelink.search.windows.net' + 'privatelink.blob.core.windows.net' + 'privatelink.documents.azure.com' +] + +param keyVaultName = '' +param keyName = '' +param keyVersion = '' + + +// Network configuration (behavior depends on `existingVnetResourceId`) +// +// - NEW VNet (existingVnetResourceId is empty): +// The values below are used to CREATE the VNet and the two subnets. +// Provide explicit, non-overlapping CIDR ranges when creating a new VNet. +// +// - EXISTING VNet (existingVnetResourceId is provided): +// The module will reference the existing VNet. Subnet handling depends on the +// values you provide: +// * If `agentSubnetPrefix` or `peSubnetPrefix` are empty, the module may +// auto-derive subnet CIDRs from the existing VNet's address space +// (using cidrSubnet). This can produce /24 (or configured) subnets +// starting at index 0, 1, etc. +// * If you provide explicit subnet prefixes, the module will attempt to +// create or update subnets with those prefixes in the existing VNet. +// +// Important operational notes and risks (when existingVnetResourceId is provided): +// - Avoid CIDR overlaps with any existing subnets in the target VNet. Overlap +// leads to `NetcfgSubnetRangesOverlap` and failed deployments. +// - For highest safety when using an existing VNet, supply the existing `agentSubnetPrefix` and `peSubnetPrefix`. +param vnetAddressPrefix = '' +param agentSubnetPrefix = '' +param peSubnetPrefix = '' + diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/add-project-capability-host.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/add-project-capability-host.bicep new file mode 100644 index 000000000..dd2ac3297 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/add-project-capability-host.bicep @@ -0,0 +1,34 @@ +param cosmosDBConnection string +param azureStorageConnection string +param aiSearchConnection string +param projectName string +param accountName string +param projectCapHost string + +var threadConnections = ['${cosmosDBConnection}'] +var storageConnections = ['${azureStorageConnection}'] +var vectorStoreConnections = ['${aiSearchConnection}'] + + +resource account 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' existing = { + name: accountName +} + +resource project 'Microsoft.CognitiveServices/accounts/projects@2025-04-01-preview' existing = { + name: projectName + parent: account +} + +resource projectCapabilityHost 'Microsoft.CognitiveServices/accounts/projects/capabilityHosts@2025-04-01-preview' = { + name: projectCapHost + parent: project + properties: { + capabilityHostKind: 'Agents' + vectorStoreConnections: vectorStoreConnections + storageConnections: storageConnections + threadStorageConnections: threadConnections + } + +} + +output projectCapHost string = projectCapabilityHost.name diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-account-encryption.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-account-encryption.bicep new file mode 100644 index 000000000..580c220e5 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-account-encryption.bicep @@ -0,0 +1,83 @@ +param aiFoundryName string +param aiFoundryPrincipal string +param location string +param keyVaultName string +param keyVaultUri string +param keyName string +param keyVersion string + +// Reference account post creation, since we must wait for managed identity to be created to give access to CMK key vault +resource existingAccount 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' existing = { + name: aiFoundryName +} +// Reference the existing Key Vault +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { + name: keyVaultName +} + + +resource KeyVaultCryptoServiceEncryptionUser 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name:'e147488a-f6f5-4113-8e2d-b22465e65bf6' // Built-in role for Key Vault Crypto Service Encryption User + scope: resourceGroup() +} + +resource KeyVaultCryptoServiceEncryptionUserassignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: keyVault + name: guid(aiFoundryPrincipal, KeyVaultCryptoServiceEncryptionUser.id, keyVault.id) + properties: { + principalId: aiFoundryPrincipal + roleDefinitionId: KeyVaultCryptoServiceEncryptionUser.id + principalType: 'ServicePrincipal' + } +} + +resource KeyVaultCryptoServiceEncryptioncosmosassignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: keyVault + name: guid('a232010e-820c-4083-83bb-3ace5fc29d0b', KeyVaultCryptoServiceEncryptionUser.id, keyVault.id) + properties: { + principalId: 'a232010e-820c-4083-83bb-3ace5fc29d0b' + roleDefinitionId: KeyVaultCryptoServiceEncryptionUser.id + principalType: 'ServicePrincipal' + } +} + + + +// Set customer-managed key encryption on account +resource accountUpdate 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' = { + name: existingAccount.name + location: location + identity: { + type: 'SystemAssigned' + } + kind: 'AIServices' + sku: { + name: 'S0' + } + properties: { + // new + encryption: { + keySource: 'Microsoft.KeyVault' + keyVaultProperties: { + keyVaultUri: keyVaultUri + keyName: keyName + keyVersion: keyVersion + } + } + networkAcls: { + defaultAction: 'Deny' + virtualNetworkRules: [] + ipRules: [] + bypass:'AzureServices' + } + + publicNetworkAccess: 'Disabled' + allowProjectManagement: true + customSubDomainName: aiFoundryName + disableLocalAuth: false + } + dependsOn: [ + KeyVaultCryptoServiceEncryptionUserassignment + KeyVaultCryptoServiceEncryptioncosmosassignment + ] +} diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-account-identity.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-account-identity.bicep new file mode 100644 index 000000000..fbcb95740 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-account-identity.bicep @@ -0,0 +1,63 @@ +param accountName string +param location string +param modelName string +param modelFormat string +param modelVersion string +param modelSkuName string +param modelCapacity int +param agentSubnetId string +param networkInjection string = 'true' + +#disable-next-line BCP036 +resource account 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' = { + name: accountName + location: location + sku: { + name: 'S0' + } + kind: 'AIServices' + identity: { + type: 'SystemAssigned' + } + properties: { + allowProjectManagement: true + customSubDomainName: accountName + networkAcls: { + defaultAction: 'Deny' + virtualNetworkRules: [] + ipRules: [] + bypass:'AzureServices' + } + publicNetworkAccess: 'Disabled' + networkInjections:((networkInjection == 'true') ? [ + { + scenario: 'agent' + subnetArmId: agentSubnetId + useMicrosoftManagedNetwork: false + } + ] : null ) + disableLocalAuth: false + } +} + +#disable-next-line BCP081 +// resource modelDeployment 'Microsoft.CognitiveServices/accounts/deployments@2025-04-01-preview'= { +// parent: account +// name: modelName +// sku : { +// capacity: modelCapacity +// name: modelSkuName +// } +// properties: { +// model:{ +// name: modelName +// format: modelFormat +// version: modelVersion +// } +// } +// } + +output accountName string = account.name +output accountID string = account.id +output accountTarget string = account.properties.endpoint +output accountPrincipalId string = account.identity.principalId diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-project-identity-unique.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-project-identity-unique.bicep new file mode 100644 index 000000000..471e1fb98 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-project-identity-unique.bicep @@ -0,0 +1,106 @@ +param accountName string +param location string +param projectName string +param projectDescription string +param displayName string + +param aiSearchName string +param aiSearchServiceResourceGroupName string +param aiSearchServiceSubscriptionId string + +param cosmosDBName string +param cosmosDBSubscriptionId string +param cosmosDBResourceGroupName string + +param azureStorageName string +param azureStorageSubscriptionId string +param azureStorageResourceGroupName string + +// Add unique connection name parameter +param uniqueConnectionSuffix string = '' + +resource searchService 'Microsoft.Search/searchServices@2024-06-01-preview' existing = { + name: aiSearchName + scope: resourceGroup(aiSearchServiceSubscriptionId, aiSearchServiceResourceGroupName) +} +resource cosmosDBAccount 'Microsoft.DocumentDB/databaseAccounts@2024-12-01-preview' existing = { + name: cosmosDBName + scope: resourceGroup(cosmosDBSubscriptionId, cosmosDBResourceGroupName) +} +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-05-01' existing = { + name: azureStorageName + scope: resourceGroup(azureStorageSubscriptionId, azureStorageResourceGroupName) +} + +resource account 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' existing = { + name: accountName + scope: resourceGroup() +} + +resource project 'Microsoft.CognitiveServices/accounts/projects@2025-04-01-preview' = { + parent: account + name: projectName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + description: projectDescription + displayName: displayName + } + + // Use unique connection names by appending the suffix + resource project_connection_cosmosdb_account 'connections@2025-04-01-preview' = { + name: '${cosmosDBName}${uniqueConnectionSuffix}' + properties: { + category: 'CosmosDB' + target: cosmosDBAccount.properties.documentEndpoint + authType: 'AAD' + metadata: { + ApiType: 'Azure' + ResourceId: cosmosDBAccount.id + location: cosmosDBAccount.location + } + } + } + + resource project_connection_azure_storage 'connections@2025-04-01-preview' = { + name: '${azureStorageName}${uniqueConnectionSuffix}' + properties: { + category: 'AzureStorageAccount' + target: storageAccount.properties.primaryEndpoints.blob + authType: 'AAD' + metadata: { + ApiType: 'Azure' + ResourceId: storageAccount.id + location: storageAccount.location + } + } + } + + resource project_connection_azureai_search 'connections@2025-04-01-preview' = { + name: '${aiSearchName}${uniqueConnectionSuffix}' + properties: { + category: 'CognitiveSearch' + target: 'https://${aiSearchName}.search.windows.net' + authType: 'AAD' + metadata: { + ApiType: 'Azure' + ResourceId: searchService.id + location: searchService.location + } + } + } +} + +output projectName string = project.name +output projectId string = project.id +output projectPrincipalId string = project.identity.principalId + +#disable-next-line BCP053 +output projectWorkspaceId string = project.properties.internalId + +// Return the unique connection names +output cosmosDBConnection string = '${cosmosDBName}${uniqueConnectionSuffix}' +output azureStorageConnection string = '${azureStorageName}${uniqueConnectionSuffix}' +output aiSearchConnection string = '${aiSearchName}${uniqueConnectionSuffix}' diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-project-identity.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-project-identity.bicep new file mode 100644 index 000000000..90aebfbd3 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-project-identity.bicep @@ -0,0 +1,103 @@ +param accountName string +param location string +param projectName string +param projectDescription string +param displayName string + +param aiSearchName string +param aiSearchServiceResourceGroupName string +param aiSearchServiceSubscriptionId string + +param cosmosDBName string +param cosmosDBSubscriptionId string +param cosmosDBResourceGroupName string + +param azureStorageName string +param azureStorageSubscriptionId string +param azureStorageResourceGroupName string + +resource searchService 'Microsoft.Search/searchServices@2024-06-01-preview' existing = { + name: aiSearchName + scope: resourceGroup(aiSearchServiceSubscriptionId, aiSearchServiceResourceGroupName) +} +resource cosmosDBAccount 'Microsoft.DocumentDB/databaseAccounts@2024-12-01-preview' existing = { + name: cosmosDBName + scope: resourceGroup(cosmosDBSubscriptionId, cosmosDBResourceGroupName) +} +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-05-01' existing = { + name: azureStorageName + scope: resourceGroup(azureStorageSubscriptionId, azureStorageResourceGroupName) +} + +resource account 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' existing = { + name: accountName + scope: resourceGroup() +} + +resource project 'Microsoft.CognitiveServices/accounts/projects@2025-04-01-preview' = { + parent: account + name: projectName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + description: projectDescription + displayName: displayName + } + + resource project_connection_cosmosdb_account 'connections@2025-04-01-preview' = { + name: cosmosDBName + properties: { + category: 'CosmosDB' + target: cosmosDBAccount.properties.documentEndpoint + authType: 'AAD' + metadata: { + ApiType: 'Azure' + ResourceId: cosmosDBAccount.id + location: cosmosDBAccount.location + } + } + } + + resource project_connection_azure_storage 'connections@2025-04-01-preview' = { + name: azureStorageName + properties: { + category: 'AzureStorageAccount' + target: storageAccount.properties.primaryEndpoints.blob + authType: 'AAD' + metadata: { + ApiType: 'Azure' + ResourceId: storageAccount.id + location: storageAccount.location + } + } + } + + resource project_connection_azureai_search 'connections@2025-04-01-preview' = { + name: aiSearchName + properties: { + category: 'CognitiveSearch' + target: 'https://${aiSearchName}.search.windows.net' + authType: 'AAD' + metadata: { + ApiType: 'Azure' + ResourceId: searchService.id + location: searchService.location + } + } + } + +} + +output projectName string = project.name +output projectId string = project.id +output projectPrincipalId string = project.identity.principalId + +#disable-next-line BCP053 +output projectWorkspaceId string = project.properties.internalId + +// return the BYO connection names +output cosmosDBConnection string = cosmosDBName +output azureStorageConnection string = azureStorageName +output aiSearchConnection string = aiSearchName diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-search-role-assignments.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-search-role-assignments.bicep new file mode 100644 index 000000000..715663a6c --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/ai-search-role-assignments.bicep @@ -0,0 +1,43 @@ +// Assigns the necessary roles to the AI project + +@description('Name of the AI Search resource') +param aiSearchName string + +@description('Principal ID of the AI project') +param projectPrincipalId string + +resource searchService 'Microsoft.Search/searchServices@2024-06-01-preview' existing = { + name: aiSearchName + scope: resourceGroup() +} + +// search roles +resource searchIndexDataContributorRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '8ebe5a00-799e-43f5-93ac-243d3dce84a7' + scope: resourceGroup() +} + +resource searchIndexDataContributorAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: searchService + name: guid(projectPrincipalId, searchIndexDataContributorRole.id, searchService.id) + properties: { + principalId: projectPrincipalId + roleDefinitionId: searchIndexDataContributorRole.id + principalType: 'ServicePrincipal' + } +} + +resource searchServiceContributorRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '7ca78c08-252a-4471-8644-bb5ff32d4ba0' + scope: resourceGroup() +} + +resource searchServiceContributorRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: searchService + name: guid(projectPrincipalId, searchServiceContributorRole.id, searchService.id) + properties: { + principalId: projectPrincipalId + roleDefinitionId: searchServiceContributorRole.id + principalType: 'ServicePrincipal' + } +} diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/azure-storage-account-role-assignment.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/azure-storage-account-role-assignment.bicep new file mode 100644 index 000000000..afc355a48 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/azure-storage-account-role-assignment.bicep @@ -0,0 +1,24 @@ +param azureStorageName string +param projectPrincipalId string + +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-05-01' existing = { + name: azureStorageName + scope: resourceGroup() +} + +// Blob Storage Owner: b7e6dc6d-f1e8-4753-8033-0f276bb0955b +// Blob Storage Contributor: ba92f5b4-2d11-453d-a403-e96b0029c9fe +resource storageBlobDataContributor 'Microsoft.Authorization/roleDefinitions@2022-05-01-preview' existing = { + name: 'ba92f5b4-2d11-453d-a403-e96b0029c9fe' + scope: resourceGroup() +} + +resource storageBlobDataContributorRoleAssignmentProject 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: storageAccount + name: guid(projectPrincipalId, storageBlobDataContributor.id, storageAccount.id) + properties: { + principalId: projectPrincipalId + roleDefinitionId: storageBlobDataContributor.id + principalType: 'ServicePrincipal' + } +} diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/blob-storage-container-role-assignments-unique.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/blob-storage-container-role-assignments-unique.bicep new file mode 100644 index 000000000..2535a42c9 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/blob-storage-container-role-assignments-unique.bicep @@ -0,0 +1,38 @@ +@description('Name of the storage account') +param storageName string + +@description('Principal ID of the AI Project') +param aiProjectPrincipalId string + +@description('Workspace Id of the AI Project') +param workspaceId string + +@description('Unique suffix to make role assignment unique') +param uniqueSuffix string + +// Reference existing storage account +resource storage 'Microsoft.Storage/storageAccounts@2022-05-01' existing = { + name: storageName + scope: resourceGroup() +} + +// Storage Blob Data Owner Role +resource storageBlobDataOwner 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b' // Built-in role ID + scope: resourceGroup() +} + +var conditionStr= '((!(ActionMatches{\'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/tags/read\'}) AND !(ActionMatches{\'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/filter/action\'}) AND !(ActionMatches{\'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/tags/write\'}) ) OR (@Resource[Microsoft.Storage/storageAccounts/blobServices/containers:name] StringStartsWithIgnoreCase \'${workspaceId}\' AND @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:name] StringLikeIgnoreCase \'*-azureml-agent\'))' + +// Assign Storage Blob Data Owner role with unique name +resource storageBlobDataOwnerAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: storage + name: guid(storageBlobDataOwner.id, storage.id, aiProjectPrincipalId, uniqueSuffix) + properties: { + principalId: aiProjectPrincipalId + roleDefinitionId: storageBlobDataOwner.id + principalType: 'ServicePrincipal' + conditionVersion: '2.0' + condition: conditionStr + } +} diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/blob-storage-container-role-assignments.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/blob-storage-container-role-assignments.bicep new file mode 100644 index 000000000..71abc97d6 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/blob-storage-container-role-assignments.bicep @@ -0,0 +1,36 @@ +@description('Name of the storage account') +param storageName string + +@description('Principal ID of the AI Project') +param aiProjectPrincipalId string + +@description('Workspace Id of the AI Project') +param workspaceId string + + +// Reference existing storage account +resource storage 'Microsoft.Storage/storageAccounts@2022-05-01' existing = { + name: storageName + scope: resourceGroup() +} + +// Storage Blob Data Owner Role +resource storageBlobDataOwner 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b' // Built-in role ID + scope: resourceGroup() +} + +var conditionStr= '((!(ActionMatches{\'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/tags/read\'}) AND !(ActionMatches{\'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/filter/action\'}) AND !(ActionMatches{\'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/tags/write\'}) ) OR (@Resource[Microsoft.Storage/storageAccounts/blobServices/containers:name] StringStartsWithIgnoreCase \'${workspaceId}\' AND @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:name] StringLikeIgnoreCase \'*-azureml-agent\'))' + +// Assign Storage Blob Data Owner role +resource storageBlobDataOwnerAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: storage + name: guid(storageBlobDataOwner.id, storage.id) + properties: { + principalId: aiProjectPrincipalId + roleDefinitionId: storageBlobDataOwner.id + principalType: 'ServicePrincipal' + conditionVersion: '2.0' + condition: conditionStr + } +} diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/cosmos-container-role-assignments.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/cosmos-container-role-assignments.bicep new file mode 100644 index 000000000..bfac1da34 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/cosmos-container-role-assignments.bicep @@ -0,0 +1,32 @@ +// Assigns the necessary roles to the AI project + +@description('Name of the AI Search resource') +param cosmosAccountName string + +@description('Project name') +param projectPrincipalId string + +param projectWorkspaceId string + +resource cosmosAccount 'Microsoft.DocumentDB/databaseAccounts@2024-12-01-preview' existing = { + name: cosmosAccountName + scope: resourceGroup() +} + +var roleDefinitionId = resourceId( + 'Microsoft.DocumentDB/databaseAccounts/sqlRoleDefinitions', + cosmosAccountName, + '00000000-0000-0000-0000-000000000002' +) + +var accountScope = '/subscriptions/${subscription().subscriptionId}/resourceGroups/${resourceGroup().name}/providers/Microsoft.DocumentDB/databaseAccounts/${cosmosAccountName}/dbs/enterprise_memory' + +resource containerRoleAssignmentUserContainer 'Microsoft.DocumentDB/databaseAccounts/sqlRoleAssignments@2022-05-15' = { + parent: cosmosAccount + name: guid(projectWorkspaceId, cosmosAccountName, roleDefinitionId, projectPrincipalId) + properties: { + principalId: projectPrincipalId + roleDefinitionId: roleDefinitionId + scope: accountScope + } +} diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/cosmosdb-account-role-assignment.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/cosmosdb-account-role-assignment.bicep new file mode 100644 index 000000000..d5d083486 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/cosmosdb-account-role-assignment.bicep @@ -0,0 +1,27 @@ +// Assigns Role Cosmos DB Operator to the Project Principal ID +@description('Name of the Cosmos DB resource') +param cosmosDBName string + +@description('Principal ID of the AI project') +param projectPrincipalId string + + +resource cosmosDBAccount 'Microsoft.DocumentDB/databaseAccounts@2024-12-01-preview' existing = { + name: cosmosDBName + scope: resourceGroup() +} + +resource cosmosDBOperatorRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '230815da-be43-4aae-9cb4-875f7bd000aa' + scope: resourceGroup() +} + +resource cosmosDBOperatorRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: cosmosDBAccount + name: guid(projectPrincipalId, cosmosDBOperatorRole.id, cosmosDBAccount.id) + properties: { + principalId: projectPrincipalId + roleDefinitionId: cosmosDBOperatorRole.id + principalType: 'ServicePrincipal' + } +} diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/existing-vnet.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/existing-vnet.bicep new file mode 100644 index 000000000..b371d61e5 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/existing-vnet.bicep @@ -0,0 +1,89 @@ +/* +Virtual Network Module +This module works with existing virtual networks and required subnets. + +1. Flexibility: + - Works with any existing VNet address space + - Can use existing subnets or create new ones + - Cross-resource group support + +2. Security Features: + - Network isolation + - Subnet delegation for containerized workloads + - Private endpoint subnet for secure connectivity +*/ + + +@description('The name of the existing virtual network') +param vnetName string + +@description('Subscription ID of virtual network (if different from current subscription)') +param vnetSubscriptionId string = subscription().subscriptionId + +@description('Resource Group name of the existing VNet (if different from current resource group)') +param vnetResourceGroupName string = resourceGroup().name + +@description('The name of Agents Subnet') +param agentSubnetName string = 'agent-subnet' + +@description('The name of Private Endpoint subnet') +param peSubnetName string = 'pe-subnet' + +@description('Address prefix for the agent subnet (only needed if creating new subnet)') +param agentSubnetPrefix string = '' + +@description('Address prefix for the private endpoint subnet (only needed if creating new subnet)') +param peSubnetPrefix string = '' + +// Get the address space (array of CIDR strings) +var vnetAddressSpace = existingVNet.properties.addressSpace.addressPrefixes[0] + +var agentSubnetSpaces = empty(agentSubnetPrefix) ? cidrSubnet(vnetAddressSpace, 24, 0) : agentSubnetPrefix +var peSubnetSpaces = empty(peSubnetPrefix) ? cidrSubnet(vnetAddressSpace, 24, 1) : peSubnetPrefix + +// Reference the existing virtual network +resource existingVNet 'Microsoft.Network/virtualNetworks@2024-05-01' existing = { + name: vnetName + scope: resourceGroup(vnetResourceGroupName) +} + +// Create the agent subnet if requested +module agentSubnet 'subnet.bicep' = { + name: 'agent-subnet-${uniqueString(deployment().name, agentSubnetName)}' + scope: resourceGroup(vnetResourceGroupName) + params: { + vnetName: vnetName + subnetName: agentSubnetName + addressPrefix: agentSubnetSpaces + delegations: [ + { + name: 'Microsoft.App/environments' + properties: { + serviceName: 'Microsoft.App/environments' + } + } + ] + } +} + +// Create the private endpoint subnet if requested +module peSubnet 'subnet.bicep' = { + name: 'pe-subnet-${uniqueString(deployment().name, peSubnetName)}' + scope: resourceGroup(vnetResourceGroupName) + params: { + vnetName: vnetName + subnetName: peSubnetName + addressPrefix: peSubnetSpaces + delegations: [] + } +} + +// Output variables +output peSubnetName string = peSubnetName +output agentSubnetName string = agentSubnetName +output agentSubnetId string = '${existingVNet.id}/subnets/${agentSubnetName}' +output peSubnetId string = '${existingVNet.id}/subnets/${peSubnetName}' +output virtualNetworkName string = existingVNet.name +output virtualNetworkId string = existingVNet.id +output virtualNetworkResourceGroup string = vnetResourceGroupName +output virtualNetworkSubscriptionId string = vnetSubscriptionId diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/format-project-workspace-id.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/format-project-workspace-id.bicep new file mode 100644 index 000000000..ac7d0c3f2 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/format-project-workspace-id.bicep @@ -0,0 +1,12 @@ + +param projectWorkspaceId string + +var part1 = substring(projectWorkspaceId, 0, 8) // First 8 characters +var part2 = substring(projectWorkspaceId, 8, 4) // Next 4 characters +var part3 = substring(projectWorkspaceId, 12, 4) // Next 4 characters +var part4 = substring(projectWorkspaceId, 16, 4) // Next 4 characters +var part5 = substring(projectWorkspaceId, 20, 12) // Remaining 12 characters + +var formattedGuid = '${part1}-${part2}-${part3}-${part4}-${part5}' + +output projectWorkspaceIdGuid string = formattedGuid diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/network-agent-vnet.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/network-agent-vnet.bicep new file mode 100644 index 000000000..bad8a4f27 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/network-agent-vnet.bicep @@ -0,0 +1,67 @@ +@description('Azure region for the deployment') +param location string + +@description('The name of the virtual network') +param vnetName string + +@description('Indicates if an existing VNet should be used') +param useExistingVnet bool = false + +@description('Subscription ID of the existing VNet (if different from current subscription)') +param existingVnetSubscriptionId string = subscription().subscriptionId + +@description('Resource Group name of the existing VNet (if different from current resource group)') +param existingVnetResourceGroupName string = resourceGroup().name + +@description('The name of Agents Subnet') +param agentSubnetName string = 'agent-subnet' + +@description('The name of Private Endpoint subnet') +param peSubnetName string = 'pe-subnet' + +@description('Address space for the VNet (only used for new VNet)') +param vnetAddressPrefix string = '' + +@description('Address prefix for the agent subnet') +param agentSubnetPrefix string = '' + +@description('Address prefix for the private endpoint subnet') +param peSubnetPrefix string = '' + +// Create new VNet if needed +module newVNet 'vnet.bicep' = if (!useExistingVnet) { + name: 'vnet-deployment' + params: { + location: location + vnetName: vnetName + agentSubnetName: agentSubnetName + peSubnetName: peSubnetName + vnetAddressPrefix: vnetAddressPrefix + agentSubnetPrefix: agentSubnetPrefix + peSubnetPrefix: peSubnetPrefix + } +} + +// Use existing VNet if requested +module existingVNet 'existing-vnet.bicep' = if (useExistingVnet) { + name: 'existing-vnet-deployment' + params: { + vnetName: vnetName + vnetResourceGroupName: existingVnetResourceGroupName + vnetSubscriptionId: existingVnetSubscriptionId + agentSubnetName: agentSubnetName + peSubnetName: peSubnetName + agentSubnetPrefix: agentSubnetPrefix + peSubnetPrefix: peSubnetPrefix + } +} + +// Provide unified outputs regardless of which module was used +output virtualNetworkName string = useExistingVnet ? existingVNet.outputs.virtualNetworkName : newVNet.outputs.virtualNetworkName +output virtualNetworkId string = useExistingVnet ? existingVNet.outputs.virtualNetworkId : newVNet.outputs.virtualNetworkId +output virtualNetworkSubscriptionId string = useExistingVnet ? existingVNet.outputs.virtualNetworkSubscriptionId : newVNet.outputs.virtualNetworkSubscriptionId +output virtualNetworkResourceGroup string = useExistingVnet ? existingVNet.outputs.virtualNetworkResourceGroup : newVNet.outputs.virtualNetworkResourceGroup +output agentSubnetName string = agentSubnetName +output peSubnetName string = peSubnetName +output agentSubnetId string = useExistingVnet ? existingVNet.outputs.agentSubnetId : newVNet.outputs.agentSubnetId +output peSubnetId string = useExistingVnet ? existingVNet.outputs.peSubnetId : newVNet.outputs.peSubnetId diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/private-endpoint-and-dns.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/private-endpoint-and-dns.bicep new file mode 100644 index 000000000..a73c59468 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/private-endpoint-and-dns.bicep @@ -0,0 +1,405 @@ +/* +Private Endpoint and DNS Configuration Module +------------------------------------------ +This module configures private network access for Azure services using: + +1. Private Endpoints: + - Creates network interfaces in the specified subnet + - Establishes private connections to Azure services + - Enables secure access without public internet exposure + +2. Private DNS Zones: + - Enables custom DNS resolution for private endpoints + +3. DNS Zone Links: + - Links private DNS zones to the VNet + - Enables name resolution for resources in the VNet + - Prevents DNS resolution conflicts + +Security Benefits: +- Eliminates public internet exposure +- Enables secure access from within VNet +- Prevents data exfiltration through network +*/ + +// Resource names and identifiers +@description('Name of the AI Foundry account') +param aiAccountName string +@description('Name of the AI Search service') +param aiSearchName string +@description('Name of the storage account') +param storageName string +@description('Name of the Cosmos DB account') +param cosmosDBName string +@description('Name of the Vnet') +param vnetName string +@description('Name of the Customer subnet') +param peSubnetName string +@description('Suffix for unique resource names') +param suffix string + +@description('Resource Group name for existing Virtual Network (if different from current resource group)') +param vnetResourceGroupName string = resourceGroup().name + +@description('Subscription ID for Virtual Network') +param vnetSubscriptionId string = subscription().subscriptionId + +@description('Resource Group name for Storage Account') +param storageAccountResourceGroupName string = resourceGroup().name + +@description('Subscription ID for Storage account') +param storageAccountSubscriptionId string = subscription().subscriptionId + +@description('Subscription ID for AI Search service') +param aiSearchSubscriptionId string = subscription().subscriptionId + +@description('Resource Group name for AI Search service') +param aiSearchResourceGroupName string = resourceGroup().name + +@description('Subscription ID for Cosmos DB account') +param cosmosDBSubscriptionId string = subscription().subscriptionId + +@description('Resource group name for Cosmos DB account') +param cosmosDBResourceGroupName string = resourceGroup().name + +@description('Map of DNS zone FQDNs to resource group names. If provided, reference existing DNS zones in this resource group instead of creating them.') +param existingDnsZones object = { + 'privatelink.services.ai.azure.com': '' + 'privatelink.openai.azure.com': '' + 'privatelink.cognitiveservices.azure.com': '' + 'privatelink.search.windows.net': '' + 'privatelink.blob.${environment().suffixes.storage}': '' + 'privatelink.documents.azure.com': '' +} + +@description('Subscription ID where existing private DNS zones are located. Should be resolved to current subscription if empty.') +param dnsZonesSubscriptionId string + +// ---- Resource references ---- +resource aiAccount 'Microsoft.CognitiveServices/accounts@2023-05-01' existing = { + name: aiAccountName + scope: resourceGroup() +} + +resource aiSearch 'Microsoft.Search/searchServices@2023-11-01' existing = { + name: aiSearchName + scope: resourceGroup(aiSearchSubscriptionId, aiSearchResourceGroupName) +} + +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-05-01' existing = { + name: storageName + scope: resourceGroup(storageAccountSubscriptionId, storageAccountResourceGroupName) +} + +resource cosmosDBAccount 'Microsoft.DocumentDB/databaseAccounts@2024-11-15' existing = { + name: cosmosDBName + scope: resourceGroup(cosmosDBSubscriptionId, cosmosDBResourceGroupName) +} + +// Reference existing network resources +resource vnet 'Microsoft.Network/virtualNetworks@2024-05-01' existing = { + name: vnetName + scope: resourceGroup(vnetSubscriptionId, vnetResourceGroupName) +} +resource peSubnet 'Microsoft.Network/virtualNetworks/subnets@2024-05-01' existing = { + parent: vnet + name: peSubnetName +} + +/* -------------------------------------------- AI Foundry Account Private Endpoint -------------------------------------------- */ + +// Private endpoint for AI Services account +// - Creates network interface in customer hub subnet +// - Establishes private connection to AI Services account +resource aiAccountPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-05-01' = { + name: '${aiAccountName}-private-endpoint' + location: resourceGroup().location + properties: { + subnet: { id: peSubnet.id } // Deploy in customer hub subnet + privateLinkServiceConnections: [ + { + name: '${aiAccountName}-private-link-service-connection' + properties: { + privateLinkServiceId: aiAccount.id + groupIds: [ 'account' ] // Target AI Services account + } + } + ] + } +} + +/* -------------------------------------------- AI Search Private Endpoint -------------------------------------------- */ + +// Private endpoint for AI Search +// - Creates network interface in customer hub subnet +// - Establishes private connection to AI Search service +resource aiSearchPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-05-01' = { + name: '${aiSearchName}-private-endpoint' + location: resourceGroup().location + properties: { + subnet: { id: peSubnet.id } // Deploy in customer hub subnet + privateLinkServiceConnections: [ + { + name: '${aiSearchName}-private-link-service-connection' + properties: { + privateLinkServiceId: aiSearch.id + groupIds: [ 'searchService' ] // Target search service + } + } + ] + } +} + +/* -------------------------------------------- Storage Private Endpoint -------------------------------------------- */ + +// Private endpoint for Storage Account +// - Creates network interface in customer hub subnet +// - Establishes private connection to blob storage +resource storagePrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-05-01' = { + name: '${storageName}-private-endpoint' + location: resourceGroup().location + properties: { + subnet: { id: peSubnet.id } // Deploy in customer hub subnet + privateLinkServiceConnections: [ + { + name: '${storageName}-private-link-service-connection' + properties: { + privateLinkServiceId: storageAccount.id // Target blob storage + groupIds: [ 'blob' ] + } + } + ] + } +} + +/*--------------------------------------------- Cosmos DB Private Endpoint -------------------------------------*/ + +resource cosmosDBPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-05-01' = { + name: '${cosmosDBName}-private-endpoint' + location: resourceGroup().location + properties: { + subnet: { id: peSubnet.id } // Deploy in customer hub subnet + privateLinkServiceConnections: [ + { + name: '${cosmosDBName}-private-link-service-connection' + properties: { + privateLinkServiceId: cosmosDBAccount.id // Target Cosmos DB account + groupIds: [ 'Sql' ] + } + } + ] + } +} + +/* -------------------------------------------- Private DNS Zones -------------------------------------------- */ + +// Format: 1) Private DNS Zone +// 2) Link Private DNS Zone to VNet +// 3) Create DNS Zone Group for Private Endpoint + +// Private DNS Zone for AI Services (Account) +// 1) Enables custom DNS resolution for AI Services private endpoint + +var aiServicesDnsZoneName = 'privatelink.services.ai.azure.com' +var openAiDnsZoneName = 'privatelink.openai.azure.com' +var cognitiveServicesDnsZoneName = 'privatelink.cognitiveservices.azure.com' +var aiSearchDnsZoneName = 'privatelink.search.windows.net' +var storageDnsZoneName = 'privatelink.blob.${environment().suffixes.storage}' +var cosmosDBDnsZoneName = 'privatelink.documents.azure.com' + +// ---- DNS Zone Resource Group lookups ---- +var aiServicesDnsZoneRG = existingDnsZones[aiServicesDnsZoneName] +var openAiDnsZoneRG = existingDnsZones[openAiDnsZoneName] +var cognitiveServicesDnsZoneRG = existingDnsZones[cognitiveServicesDnsZoneName] +var aiSearchDnsZoneRG = existingDnsZones[aiSearchDnsZoneName] +var storageDnsZoneRG = existingDnsZones[storageDnsZoneName] +var cosmosDBDnsZoneRG = existingDnsZones[cosmosDBDnsZoneName] + +// ---- DNS Zone Resources and References ---- +resource aiServicesPrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (empty(aiServicesDnsZoneRG)) { + name: aiServicesDnsZoneName + location: 'global' +} + +// Reference existing private DNS zone if provided +resource existingAiServicesPrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = if (!empty(aiServicesDnsZoneRG)) { + name: aiServicesDnsZoneName + scope: resourceGroup(dnsZonesSubscriptionId, aiServicesDnsZoneRG) +} +//creating condition if user pass existing dns zones or not +var aiServicesDnsZoneId = empty(aiServicesDnsZoneRG) ? aiServicesPrivateDnsZone.id : existingAiServicesPrivateDnsZone.id + +resource openAiPrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (empty(openAiDnsZoneRG)) { + name: openAiDnsZoneName + location: 'global' +} + +// Reference existing private DNS zone if provided +resource existingOpenAiPrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = if (!empty(openAiDnsZoneRG)) { + name: openAiDnsZoneName + scope: resourceGroup(dnsZonesSubscriptionId, openAiDnsZoneRG) +} +//creating condition if user pass existing dns zones or not +var openAiDnsZoneId = empty(openAiDnsZoneRG) ? openAiPrivateDnsZone.id : existingOpenAiPrivateDnsZone.id + +resource cognitiveServicesPrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (empty(cognitiveServicesDnsZoneRG)) { + name: cognitiveServicesDnsZoneName + location: 'global' +} + +// Reference existing private DNS zone if provided +resource existingCognitiveServicesPrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = if (!empty(cognitiveServicesDnsZoneRG)) { + name: cognitiveServicesDnsZoneName + scope: resourceGroup(dnsZonesSubscriptionId, cognitiveServicesDnsZoneRG) +} +//creating condition if user pass existing dns zones or not +var cognitiveServicesDnsZoneId = empty(cognitiveServicesDnsZoneRG) ? cognitiveServicesPrivateDnsZone.id : existingCognitiveServicesPrivateDnsZone.id + +resource aiSearchPrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (empty(aiSearchDnsZoneRG)) { + name: aiSearchDnsZoneName + location: 'global' +} + +// Reference existing private DNS zone if provided +resource existingAiSearchPrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = if (!empty(aiSearchDnsZoneRG)) { + name: aiSearchDnsZoneName + scope: resourceGroup(dnsZonesSubscriptionId, aiSearchDnsZoneRG) +} +//creating condition if user pass existing dns zones or not +var aiSearchDnsZoneId = empty(aiSearchDnsZoneRG) ? aiSearchPrivateDnsZone.id : existingAiSearchPrivateDnsZone.id + +resource storagePrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (empty(storageDnsZoneRG)) { + name: storageDnsZoneName + location: 'global' +} + +// Reference existing private DNS zone if provided +resource existingStoragePrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = if (!empty(storageDnsZoneRG)) { + name: storageDnsZoneName + scope: resourceGroup(dnsZonesSubscriptionId, storageDnsZoneRG) +} +//creating condition if user pass existing dns zones or not +var storageDnsZoneId = empty(storageDnsZoneRG) ? storagePrivateDnsZone.id : existingStoragePrivateDnsZone.id + +resource cosmosDBPrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (empty(cosmosDBDnsZoneRG)) { + name: cosmosDBDnsZoneName + location: 'global' +} + +// Reference existing private DNS zone if provided +resource existingCosmosDBPrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = if (!empty(cosmosDBDnsZoneRG)) { + name: cosmosDBDnsZoneName + scope: resourceGroup(dnsZonesSubscriptionId, cosmosDBDnsZoneRG) +} +//creating condition if user pass existing dns zones or not +var cosmosDBDnsZoneId = empty(cosmosDBDnsZoneRG) ? cosmosDBPrivateDnsZone.id : existingCosmosDBPrivateDnsZone.id + +// ---- DNS VNet Links ---- +resource aiServicesLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (empty(aiServicesDnsZoneRG)) { + parent: aiServicesPrivateDnsZone + location: 'global' + name: 'aiServices-${suffix}-link' + properties: { + virtualNetwork: { id: vnet.id } + registrationEnabled: false + } +} +resource openAiLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (empty(openAiDnsZoneRG)) { + parent: openAiPrivateDnsZone + location: 'global' + name: 'aiServicesOpenAI-${suffix}-link' + properties: { + virtualNetwork: { id: vnet.id } + registrationEnabled: false + } +} +resource cognitiveServicesLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (empty(cognitiveServicesDnsZoneRG)) { + parent: cognitiveServicesPrivateDnsZone + location: 'global' + name: 'aiServicesCognitiveServices-${suffix}-link' + properties: { + virtualNetwork: { id: vnet.id } + registrationEnabled: false + } +} +resource aiSearchLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (empty(aiSearchDnsZoneRG)) { + parent: aiSearchPrivateDnsZone + location: 'global' + name: 'aiSearch-${suffix}-link' + properties: { + virtualNetwork: { id: vnet.id } + registrationEnabled: false + } +} +resource storageLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (empty(storageDnsZoneRG)) { + parent: storagePrivateDnsZone + location: 'global' + name: 'storage-${suffix}-link' + properties: { + virtualNetwork: { id: vnet.id } + registrationEnabled: false + } +} +resource cosmosDBLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (empty(cosmosDBDnsZoneRG)) { + parent: cosmosDBPrivateDnsZone + location: 'global' + name: 'cosmosDB-${suffix}-link' + properties: { + virtualNetwork: { id: vnet.id } + registrationEnabled: false + } +} + +// ---- DNS Zone Groups ---- +resource aiServicesDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2024-05-01' = { + parent: aiAccountPrivateEndpoint + name: '${aiAccountName}-dns-group' + properties: { + privateDnsZoneConfigs: [ + { name: '${aiAccountName}-dns-aiserv-config', properties: { privateDnsZoneId: aiServicesDnsZoneId } } + { name: '${aiAccountName}-dns-openai-config', properties: { privateDnsZoneId: openAiDnsZoneId } } + { name: '${aiAccountName}-dns-cogserv-config', properties: { privateDnsZoneId: cognitiveServicesDnsZoneId } } + ] + } + dependsOn: [ + empty(aiServicesDnsZoneRG) ? aiServicesLink : null + empty(openAiDnsZoneRG) ? openAiLink : null + empty(cognitiveServicesDnsZoneRG) ? cognitiveServicesLink : null + ] +} +resource aiSearchDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2024-05-01' = { + parent: aiSearchPrivateEndpoint + name: '${aiSearchName}-dns-group' + properties: { + privateDnsZoneConfigs: [ + { name: '${aiSearchName}-dns-config', properties: { privateDnsZoneId: aiSearchDnsZoneId } } + ] + } + dependsOn: [ + empty(aiSearchDnsZoneRG) ? aiSearchLink : null + ] +} +resource storageDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2024-05-01' = { + parent: storagePrivateEndpoint + name: '${storageName}-dns-group' + properties: { + privateDnsZoneConfigs: [ + { name: '${storageName}-dns-config', properties: { privateDnsZoneId: storageDnsZoneId } } + ] + } + dependsOn: [ + empty(storageDnsZoneRG) ? storageLink : null + ] +} +resource cosmosDBDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2024-05-01' = { + parent: cosmosDBPrivateEndpoint + name: '${cosmosDBName}-dns-group' + properties: { + privateDnsZoneConfigs: [ + { name: '${cosmosDBName}-dns-config', properties: { privateDnsZoneId: cosmosDBDnsZoneId } } + ] + } + dependsOn: [ + empty(cosmosDBDnsZoneRG) ? cosmosDBLink : null + ] +} diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/standard-dependent-resources.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/standard-dependent-resources.bicep new file mode 100644 index 000000000..c4c9fb657 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/standard-dependent-resources.bicep @@ -0,0 +1,148 @@ +// Creates Azure dependent resources for Azure AI Agent Service standard agent setup + +@description('Azure region of the deployment') +param location string + +// @description('The name of the Key Vault') +// param keyvaultName string + +@description('The name of the AI Search resource') +param aiSearchName string + +@description('Name of the storage account') +param azureStorageName string + +@description('Name of the new Cosmos DB account') +param cosmosDBName string + +@description('The AI Search Service full ARM Resource ID. This is an optional field, and if not provided, the resource will be created.') +param aiSearchResourceId string + +@description('The AI Storage Account full ARM Resource ID. This is an optional field, and if not provided, the resource will be created.') +param azureStorageAccountResourceId string + +@description('The Cosmos DB Account full ARM Resource ID. This is an optional field, and if not provided, the resource will be created.') +param cosmosDBResourceId string + +// param aiServiceExists bool +param aiSearchExists bool +param azureStorageExists bool +param cosmosDBExists bool + +var cosmosParts = split(cosmosDBResourceId, '/') + +resource existingCosmosDB 'Microsoft.DocumentDB/databaseAccounts@2024-11-15' existing = if (cosmosDBExists) { + name: cosmosParts[8] + scope: resourceGroup(cosmosParts[2], cosmosParts[4]) +} + +// CosmosDB creation + +var canaryRegions = ['eastus2euap', 'centraluseuap'] +var cosmosDbRegion = contains(canaryRegions, location) ? 'westus' : location +resource cosmosDB 'Microsoft.DocumentDB/databaseAccounts@2024-11-15' = if(!cosmosDBExists) { + name: cosmosDBName + location: cosmosDbRegion + kind: 'GlobalDocumentDB' + properties: { + consistencyPolicy: { + defaultConsistencyLevel: 'Session' + } + disableLocalAuth: true + enableAutomaticFailover: false + enableMultipleWriteLocations: false + publicNetworkAccess: 'Disabled' + enableFreeTier: false + locations: [ + { + locationName: location + failoverPriority: 0 + isZoneRedundant: false + } + ] + databaseAccountOfferType: 'Standard' + } +} + +var acsParts = split(aiSearchResourceId, '/') + +resource existingSearchService 'Microsoft.Search/searchServices@2024-06-01-preview' existing = if (aiSearchExists) { + name: acsParts[8] + scope: resourceGroup(acsParts[2], acsParts[4]) +} + +// AI Search creation + +resource aiSearch 'Microsoft.Search/searchServices@2024-06-01-preview' = if(!aiSearchExists) { + name: aiSearchName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + disableLocalAuth: false + authOptions: { aadOrApiKey: { aadAuthFailureMode: 'http401WithBearerChallenge'}} + encryptionWithCmk: { + enforcement: 'Unspecified' + } + hostingMode: 'default' + partitionCount: 1 + publicNetworkAccess: 'disabled' + replicaCount: 1 + semanticSearch: 'disabled' + networkRuleSet: { + bypass: 'None' + ipRules: [] + } + } + sku: { + name: 'standard' + } +} + +var azureStorageParts = split(azureStorageAccountResourceId, '/') + +resource existingAzureStorageAccount 'Microsoft.Storage/storageAccounts@2023-05-01' existing = if (azureStorageExists) { + name: azureStorageParts[8] + scope: resourceGroup(azureStorageParts[2], azureStorageParts[4]) +} + +// Some regions doesn't support Standard Zone-Redundant storage, need to use Geo-redundant storage +param noZRSRegions array = ['southindia', 'westus'] +param sku object = contains(noZRSRegions, location) ? { name: 'Standard_GRS' } : { name: 'Standard_ZRS' } + +// Storage creation + +resource storage 'Microsoft.Storage/storageAccounts@2023-05-01' = if(!azureStorageExists) { + name: azureStorageName + location: location + kind: 'StorageV2' + sku: sku + properties: { + minimumTlsVersion: 'TLS1_2' + allowBlobPublicAccess: false + publicNetworkAccess: 'Disabled' + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + virtualNetworkRules: [] + } + allowSharedKeyAccess: false + } +} + +output aiSearchName string = aiSearchExists ? existingSearchService.name : aiSearch.name +output aiSearchID string = aiSearchExists ? existingSearchService.id : aiSearch.id +output aiSearchServiceResourceGroupName string = aiSearchExists ? acsParts[4] : resourceGroup().name +output aiSearchServiceSubscriptionId string = aiSearchExists ? acsParts[2] : subscription().subscriptionId + +output azureStorageName string = azureStorageExists ? existingAzureStorageAccount.name : storage.name +output azureStorageId string = azureStorageExists ? existingAzureStorageAccount.id : storage.id +output azureStorageResourceGroupName string = azureStorageExists ? azureStorageParts[4] : resourceGroup().name +output azureStorageSubscriptionId string = azureStorageExists ? azureStorageParts[2] : subscription().subscriptionId + +output cosmosDBName string = cosmosDBExists ? existingCosmosDB.name : cosmosDB.name +output cosmosDBId string = cosmosDBExists ? existingCosmosDB.id : cosmosDB.id +output cosmosDBResourceGroupName string = cosmosDBExists ? cosmosParts[4] : resourceGroup().name +output cosmosDBSubscriptionId string = cosmosDBExists ? cosmosParts[2] : subscription().subscriptionId +// output keyvaultId string = keyVault.id diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/subnet.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/subnet.bicep new file mode 100644 index 000000000..bf81553d8 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/subnet.bicep @@ -0,0 +1,22 @@ +@description('Name of the virtual network') +param vnetName string + +@description('Name of the subnet') +param subnetName string + +@description('Address prefix for the subnet') +param addressPrefix string + +@description('Array of subnet delegations') +param delegations array = [] + +resource subnet 'Microsoft.Network/virtualNetworks/subnets@2024-05-01' = { + name: '${vnetName}/${subnetName}' + properties: { + addressPrefix: addressPrefix + delegations: delegations + } +} + +output subnetId string = subnet.id +output subnetName string = subnetName diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/validate-existing-resources.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/validate-existing-resources.bicep new file mode 100644 index 000000000..0ba9c7308 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/validate-existing-resources.bicep @@ -0,0 +1,94 @@ +// @description('Resource ID of the AI Service Account. ') +// param aiServiceAccountResourceId string + +@description('Resource ID of the AI Search Service.') +param aiSearchResourceId string + +@description('Resource ID of the Azure Storage Account.') +param azureStorageAccountResourceId string + +@description('ResourceId of Cosmos DB Account') +param azureCosmosDBAccountResourceId string + +// Check if existing resources have been passed in +var storagePassedIn = azureStorageAccountResourceId != '' +var searchPassedIn = aiSearchResourceId != '' +var cosmosPassedIn = azureCosmosDBAccountResourceId != '' + +var storageParts = split(azureStorageAccountResourceId, '/') +var azureStorageSubscriptionId = storagePassedIn && length(storageParts) > 2 ? storageParts[2] : subscription().subscriptionId +var azureStorageResourceGroupName = storagePassedIn && length(storageParts) > 4 ? storageParts[4] : resourceGroup().name + +var acsParts = split(aiSearchResourceId, '/') +var aiSearchServiceSubscriptionId = searchPassedIn && length(acsParts) > 2 ? acsParts[2] : subscription().subscriptionId +var aiSearchServiceResourceGroupName = searchPassedIn && length(acsParts) > 4 ? acsParts[4] : resourceGroup().name + +var cosmosParts = split(azureCosmosDBAccountResourceId, '/') +var cosmosDBSubscriptionId = cosmosPassedIn && length(cosmosParts) > 2 ? cosmosParts[2] : subscription().subscriptionId +var cosmosDBResourceGroupName = cosmosPassedIn && length(cosmosParts) > 4 ? cosmosParts[4] : resourceGroup().name + +// Validate AI Search +resource aiSearch 'Microsoft.Search/searchServices@2024-06-01-preview' existing = if (searchPassedIn) { + name: last(split(aiSearchResourceId, '/')) + scope: resourceGroup(aiSearchServiceSubscriptionId, aiSearchServiceResourceGroupName) +} + +// Validate Cosmos DB Account +resource cosmosDBAccount 'Microsoft.DocumentDB/databaseAccounts@2024-12-01-preview' existing = if (cosmosPassedIn) { + name: last(split(azureCosmosDBAccountResourceId, '/')) + scope: resourceGroup(cosmosDBSubscriptionId,cosmosDBResourceGroupName) +} + +// Validate Storage Account +resource azureStorageAccount 'Microsoft.Storage/storageAccounts@2024-01-01' existing = if (storagePassedIn) { + name: last(split(azureStorageAccountResourceId, '/')) + scope: resourceGroup(azureStorageSubscriptionId,azureStorageResourceGroupName) +} + +// output aiServiceExists bool = aiServicesPassedIn && (aiServiceAccount.name == aiServiceParts[8]) +output aiSearchExists bool = searchPassedIn && (aiSearch.name == acsParts[8]) +output cosmosDBExists bool = cosmosPassedIn && (cosmosDBAccount.name == cosmosParts[8]) +output azureStorageExists bool = storagePassedIn && (azureStorageAccount.name == storageParts[8]) + +output aiSearchServiceSubscriptionId string = aiSearchServiceSubscriptionId +output aiSearchServiceResourceGroupName string = aiSearchServiceResourceGroupName + +output cosmosDBSubscriptionId string = cosmosDBSubscriptionId +output cosmosDBResourceGroupName string = cosmosDBResourceGroupName + +output azureStorageSubscriptionId string = azureStorageSubscriptionId +output azureStorageResourceGroupName string = azureStorageResourceGroupName + +// Adding DNS Zone Check + +@description('Object mapping DNS zone names to their resource group, or empty string to indicate creation') +param existingDnsZones object + +@description('Subscription ID where existing private DNS zones are located. Should be resolved to current subscription if empty.') +param dnsZonesSubscriptionId string + +@description('List of private DNS zone names to validate') +param dnsZoneNames array + +var dnsZoneTypes = [ + 'Microsoft.Network/privateDnsZones' +] + +// Output whether each DNS zone exists +output dnsZoneExists array = [ + for zoneName in dnsZoneNames: { + name: zoneName + exists: !empty(existingDnsZones[zoneName]) + } +] + +/* +// Helper function to check existence +function resourceExists(resourceType: string, name: string, rg: string): bool { + // Use the existing resource reference to check + var res = existing resource dnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: name + scope: resourceGroup(rg) + } + return !empty(res.id) +}*/ diff --git a/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/vnet.bicep b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/vnet.bicep new file mode 100644 index 000000000..d5b8db277 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/33-customer-managed-keys-private-network-standard-agent/modules-network-secured/vnet.bicep @@ -0,0 +1,83 @@ +/* +Virtual Network Module +This module deploys the core network infrastructure with security controls: + +1. Address Space: + - VNet CIDR: 172.16.0.0/16 OR 192.168.0.0/16 + - Agents Subnet: 172.16.0.0/24 OR 192.168.0.0/24 + - Private Endpoint Subnet: 172.16.101.0/24 OR 192.168.1.0/24 + +2. Security Features: + - Network isolation + - Subnet delegation + - Private endpoint subnet +*/ + +@description('Azure region for the deployment') +param location string + +@description('The name of the virtual network') +param vnetName string = 'agents-vnet-test' + +@description('The name of Agents Subnet') +param agentSubnetName string = 'agent-subnet' + +@description('The name of Hub subnet') +param peSubnetName string = 'pe-subnet' + + +@description('Address space for the VNet') +param vnetAddressPrefix string = '' + +@description('Address prefix for the agent subnet') +param agentSubnetPrefix string = '' + +@description('Address prefix for the private endpoint subnet') +param peSubnetPrefix string = '' +var defaultVnetAddressPrefix = '192.168.0.0/16' +var vnetAddress = empty(vnetAddressPrefix) ? defaultVnetAddressPrefix : vnetAddressPrefix +var agentSubnet = empty(agentSubnetPrefix) ? cidrSubnet(vnetAddress, 24, 0) : agentSubnetPrefix +var peSubnet = empty(peSubnetPrefix) ? cidrSubnet(vnetAddress, 24, 1) : peSubnetPrefix + +resource virtualNetwork 'Microsoft.Network/virtualNetworks@2024-05-01' = { + name: vnetName + location: location + properties: { + addressSpace: { + addressPrefixes: [ + vnetAddress + ] + } + subnets: [ + { + name: agentSubnetName + properties: { + addressPrefix: agentSubnet + delegations: [ + { + name: 'Microsoft.app/environments' + properties: { + serviceName: 'Microsoft.App/environments' + } + } + ] + } + } + { + name: peSubnetName + properties: { + addressPrefix: peSubnet + } + } + ] + } +} +// Output variables +output peSubnetName string = peSubnetName +output agentSubnetName string = agentSubnetName +output agentSubnetId string = '${virtualNetwork.id}/subnets/${agentSubnetName}' +output peSubnetId string = '${virtualNetwork.id}/subnets/${peSubnetName}' +output virtualNetworkName string = virtualNetwork.name +output virtualNetworkId string = virtualNetwork.id +output virtualNetworkResourceGroup string = resourceGroup().name +output virtualNetworkSubscriptionId string = subscription().subscriptionId