diff --git a/.cspell/general-technical.txt b/.cspell/general-technical.txt index 01b642e2..a4da2b56 100644 --- a/.cspell/general-technical.txt +++ b/.cspell/general-technical.txt @@ -1554,6 +1554,9 @@ issecret LASTEXITCODE showvariable Hashtable +jqlang +choco +Farah # Camera & Video Equipment Vendors Hikvision diff --git a/.cspell/iot-operations.txt b/.cspell/iot-operations.txt index 7514db63..79a84177 100644 --- a/.cspell/iot-operations.txt +++ b/.cspell/iot-operations.txt @@ -25,6 +25,9 @@ MCAP edgevolume storageaccountendpoint arccontainerstorage +cainjector +certmanager +containerstorage inotify unregistration customendpoint @@ -83,6 +86,7 @@ thermodyanmics fanout opcfoundation secretproviderclass +storagepools connectorinstance connectorinstances connectortemplate diff --git a/.cspell/microsoft-sample-companies.txt b/.cspell/microsoft-sample-companies.txt index e55d9a58..1b92ed49 100644 --- a/.cspell/microsoft-sample-companies.txt +++ b/.cspell/microsoft-sample-companies.txt @@ -29,3 +29,5 @@ Northwind myorg myorga myorgb +Lakeshore +lakeshore diff --git a/.cspell/project-specific.txt b/.cspell/project-specific.txt index 9a3ff2d0..c2ab700b 100644 --- a/.cspell/project-specific.txt +++ b/.cspell/project-specific.txt @@ -109,6 +109,23 @@ wowza steveyegge dorny nobuild +AADSTS +dimproducts +dimstore +factsales +jointable +tablename +TMDL +XMLA +CORAX +corax +Fanuc +robotgroup +roboticsystem +roboticenvironment +positionmeasure +orientationmeasure +posemeasure fixturing mqttui strobing diff --git a/blueprints/azure-local/terraform/README.md b/blueprints/azure-local/terraform/README.md index 4c920ff4..7da2d720 100644 --- a/blueprints/azure-local/terraform/README.md +++ b/blueprints/azure-local/terraform/README.md @@ -35,6 +35,7 @@ Deploys the cloud and edge resources required to run Azure IoT Operations on an | cloud\_observability | ../../../src/000-cloud/020-observability/terraform | n/a | | cloud\_resource\_group | ../../../src/000-cloud/000-resource-group/terraform | n/a | | cloud\_security\_identity | ../../../src/000-cloud/010-security-identity/terraform | n/a | +| edge\_arc\_extensions | ../../../src/100-edge/109-arc-extensions/terraform | n/a | | edge\_assets | ../../../src/100-edge/111-assets/terraform | n/a | | edge\_iot\_ops | ../../../src/100-edge/110-iot-ops/terraform | n/a | | edge\_messaging | ../../../src/100-edge/130-messaging/terraform | n/a | diff --git a/blueprints/azure-local/terraform/main.tf b/blueprints/azure-local/terraform/main.tf index 9b390b1f..895dbc59 100644 --- a/blueprints/azure-local/terraform/main.tf +++ b/blueprints/azure-local/terraform/main.tf @@ -124,10 +124,18 @@ module "azure_local_host" { aad_profile = var.azure_local_aad_profile } +module "edge_arc_extensions" { + source = "../../../src/100-edge/109-arc-extensions/terraform" + + depends_on = [module.azure_local_host] + + arc_connected_cluster = module.azure_local_host +} + module "edge_iot_ops" { source = "../../../src/100-edge/110-iot-ops/terraform" - depends_on = [module.azure_local_host, module.cloud_security_identity] + depends_on = [module.edge_arc_extensions, module.cloud_security_identity] adr_schema_registry = module.cloud_data.schema_registry adr_namespace = module.cloud_data.adr_namespace diff --git a/blueprints/dual-peered-single-node-cluster/terraform/README.md b/blueprints/dual-peered-single-node-cluster/terraform/README.md index 3425a3c3..246dd560 100644 --- a/blueprints/dual-peered-single-node-cluster/terraform/README.md +++ b/blueprints/dual-peered-single-node-cluster/terraform/README.md @@ -43,6 +43,7 @@ Each cluster operates independently but can communicate through the peered virtu | cluster\_a\_cloud\_resource\_group | ../../../src/000-cloud/000-resource-group/terraform | n/a | | cluster\_a\_cloud\_security\_identity | ../../../src/000-cloud/010-security-identity/terraform | n/a | | cluster\_a\_cloud\_vm\_host | ../../../src/000-cloud/051-vm-host/terraform | n/a | +| cluster\_a\_edge\_arc\_extensions | ../../../src/100-edge/109-arc-extensions/terraform | n/a | | cluster\_a\_edge\_assets | ../../../src/100-edge/111-assets/terraform | n/a | | cluster\_a\_edge\_cncf\_cluster | ../../../src/100-edge/100-cncf-cluster/terraform | n/a | | cluster\_a\_edge\_iot\_ops | ../../../src/100-edge/110-iot-ops/terraform | n/a | @@ -57,6 +58,7 @@ Each cluster operates independently but can communicate through the peered virtu | cluster\_b\_cloud\_resource\_group | ../../../src/000-cloud/000-resource-group/terraform | n/a | | cluster\_b\_cloud\_security\_identity | ../../../src/000-cloud/010-security-identity/terraform | n/a | | cluster\_b\_cloud\_vm\_host | ../../../src/000-cloud/051-vm-host/terraform | n/a | +| cluster\_b\_edge\_arc\_extensions | ../../../src/100-edge/109-arc-extensions/terraform | n/a | | cluster\_b\_edge\_assets | ../../../src/100-edge/111-assets/terraform | n/a | | cluster\_b\_edge\_cncf\_cluster | ../../../src/100-edge/100-cncf-cluster/terraform | n/a | | cluster\_b\_edge\_iot\_ops | ../../../src/100-edge/110-iot-ops/terraform | n/a | diff --git a/blueprints/dual-peered-single-node-cluster/terraform/main.tf b/blueprints/dual-peered-single-node-cluster/terraform/main.tf index 45d75afa..2e49efb3 100644 --- a/blueprints/dual-peered-single-node-cluster/terraform/main.tf +++ b/blueprints/dual-peered-single-node-cluster/terraform/main.tf @@ -192,10 +192,18 @@ module "cluster_a_edge_cncf_cluster" { key_vault = module.cluster_a_cloud_security_identity.key_vault } +module "cluster_a_edge_arc_extensions" { + source = "../../../src/100-edge/109-arc-extensions/terraform" + + depends_on = [module.cluster_a_edge_cncf_cluster] + + arc_connected_cluster = module.cluster_a_edge_cncf_cluster.arc_connected_cluster +} + module "cluster_a_edge_iot_ops" { source = "../../../src/100-edge/110-iot-ops/terraform" - depends_on = [module.cluster_a_edge_cncf_cluster] + depends_on = [module.cluster_a_edge_arc_extensions] adr_schema_registry = module.cluster_a_cloud_data.schema_registry resource_group = module.cluster_a_cloud_resource_group.resource_group @@ -437,10 +445,18 @@ module "cluster_b_edge_cncf_cluster" { key_vault = module.cluster_b_cloud_security_identity.key_vault } +module "cluster_b_edge_arc_extensions" { + source = "../../../src/100-edge/109-arc-extensions/terraform" + + depends_on = [module.cluster_b_edge_cncf_cluster] + + arc_connected_cluster = module.cluster_b_edge_cncf_cluster.arc_connected_cluster +} + module "cluster_b_edge_iot_ops" { source = "../../../src/100-edge/110-iot-ops/terraform" - depends_on = [module.cluster_b_edge_cncf_cluster] + depends_on = [module.cluster_b_edge_arc_extensions] adr_schema_registry = module.cluster_b_cloud_data.schema_registry resource_group = module.cluster_b_cloud_resource_group.resource_group diff --git a/blueprints/full-multi-node-cluster/bicep/README.md b/blueprints/full-multi-node-cluster/bicep/README.md index fa46de6b..13dae4f4 100644 --- a/blueprints/full-multi-node-cluster/bicep/README.md +++ b/blueprints/full-multi-node-cluster/bicep/README.md @@ -70,6 +70,7 @@ Deploys a complete end-to-end environment for Azure IoT Operations on a multi-no | cloudAiFoundry | `Microsoft.Resources/deployments` | 2025-04-01 | | cloudKubernetes | `Microsoft.Resources/deployments` | 2025-04-01 | | edgeCncfCluster | `Microsoft.Resources/deployments` | 2025-04-01 | +| edgeArcExtensions | `Microsoft.Resources/deployments` | 2025-04-01 | | edgeIotOps | `Microsoft.Resources/deployments` | 2025-04-01 | | edgeAssets | `Microsoft.Resources/deployments` | 2025-04-01 | | edgeObservability | `Microsoft.Resources/deployments` | 2025-04-01 | @@ -91,6 +92,7 @@ Deploys a complete end-to-end environment for Azure IoT Operations on a multi-no | cloudAiFoundry | Deploys Microsoft Foundry account with optional projects, model deployments, RAI policies, and private endpoint support. | | cloudKubernetes | Deploys optionally Azure Kubernetes Service (AKS) resources. | | edgeCncfCluster | This module provisions and deploys automation scripts to a VM host that create and configure a K3s Kubernetes cluster with Arc connectivity.
The scripts handle primary and secondary node(s) setup, cluster administration, workload identity enablement, and installation of required Azure Arc extensions. | +| edgeArcExtensions | Deploys foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA). | | edgeIotOps | Deploys Azure IoT Operations extensions, instances, and configurations on Azure Arc-enabled Kubernetes clusters. | | edgeAssets | Deploys Kubernetes asset definitions to a connected cluster using the namespaced Device Registry model. This component facilitates the management of devices and assets within ADR namespaces. | | edgeObservability | Deploys observability resources including cluster extensions for metrics and logs collection, and rule groups for monitoring. | @@ -416,7 +418,7 @@ Provisions virtual machines and networking infrastructure for hosting Azure IoT ### cloudVpnGateway -Creates a VPN Gateway with Point-to-Site and optional Site-to-Site connectivity. +Creates a VPN Gateway with Point-to-Site and optional Site-to-Site connectivity. Ths component currently only supports Azure AD (Entra ID) authentication for Point-to-Site VPN connections. #### Parameters for cloudVpnGateway @@ -591,7 +593,7 @@ Deploys optionally Azure Kubernetes Service (AKS) resources. ### edgeCncfCluster -This module provisions and deploys automation scripts to a VM host that create and configure a K3s Kubernetes cluster with Arc connectivity. +This module provisions and deploys automation scripts to a VM host that create and configure a K3s Kubernetes cluster with Arc connectivity. The scripts handle primary and secondary node(s) setup, cluster administration, workload identity enablement, and installation of required Azure Arc extensions. #### Parameters for edgeCncfCluster @@ -647,57 +649,83 @@ The scripts handle primary and secondary node(s) setup, cluster administration, | clusterServerScriptSecretShowCommand | `string` | The AZ CLI command to get the cluster server script from Key Vault | | clusterNodeScriptSecretShowCommand | `string` | The AZ CLI command to get the cluster node script from Key Vault | +### edgeArcExtensions + +Deploys foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA). + +#### Parameters for edgeArcExtensions + +| Name | Description | Type | Default | Required | +|:------------------------|:----------------------------------------------------------------------|:------------------------------------------------------|:----------------------------------------------------|:---------| +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | +| certManagerConfig | The settings for the cert-manager Extension. | `[_1.CertManagerExtension](#user-defined-types)` | [variables('_1.certManagerExtensionDefaults')] | no | +| containerStorageConfig | The settings for the Azure Container Storage for Azure Arc Extension. | `[_1.ContainerStorageExtension](#user-defined-types)` | [variables('_1.containerStorageExtensionDefaults')] | no | + +#### Resources for edgeArcExtensions + +| Name | Type | API Version | +|:-----------------|:-----------------------------------------------|:------------| +| aioCertManager | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | +| containerStorage | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | + +#### Outputs for edgeArcExtensions + +| Name | Type | Description | +|:------------------------------|:---------|:----------------------------------------------------------| +| certManagerExtensionId | `string` | The resource ID of the cert-manager extension. | +| certManagerExtensionName | `string` | The name of the cert-manager extension. | +| containerStorageExtensionId | `string` | The resource ID of the Azure Container Storage extension. | +| containerStorageExtensionName | `string` | The name of the Azure Container Storage extension. | + ### edgeIotOps Deploys Azure IoT Operations extensions, instances, and configurations on Azure Arc-enabled Kubernetes clusters. #### Parameters for edgeIotOps -| Name | Description | Type | Default | Required | -|:------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------|:---------| -| common | The common component configuration. | `[_2.Common](#user-defined-types)` | n/a | yes | -| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | -| containerStorageConfig | The settings for the Azure Container Store for Azure Arc Extension. | `[_1.ContainerStorageExtension](#user-defined-types)` | [variables('_1.containerStorageExtensionDefaults')] | no | -| aioCertManagerConfig | The settings for the Azure IoT Operations Platform Extension. | `[_1.AioCertManagerExtension](#user-defined-types)` | [variables('_1.aioCertManagerExtensionDefaults')] | no | -| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | [variables('_1.secretStoreExtensionDefaults')] | no | -| shouldInitAio | Whether to deploy the Azure IoT Operations initial connected cluster resources, Secret Sync, ACSA, OSM, AIO Platform. | `bool` | `true` | no | -| aioIdentityName | The name of the User Assigned Managed Identity for Azure IoT Operations. | `string` | n/a | yes | -| aioExtensionConfig | The settings for the Azure IoT Operations Extension. | `[_1.AioExtension](#user-defined-types)` | [variables('_1.aioExtensionDefaults')] | no | -| aioFeatures | AIO Instance features. | `[_1.AioFeatures](#user-defined-types)` | n/a | no | -| aioInstanceName | The name for the Azure IoT Operations Instance resource. | `string` | [format('{0}-ops-instance', parameters('arcConnectedClusterName'))] | no | -| aioDataFlowInstanceConfig | The settings for Azure IoT Operations Data Flow Instances. | `[_1.AioDataFlowInstance](#user-defined-types)` | [variables('_1.aioDataFlowInstanceDefaults')] | no | -| aioMqBrokerConfig | The settings for the Azure IoT Operations MQ Broker. | `[_1.AioMqBroker](#user-defined-types)` | [variables('_1.aioMqBrokerDefaults')] | no | -| brokerListenerAnonymousConfig | Configuration for the insecure anonymous AIO MQ Broker Listener. | `[_1.AioMqBrokerAnonymous](#user-defined-types)` | [variables('_1.aioMqBrokerAnonymousDefaults')] | no | -| configurationSettingsOverride | Optional configuration settings to override default IoT Operations extension configuration. Use the same key names as the az iot ops --ops-config parameter. | `object` | {} | no | -| schemaRegistryName | The resource name for the ADR Schema Registry for Azure IoT Operations. | `string` | n/a | yes | -| adrNamespaceName | The resource name for the ADR Namespace for Azure IoT Operations. | `string` | n/a | no | -| shouldDeployAio | Whether to deploy an Azure IoT Operations Instance and all of its required components into the connected cluster. | `bool` | `true` | no | -| shouldDeployResourceSyncRules | Whether or not to deploy the Custom Locations Resource Sync Rules for the Azure IoT Operations resources. | `bool` | `true` | no | -| shouldCreateAnonymousBrokerListener | Whether to enable an insecure anonymous AIO MQ Broker Listener. (Should only be used for dev or test environments) | `bool` | `false` | no | -| shouldEnableOtelCollector | Whether or not to enable the Open Telemetry Collector for Azure IoT Operations. | `bool` | `true` | no | -| shouldEnableOpcUaSimulator | Whether or not to enable the OPC UA Simulator for Azure IoT Operations. | `bool` | `true` | no | -| shouldEnableAkriRestConnector | Deploy Akri REST HTTP Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriMediaConnector | Deploy Akri Media Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriOnvifConnector | Deploy Akri ONVIF Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriSseConnector | Deploy Akri SSE Connector template to the IoT Operations instance. | `bool` | `false` | no | -| customAkriConnectors | List of custom Akri connector templates with user-defined endpoint types and container images. | `array` | [] | no | -| akriMqttSharedConfig | Shared MQTT connection configuration for all Akri connectors. | `[_1.AkriMqttConfig](#user-defined-types)` | {'host': 'aio-broker:18883', 'audience': 'aio-internal', 'caConfigmap': 'azure-iot-operations-aio-ca-trust-bundle'} | no | -| customLocationName | The name for the Custom Locations resource. | `string` | [format('{0}-cl', parameters('arcConnectedClusterName'))] | no | -| additionalClusterExtensionIds | Additional cluster extension IDs to include in the custom location. (Appended to the default Secret Store and IoT Operations extension IDs) | `array` | [] | no | -| trustIssuerSettings | The trust issuer settings for Customer Managed Azure IoT Operations Settings. | `[_1.TrustIssuerConfig](#user-defined-types)` | {'trustSource': 'SelfSigned'} | no | -| sseKeyVaultName | The name of the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | n/a | yes | -| sseIdentityName | The name of the User Assigned Managed Identity for Secret Sync. | `string` | n/a | yes | -| sseKeyVaultResourceGroupName | The name of the Resource Group for the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | [resourceGroup().name] | no | -| shouldAssignSseKeyVaultRoles | Whether to assign roles for Key Vault to the provided Secret Sync Identity. | `bool` | `true` | no | -| shouldAssignDeployIdentityRoles | Whether to assign roles to the deploy identity. | `bool` | [not(empty(parameters('deployIdentityName')))] | no | -| deployIdentityName | The resource name for a managed identity that will be given deployment admin permissions. | `string` | n/a | no | -| shouldDeployAioDeploymentScripts | Whether to deploy DeploymentScripts for Azure IoT Operations. | `bool` | `false` | no | -| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | [parameters('sseKeyVaultName')] | no | -| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [parameters('sseKeyVaultResourceGroupName')] | no | -| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | -| deploymentScriptsSecretNamePrefix | The prefix used with constructing the secret name that will have the deployment script. | `string` | [format('{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | -| shouldAddDeployScriptsToKeyVault | Whether to add the deploy scripts for DeploymentScripts to Key Vault as secrets. (Required for DeploymentScripts) | `bool` | `false` | no | -| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | +| Name | Description | Type | Default | Required | +|:------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------|:---------| +| common | The common component configuration. | `[_2.Common](#user-defined-types)` | n/a | yes | +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | +| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | [variables('_1.secretStoreExtensionDefaults')] | no | +| shouldInitAio | Whether to deploy the Azure IoT Operations initial connected cluster resources, Secret Sync, ACSA, OSM, AIO Platform. | `bool` | `true` | no | +| aioIdentityName | The name of the User Assigned Managed Identity for Azure IoT Operations. | `string` | n/a | yes | +| aioExtensionConfig | The settings for the Azure IoT Operations Extension. | `[_1.AioExtension](#user-defined-types)` | [variables('_1.aioExtensionDefaults')] | no | +| aioFeatures | AIO Instance features. | `[_1.AioFeatures](#user-defined-types)` | n/a | no | +| aioInstanceName | The name for the Azure IoT Operations Instance resource. | `string` | [format('{0}-ops-instance', parameters('arcConnectedClusterName'))] | no | +| aioDataFlowInstanceConfig | The settings for Azure IoT Operations Data Flow Instances. | `[_1.AioDataFlowInstance](#user-defined-types)` | [variables('_1.aioDataFlowInstanceDefaults')] | no | +| aioMqBrokerConfig | The settings for the Azure IoT Operations MQ Broker. | `[_1.AioMqBroker](#user-defined-types)` | [variables('_1.aioMqBrokerDefaults')] | no | +| brokerListenerAnonymousConfig | Configuration for the insecure anonymous AIO MQ Broker Listener. | `[_1.AioMqBrokerAnonymous](#user-defined-types)` | [variables('_1.aioMqBrokerAnonymousDefaults')] | no | +| configurationSettingsOverride | Optional configuration settings to override default IoT Operations extension configuration. Use the same key names as the az iot ops --ops-config parameter. | `object` | {} | no | +| schemaRegistryName | The resource name for the ADR Schema Registry for Azure IoT Operations. | `string` | n/a | yes | +| adrNamespaceName | The resource name for the ADR Namespace for Azure IoT Operations. | `string` | n/a | no | +| shouldDeployAio | Whether to deploy an Azure IoT Operations Instance and all of its required components into the connected cluster. | `bool` | `true` | no | +| shouldDeployResourceSyncRules | Whether or not to deploy the Custom Locations Resource Sync Rules for the Azure IoT Operations resources. | `bool` | `true` | no | +| shouldCreateAnonymousBrokerListener | Whether to enable an insecure anonymous AIO MQ Broker Listener. (Should only be used for dev or test environments) | `bool` | `false` | no | +| shouldEnableOtelCollector | Whether or not to enable the Open Telemetry Collector for Azure IoT Operations. | `bool` | `true` | no | +| shouldEnableOpcUaSimulator | Whether or not to enable the OPC UA Simulator for Azure IoT Operations. | `bool` | `true` | no | +| shouldEnableAkriRestConnector | Deploy Akri REST HTTP Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriMediaConnector | Deploy Akri Media Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriOnvifConnector | Deploy Akri ONVIF Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriSseConnector | Deploy Akri SSE Connector template to the IoT Operations instance. | `bool` | `false` | no | +| customAkriConnectors | List of custom Akri connector templates with user-defined endpoint types and container images. | `array` | [] | no | +| akriMqttSharedConfig | Shared MQTT connection configuration for all Akri connectors. | `[_1.AkriMqttConfig](#user-defined-types)` | {'host': 'aio-broker:18883', 'audience': 'aio-internal', 'caConfigmap': 'azure-iot-operations-aio-ca-trust-bundle'} | no | +| customLocationName | The name for the Custom Locations resource. | `string` | [format('{0}-cl', parameters('arcConnectedClusterName'))] | no | +| additionalClusterExtensionIds | Additional cluster extension IDs to include in the custom location. (Appended to the default Secret Store and IoT Operations extension IDs) | `array` | [] | no | +| trustIssuerSettings | The trust issuer settings for Customer Managed Azure IoT Operations Settings. | `[_1.TrustIssuerConfig](#user-defined-types)` | {'trustSource': 'SelfSigned'} | no | +| sseKeyVaultName | The name of the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | n/a | yes | +| sseIdentityName | The name of the User Assigned Managed Identity for Secret Sync. | `string` | n/a | yes | +| sseKeyVaultResourceGroupName | The name of the Resource Group for the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | [resourceGroup().name] | no | +| shouldAssignSseKeyVaultRoles | Whether to assign roles for Key Vault to the provided Secret Sync Identity. | `bool` | `true` | no | +| shouldAssignDeployIdentityRoles | Whether to assign roles to the deploy identity. | `bool` | [not(empty(parameters('deployIdentityName')))] | no | +| deployIdentityName | The resource name for a managed identity that will be given deployment admin permissions. | `string` | n/a | no | +| shouldDeployAioDeploymentScripts | Whether to deploy DeploymentScripts for Azure IoT Operations. | `bool` | `false` | no | +| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | [parameters('sseKeyVaultName')] | no | +| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [parameters('sseKeyVaultResourceGroupName')] | no | +| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | +| deploymentScriptsSecretNamePrefix | The prefix used with constructing the secret name that will have the deployment script. | `string` | [format('{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | +| shouldAddDeployScriptsToKeyVault | Whether to add the deploy scripts for DeploymentScripts to Key Vault as secrets. (Required for DeploymentScripts) | `bool` | `false` | no | +| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | #### Resources for edgeIotOps @@ -716,24 +744,22 @@ Deploys Azure IoT Operations extensions, instances, and configurations on Azure #### Outputs for edgeIotOps -| Name | Type | Description | -|:------------------------------|:---------|:-------------------------------------------------------------------| -| containerStorageExtensionId | `string` | The ID of the Container Storage Extension. | -| containerStorageExtensionName | `string` | The name of the Container Storage Extension. | -| aioCertManagerExtensionId | `string` | The ID of the Azure IoT Operations Cert-Manager Extension. | -| aioCertManagerExtensionName | `string` | The name of the Azure IoT Operations Cert-Manager Extension. | -| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | -| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | -| customLocationId | `string` | The ID of the deployed Custom Location. | -| customLocationName | `string` | The name of the deployed Custom Location. | -| aioInstanceId | `string` | The ID of the deployed Azure IoT Operations instance. | -| aioInstanceName | `string` | The name of the deployed Azure IoT Operations instance. | -| dataFlowProfileId | `string` | The ID of the deployed Azure IoT Operations Data Flow Profile. | -| dataFlowProfileName | `string` | The name of the deployed Azure IoT Operations Data Flow Profile. | -| dataFlowEndpointId | `string` | The ID of the deployed Azure IoT Operations Data Flow Endpoint. | -| dataFlowEndpointName | `string` | The name of the deployed Azure IoT Operations Data Flow Endpoint. | -| akriConnectorTemplates | `array` | Map of deployed Akri connector templates by name with id and type. | -| akriConnectorTypesDeployed | `array` | List of Akri connector types that were deployed. | +| Name | Type | Description | +|:---------------------------|:---------|:-------------------------------------------------------------------| +| aioPlatformExtensionId | `string` | The ID of the Azure IoT Operations Platform Extension. | +| aioPlatformExtensionName | `string` | The name of the Azure IoT Operations Platform Extension. | +| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | +| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | +| customLocationId | `string` | The ID of the deployed Custom Location. | +| customLocationName | `string` | The name of the deployed Custom Location. | +| aioInstanceId | `string` | The ID of the deployed Azure IoT Operations instance. | +| aioInstanceName | `string` | The name of the deployed Azure IoT Operations instance. | +| dataFlowProfileId | `string` | The ID of the deployed Azure IoT Operations Data Flow Profile. | +| dataFlowProfileName | `string` | The name of the deployed Azure IoT Operations Data Flow Profile. | +| dataFlowEndpointId | `string` | The ID of the deployed Azure IoT Operations Data Flow Endpoint. | +| dataFlowEndpointName | `string` | The name of the deployed Azure IoT Operations Data Flow Endpoint. | +| akriConnectorTemplates | `array` | Map of deployed Akri connector templates by name with id and type. | +| akriConnectorTypesDeployed | `array` | List of Akri connector types that were deployed. | ### edgeAssets @@ -963,15 +989,6 @@ Configuration for Azure IoT Operations Certificate Authority. | caCertChainPem | `securestring` | The PEM-formatted CA certificate chain. | | caKeyPem | `securestring` | The PEM-formatted CA private key. | -### `_3.AioCertManagerExtension` - -The settings for the Azure IoT Operations Platform Extension. - -| Property | Type | Description | -|:---------|:------------------------------------|:---------------------------------------| -| release | `[_3.Release](#user-defined-types)` | The common settings for the extension. | -| settings | `object` | | - ### `_3.AioDataFlowInstance` The settings for Azure IoT Operations Data Flow Instances. @@ -1102,15 +1119,6 @@ Broker persistence configuration for disk-backed message storage. | subscriberQueue | `object` | Controls which subscriber queues should be persisted to disk. | | persistentVolumeClaimSpec | `object` | Persistent volume claim specification for storage. | -### `_3.ContainerStorageExtension` - -The settings for the Azure Container Store for Azure Arc Extension. - -| Property | Type | Description | -|:---------|:------------------------------------|:---------------------------------------| -| release | `[_3.Release](#user-defined-types)` | The common settings for the extension. | -| settings | `object` | | - ### `_3.CustomerManagedByoIssuerConfig` The configuration for Customer Managed Bring Your Own Issuer for Azure IoT Operations certificates. diff --git a/blueprints/full-multi-node-cluster/bicep/main.bicep b/blueprints/full-multi-node-cluster/bicep/main.bicep index 55337e61..e10150b3 100644 --- a/blueprints/full-multi-node-cluster/bicep/main.bicep +++ b/blueprints/full-multi-node-cluster/bicep/main.bicep @@ -447,10 +447,19 @@ module edgeCncfCluster '../../../src/100-edge/100-cncf-cluster/bicep/main.bicep' } } +module edgeArcExtensions '../../../src/100-edge/109-arc-extensions/bicep/main.bicep' = { + name: '${deployment().name}-eae4' + scope: resourceGroup(resourceGroupName) + dependsOn: [cloudResourceGroup] + params: { + arcConnectedClusterName: edgeCncfCluster.outputs.connectedClusterName + } +} + module edgeIotOps '../../../src/100-edge/110-iot-ops/bicep/main.bicep' = { name: '${deployment().name}-eio5' scope: resourceGroup(resourceGroupName) - dependsOn: [cloudResourceGroup] + dependsOn: [edgeArcExtensions] params: { // Common Parameters common: common @@ -566,10 +575,10 @@ output aiFoundryEndpoint string? = shouldDeployAiFoundry ? cloudAiFoundry.?outpu output aiFoundryPrincipalId string? = shouldDeployAiFoundry ? cloudAiFoundry.?outputs.?aiFoundryPrincipalId : null @description('The ID of the Azure IoT Operations Cert-Manager Extension.') -output aioCertManagerExtensionId string = edgeIotOps.outputs.aioCertManagerExtensionId +output aioCertManagerExtensionId string = edgeArcExtensions.outputs.certManagerExtensionId @description('The name of the Azure IoT Operations Cert-Manager Extension.') -output aioCertManagerExtensionName string = edgeIotOps.outputs.aioCertManagerExtensionName +output aioCertManagerExtensionName string = edgeArcExtensions.outputs.certManagerExtensionName @description('The ID of the Secret Store Extension.') output secretStoreExtensionId string = edgeIotOps.outputs.secretStoreExtensionId diff --git a/blueprints/full-multi-node-cluster/terraform/README.md b/blueprints/full-multi-node-cluster/terraform/README.md index 560fba5a..3cf82632 100644 --- a/blueprints/full-multi-node-cluster/terraform/README.md +++ b/blueprints/full-multi-node-cluster/terraform/README.md @@ -46,6 +46,7 @@ with the single-node blueprint while preserving multi-node specific capabilities | cloud\_security\_identity | ../../../src/000-cloud/010-security-identity/terraform | n/a | | cloud\_vm\_host | ../../../src/000-cloud/051-vm-host/terraform | n/a | | cloud\_vpn\_gateway | ../../../src/000-cloud/055-vpn-gateway/terraform | n/a | +| edge\_arc\_extensions | ../../../src/100-edge/109-arc-extensions/terraform | n/a | | edge\_assets | ../../../src/100-edge/111-assets/terraform | n/a | | edge\_azureml | ../../../src/100-edge/140-azureml/terraform | n/a | | edge\_cncf\_cluster | ../../../src/100-edge/100-cncf-cluster/terraform | n/a | diff --git a/blueprints/full-multi-node-cluster/terraform/main.tf b/blueprints/full-multi-node-cluster/terraform/main.tf index 77e15cf5..0bbc0c76 100644 --- a/blueprints/full-multi-node-cluster/terraform/main.tf +++ b/blueprints/full-multi-node-cluster/terraform/main.tf @@ -411,10 +411,18 @@ module "edge_cncf_cluster" { key_vault = module.cloud_security_identity.key_vault } +module "edge_arc_extensions" { + source = "../../../src/100-edge/109-arc-extensions/terraform" + + depends_on = [module.edge_cncf_cluster] + + arc_connected_cluster = module.edge_cncf_cluster.arc_connected_cluster +} + module "edge_iot_ops" { source = "../../../src/100-edge/110-iot-ops/terraform" - depends_on = [module.edge_cncf_cluster] + depends_on = [module.edge_arc_extensions] adr_schema_registry = module.cloud_data.schema_registry adr_namespace = module.cloud_data.adr_namespace diff --git a/blueprints/full-multi-node-cluster/terraform/variables.tf b/blueprints/full-multi-node-cluster/terraform/variables.tf index 9fa2d90b..51565954 100644 --- a/blueprints/full-multi-node-cluster/terraform/variables.tf +++ b/blueprints/full-multi-node-cluster/terraform/variables.tf @@ -1,3 +1,8 @@ +/* + * Blueprint variables - Full Multi Node Cluster + */ + + /* * Core Parameters - Required */ diff --git a/blueprints/full-single-node-cluster/bicep/README.md b/blueprints/full-single-node-cluster/bicep/README.md index 0197f785..21b8159a 100644 --- a/blueprints/full-single-node-cluster/bicep/README.md +++ b/blueprints/full-single-node-cluster/bicep/README.md @@ -67,6 +67,7 @@ Deploys a complete end-to-end environment for Azure IoT Operations on a single-n | cloudAiFoundry | `Microsoft.Resources/deployments` | 2025-04-01 | | cloudKubernetes | `Microsoft.Resources/deployments` | 2025-04-01 | | edgeCncfCluster | `Microsoft.Resources/deployments` | 2025-04-01 | +| edgeArcExtensions | `Microsoft.Resources/deployments` | 2025-04-01 | | edgeIotOps | `Microsoft.Resources/deployments` | 2025-04-01 | | edgeAssets | `Microsoft.Resources/deployments` | 2025-04-01 | | edgeObservability | `Microsoft.Resources/deployments` | 2025-04-01 | @@ -88,6 +89,7 @@ Deploys a complete end-to-end environment for Azure IoT Operations on a single-n | cloudAiFoundry | Deploys Microsoft Foundry account with optional projects, model deployments, RAI policies, and private endpoint support. | | cloudKubernetes | Deploys optionally Azure Kubernetes Service (AKS) resources. | | edgeCncfCluster | This module provisions and deploys automation scripts to a VM host that create and configure a K3s Kubernetes cluster with Arc connectivity.
The scripts handle primary and secondary node(s) setup, cluster administration, workload identity enablement, and installation of required Azure Arc extensions. | +| edgeArcExtensions | Deploys foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA). | | edgeIotOps | Deploys Azure IoT Operations extensions, instances, and configurations on Azure Arc-enabled Kubernetes clusters. | | edgeAssets | Deploys Kubernetes asset definitions to a connected cluster using the namespaced Device Registry model. This component facilitates the management of devices and assets within ADR namespaces. | | edgeObservability | Deploys observability resources including cluster extensions for metrics and logs collection, and rule groups for monitoring. | @@ -413,7 +415,7 @@ Provisions virtual machines and networking infrastructure for hosting Azure IoT ### cloudVpnGateway -Creates a VPN Gateway with Point-to-Site and optional Site-to-Site connectivity. +Creates a VPN Gateway with Point-to-Site and optional Site-to-Site connectivity. Ths component currently only supports Azure AD (Entra ID) authentication for Point-to-Site VPN connections. #### Parameters for cloudVpnGateway @@ -588,7 +590,7 @@ Deploys optionally Azure Kubernetes Service (AKS) resources. ### edgeCncfCluster -This module provisions and deploys automation scripts to a VM host that create and configure a K3s Kubernetes cluster with Arc connectivity. +This module provisions and deploys automation scripts to a VM host that create and configure a K3s Kubernetes cluster with Arc connectivity. The scripts handle primary and secondary node(s) setup, cluster administration, workload identity enablement, and installation of required Azure Arc extensions. #### Parameters for edgeCncfCluster @@ -644,57 +646,83 @@ The scripts handle primary and secondary node(s) setup, cluster administration, | clusterServerScriptSecretShowCommand | `string` | The AZ CLI command to get the cluster server script from Key Vault | | clusterNodeScriptSecretShowCommand | `string` | The AZ CLI command to get the cluster node script from Key Vault | +### edgeArcExtensions + +Deploys foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA). + +#### Parameters for edgeArcExtensions + +| Name | Description | Type | Default | Required | +|:------------------------|:----------------------------------------------------------------------|:------------------------------------------------------|:----------------------------------------------------|:---------| +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | +| certManagerConfig | The settings for the cert-manager Extension. | `[_1.CertManagerExtension](#user-defined-types)` | [variables('_1.certManagerExtensionDefaults')] | no | +| containerStorageConfig | The settings for the Azure Container Storage for Azure Arc Extension. | `[_1.ContainerStorageExtension](#user-defined-types)` | [variables('_1.containerStorageExtensionDefaults')] | no | + +#### Resources for edgeArcExtensions + +| Name | Type | API Version | +|:-----------------|:-----------------------------------------------|:------------| +| aioCertManager | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | +| containerStorage | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | + +#### Outputs for edgeArcExtensions + +| Name | Type | Description | +|:------------------------------|:---------|:----------------------------------------------------------| +| certManagerExtensionId | `string` | The resource ID of the cert-manager extension. | +| certManagerExtensionName | `string` | The name of the cert-manager extension. | +| containerStorageExtensionId | `string` | The resource ID of the Azure Container Storage extension. | +| containerStorageExtensionName | `string` | The name of the Azure Container Storage extension. | + ### edgeIotOps Deploys Azure IoT Operations extensions, instances, and configurations on Azure Arc-enabled Kubernetes clusters. #### Parameters for edgeIotOps -| Name | Description | Type | Default | Required | -|:------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------|:---------| -| common | The common component configuration. | `[_2.Common](#user-defined-types)` | n/a | yes | -| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | -| containerStorageConfig | The settings for the Azure Container Store for Azure Arc Extension. | `[_1.ContainerStorageExtension](#user-defined-types)` | [variables('_1.containerStorageExtensionDefaults')] | no | -| aioCertManagerConfig | The settings for the Azure IoT Operations Platform Extension. | `[_1.AioCertManagerExtension](#user-defined-types)` | [variables('_1.aioCertManagerExtensionDefaults')] | no | -| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | [variables('_1.secretStoreExtensionDefaults')] | no | -| shouldInitAio | Whether to deploy the Azure IoT Operations initial connected cluster resources, Secret Sync, ACSA, OSM, AIO Platform. | `bool` | `true` | no | -| aioIdentityName | The name of the User Assigned Managed Identity for Azure IoT Operations. | `string` | n/a | yes | -| aioExtensionConfig | The settings for the Azure IoT Operations Extension. | `[_1.AioExtension](#user-defined-types)` | [variables('_1.aioExtensionDefaults')] | no | -| aioFeatures | AIO Instance features. | `[_1.AioFeatures](#user-defined-types)` | n/a | no | -| aioInstanceName | The name for the Azure IoT Operations Instance resource. | `string` | [format('{0}-ops-instance', parameters('arcConnectedClusterName'))] | no | -| aioDataFlowInstanceConfig | The settings for Azure IoT Operations Data Flow Instances. | `[_1.AioDataFlowInstance](#user-defined-types)` | [variables('_1.aioDataFlowInstanceDefaults')] | no | -| aioMqBrokerConfig | The settings for the Azure IoT Operations MQ Broker. | `[_1.AioMqBroker](#user-defined-types)` | [variables('_1.aioMqBrokerDefaults')] | no | -| brokerListenerAnonymousConfig | Configuration for the insecure anonymous AIO MQ Broker Listener. | `[_1.AioMqBrokerAnonymous](#user-defined-types)` | [variables('_1.aioMqBrokerAnonymousDefaults')] | no | -| configurationSettingsOverride | Optional configuration settings to override default IoT Operations extension configuration. Use the same key names as the az iot ops --ops-config parameter. | `object` | {} | no | -| schemaRegistryName | The resource name for the ADR Schema Registry for Azure IoT Operations. | `string` | n/a | yes | -| adrNamespaceName | The resource name for the ADR Namespace for Azure IoT Operations. | `string` | n/a | no | -| shouldDeployAio | Whether to deploy an Azure IoT Operations Instance and all of its required components into the connected cluster. | `bool` | `true` | no | -| shouldDeployResourceSyncRules | Whether or not to deploy the Custom Locations Resource Sync Rules for the Azure IoT Operations resources. | `bool` | `true` | no | -| shouldCreateAnonymousBrokerListener | Whether to enable an insecure anonymous AIO MQ Broker Listener. (Should only be used for dev or test environments) | `bool` | `false` | no | -| shouldEnableOtelCollector | Whether or not to enable the Open Telemetry Collector for Azure IoT Operations. | `bool` | `true` | no | -| shouldEnableOpcUaSimulator | Whether or not to enable the OPC UA Simulator for Azure IoT Operations. | `bool` | `true` | no | -| shouldEnableAkriRestConnector | Deploy Akri REST HTTP Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriMediaConnector | Deploy Akri Media Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriOnvifConnector | Deploy Akri ONVIF Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriSseConnector | Deploy Akri SSE Connector template to the IoT Operations instance. | `bool` | `false` | no | -| customAkriConnectors | List of custom Akri connector templates with user-defined endpoint types and container images. | `array` | [] | no | -| akriMqttSharedConfig | Shared MQTT connection configuration for all Akri connectors. | `[_1.AkriMqttConfig](#user-defined-types)` | {'host': 'aio-broker:18883', 'audience': 'aio-internal', 'caConfigmap': 'azure-iot-operations-aio-ca-trust-bundle'} | no | -| customLocationName | The name for the Custom Locations resource. | `string` | [format('{0}-cl', parameters('arcConnectedClusterName'))] | no | -| additionalClusterExtensionIds | Additional cluster extension IDs to include in the custom location. (Appended to the default Secret Store and IoT Operations extension IDs) | `array` | [] | no | -| trustIssuerSettings | The trust issuer settings for Customer Managed Azure IoT Operations Settings. | `[_1.TrustIssuerConfig](#user-defined-types)` | {'trustSource': 'SelfSigned'} | no | -| sseKeyVaultName | The name of the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | n/a | yes | -| sseIdentityName | The name of the User Assigned Managed Identity for Secret Sync. | `string` | n/a | yes | -| sseKeyVaultResourceGroupName | The name of the Resource Group for the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | [resourceGroup().name] | no | -| shouldAssignSseKeyVaultRoles | Whether to assign roles for Key Vault to the provided Secret Sync Identity. | `bool` | `true` | no | -| shouldAssignDeployIdentityRoles | Whether to assign roles to the deploy identity. | `bool` | [not(empty(parameters('deployIdentityName')))] | no | -| deployIdentityName | The resource name for a managed identity that will be given deployment admin permissions. | `string` | n/a | no | -| shouldDeployAioDeploymentScripts | Whether to deploy DeploymentScripts for Azure IoT Operations. | `bool` | `false` | no | -| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | [parameters('sseKeyVaultName')] | no | -| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [parameters('sseKeyVaultResourceGroupName')] | no | -| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | -| deploymentScriptsSecretNamePrefix | The prefix used with constructing the secret name that will have the deployment script. | `string` | [format('{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | -| shouldAddDeployScriptsToKeyVault | Whether to add the deploy scripts for DeploymentScripts to Key Vault as secrets. (Required for DeploymentScripts) | `bool` | `false` | no | -| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | +| Name | Description | Type | Default | Required | +|:------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------|:---------| +| common | The common component configuration. | `[_2.Common](#user-defined-types)` | n/a | yes | +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | +| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | [variables('_1.secretStoreExtensionDefaults')] | no | +| shouldInitAio | Whether to deploy the Azure IoT Operations initial connected cluster resources, Secret Sync, ACSA, OSM, AIO Platform. | `bool` | `true` | no | +| aioIdentityName | The name of the User Assigned Managed Identity for Azure IoT Operations. | `string` | n/a | yes | +| aioExtensionConfig | The settings for the Azure IoT Operations Extension. | `[_1.AioExtension](#user-defined-types)` | [variables('_1.aioExtensionDefaults')] | no | +| aioFeatures | AIO Instance features. | `[_1.AioFeatures](#user-defined-types)` | n/a | no | +| aioInstanceName | The name for the Azure IoT Operations Instance resource. | `string` | [format('{0}-ops-instance', parameters('arcConnectedClusterName'))] | no | +| aioDataFlowInstanceConfig | The settings for Azure IoT Operations Data Flow Instances. | `[_1.AioDataFlowInstance](#user-defined-types)` | [variables('_1.aioDataFlowInstanceDefaults')] | no | +| aioMqBrokerConfig | The settings for the Azure IoT Operations MQ Broker. | `[_1.AioMqBroker](#user-defined-types)` | [variables('_1.aioMqBrokerDefaults')] | no | +| brokerListenerAnonymousConfig | Configuration for the insecure anonymous AIO MQ Broker Listener. | `[_1.AioMqBrokerAnonymous](#user-defined-types)` | [variables('_1.aioMqBrokerAnonymousDefaults')] | no | +| configurationSettingsOverride | Optional configuration settings to override default IoT Operations extension configuration. Use the same key names as the az iot ops --ops-config parameter. | `object` | {} | no | +| schemaRegistryName | The resource name for the ADR Schema Registry for Azure IoT Operations. | `string` | n/a | yes | +| adrNamespaceName | The resource name for the ADR Namespace for Azure IoT Operations. | `string` | n/a | no | +| shouldDeployAio | Whether to deploy an Azure IoT Operations Instance and all of its required components into the connected cluster. | `bool` | `true` | no | +| shouldDeployResourceSyncRules | Whether or not to deploy the Custom Locations Resource Sync Rules for the Azure IoT Operations resources. | `bool` | `true` | no | +| shouldCreateAnonymousBrokerListener | Whether to enable an insecure anonymous AIO MQ Broker Listener. (Should only be used for dev or test environments) | `bool` | `false` | no | +| shouldEnableOtelCollector | Whether or not to enable the Open Telemetry Collector for Azure IoT Operations. | `bool` | `true` | no | +| shouldEnableOpcUaSimulator | Whether or not to enable the OPC UA Simulator for Azure IoT Operations. | `bool` | `true` | no | +| shouldEnableAkriRestConnector | Deploy Akri REST HTTP Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriMediaConnector | Deploy Akri Media Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriOnvifConnector | Deploy Akri ONVIF Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriSseConnector | Deploy Akri SSE Connector template to the IoT Operations instance. | `bool` | `false` | no | +| customAkriConnectors | List of custom Akri connector templates with user-defined endpoint types and container images. | `array` | [] | no | +| akriMqttSharedConfig | Shared MQTT connection configuration for all Akri connectors. | `[_1.AkriMqttConfig](#user-defined-types)` | {'host': 'aio-broker:18883', 'audience': 'aio-internal', 'caConfigmap': 'azure-iot-operations-aio-ca-trust-bundle'} | no | +| customLocationName | The name for the Custom Locations resource. | `string` | [format('{0}-cl', parameters('arcConnectedClusterName'))] | no | +| additionalClusterExtensionIds | Additional cluster extension IDs to include in the custom location. (Appended to the default Secret Store and IoT Operations extension IDs) | `array` | [] | no | +| trustIssuerSettings | The trust issuer settings for Customer Managed Azure IoT Operations Settings. | `[_1.TrustIssuerConfig](#user-defined-types)` | {'trustSource': 'SelfSigned'} | no | +| sseKeyVaultName | The name of the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | n/a | yes | +| sseIdentityName | The name of the User Assigned Managed Identity for Secret Sync. | `string` | n/a | yes | +| sseKeyVaultResourceGroupName | The name of the Resource Group for the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | [resourceGroup().name] | no | +| shouldAssignSseKeyVaultRoles | Whether to assign roles for Key Vault to the provided Secret Sync Identity. | `bool` | `true` | no | +| shouldAssignDeployIdentityRoles | Whether to assign roles to the deploy identity. | `bool` | [not(empty(parameters('deployIdentityName')))] | no | +| deployIdentityName | The resource name for a managed identity that will be given deployment admin permissions. | `string` | n/a | no | +| shouldDeployAioDeploymentScripts | Whether to deploy DeploymentScripts for Azure IoT Operations. | `bool` | `false` | no | +| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | [parameters('sseKeyVaultName')] | no | +| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [parameters('sseKeyVaultResourceGroupName')] | no | +| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | +| deploymentScriptsSecretNamePrefix | The prefix used with constructing the secret name that will have the deployment script. | `string` | [format('{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | +| shouldAddDeployScriptsToKeyVault | Whether to add the deploy scripts for DeploymentScripts to Key Vault as secrets. (Required for DeploymentScripts) | `bool` | `false` | no | +| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | #### Resources for edgeIotOps @@ -713,24 +741,22 @@ Deploys Azure IoT Operations extensions, instances, and configurations on Azure #### Outputs for edgeIotOps -| Name | Type | Description | -|:------------------------------|:---------|:-------------------------------------------------------------------| -| containerStorageExtensionId | `string` | The ID of the Container Storage Extension. | -| containerStorageExtensionName | `string` | The name of the Container Storage Extension. | -| aioCertManagerExtensionId | `string` | The ID of the Azure IoT Operations Cert-Manager Extension. | -| aioCertManagerExtensionName | `string` | The name of the Azure IoT Operations Cert-Manager Extension. | -| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | -| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | -| customLocationId | `string` | The ID of the deployed Custom Location. | -| customLocationName | `string` | The name of the deployed Custom Location. | -| aioInstanceId | `string` | The ID of the deployed Azure IoT Operations instance. | -| aioInstanceName | `string` | The name of the deployed Azure IoT Operations instance. | -| dataFlowProfileId | `string` | The ID of the deployed Azure IoT Operations Data Flow Profile. | -| dataFlowProfileName | `string` | The name of the deployed Azure IoT Operations Data Flow Profile. | -| dataFlowEndpointId | `string` | The ID of the deployed Azure IoT Operations Data Flow Endpoint. | -| dataFlowEndpointName | `string` | The name of the deployed Azure IoT Operations Data Flow Endpoint. | -| akriConnectorTemplates | `array` | Map of deployed Akri connector templates by name with id and type. | -| akriConnectorTypesDeployed | `array` | List of Akri connector types that were deployed. | +| Name | Type | Description | +|:---------------------------|:---------|:-------------------------------------------------------------------| +| aioPlatformExtensionId | `string` | The ID of the Azure IoT Operations Platform Extension. | +| aioPlatformExtensionName | `string` | The name of the Azure IoT Operations Platform Extension. | +| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | +| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | +| customLocationId | `string` | The ID of the deployed Custom Location. | +| customLocationName | `string` | The name of the deployed Custom Location. | +| aioInstanceId | `string` | The ID of the deployed Azure IoT Operations instance. | +| aioInstanceName | `string` | The name of the deployed Azure IoT Operations instance. | +| dataFlowProfileId | `string` | The ID of the deployed Azure IoT Operations Data Flow Profile. | +| dataFlowProfileName | `string` | The name of the deployed Azure IoT Operations Data Flow Profile. | +| dataFlowEndpointId | `string` | The ID of the deployed Azure IoT Operations Data Flow Endpoint. | +| dataFlowEndpointName | `string` | The name of the deployed Azure IoT Operations Data Flow Endpoint. | +| akriConnectorTemplates | `array` | Map of deployed Akri connector templates by name with id and type. | +| akriConnectorTypesDeployed | `array` | List of Akri connector types that were deployed. | ### edgeAssets @@ -960,15 +986,6 @@ Configuration for Azure IoT Operations Certificate Authority. | caCertChainPem | `securestring` | The PEM-formatted CA certificate chain. | | caKeyPem | `securestring` | The PEM-formatted CA private key. | -### `_3.AioCertManagerExtension` - -The settings for the Azure IoT Operations Platform Extension. - -| Property | Type | Description | -|:---------|:------------------------------------|:---------------------------------------| -| release | `[_3.Release](#user-defined-types)` | The common settings for the extension. | -| settings | `object` | | - ### `_3.AioDataFlowInstance` The settings for Azure IoT Operations Data Flow Instances. @@ -1099,15 +1116,6 @@ Broker persistence configuration for disk-backed message storage. | subscriberQueue | `object` | Controls which subscriber queues should be persisted to disk. | | persistentVolumeClaimSpec | `object` | Persistent volume claim specification for storage. | -### `_3.ContainerStorageExtension` - -The settings for the Azure Container Store for Azure Arc Extension. - -| Property | Type | Description | -|:---------|:------------------------------------|:---------------------------------------| -| release | `[_3.Release](#user-defined-types)` | The common settings for the extension. | -| settings | `object` | | - ### `_3.CustomerManagedByoIssuerConfig` The configuration for Customer Managed Bring Your Own Issuer for Azure IoT Operations certificates. diff --git a/blueprints/full-single-node-cluster/bicep/main.bicep b/blueprints/full-single-node-cluster/bicep/main.bicep index 72013c1b..0c935724 100644 --- a/blueprints/full-single-node-cluster/bicep/main.bicep +++ b/blueprints/full-single-node-cluster/bicep/main.bicep @@ -428,10 +428,19 @@ module edgeCncfCluster '../../../src/100-edge/100-cncf-cluster/bicep/main.bicep' } } +module edgeArcExtensions '../../../src/100-edge/109-arc-extensions/bicep/main.bicep' = { + name: '${deployment().name}-eae4' + scope: resourceGroup(resourceGroupName) + dependsOn: [cloudResourceGroup] + params: { + arcConnectedClusterName: edgeCncfCluster.outputs.connectedClusterName + } +} + module edgeIotOps '../../../src/100-edge/110-iot-ops/bicep/main.bicep' = { name: '${deployment().name}-eio5' scope: resourceGroup(resourceGroupName) - dependsOn: [cloudResourceGroup] + dependsOn: [edgeArcExtensions] params: { // Common Parameters common: common @@ -548,10 +557,10 @@ output aiFoundryEndpoint string? = shouldDeployAiFoundry ? cloudAiFoundry.?outpu output aiFoundryPrincipalId string? = shouldDeployAiFoundry ? cloudAiFoundry.?outputs.?aiFoundryPrincipalId : null @description('The ID of the Azure IoT Operations Cert-Manager Extension.') -output aioCertManagerExtensionId string = edgeIotOps.outputs.aioCertManagerExtensionId +output aioCertManagerExtensionId string = edgeArcExtensions.outputs.certManagerExtensionId @description('The name of the Azure IoT Operations Cert-Manager Extension.') -output aioCertManagerExtensionName string = edgeIotOps.outputs.aioCertManagerExtensionName +output aioCertManagerExtensionName string = edgeArcExtensions.outputs.certManagerExtensionName @description('The ID of the Secret Store Extension.') output secretStoreExtensionId string = edgeIotOps.outputs.secretStoreExtensionId diff --git a/blueprints/full-single-node-cluster/terraform/README.md b/blueprints/full-single-node-cluster/terraform/README.md index efcff634..22db4563 100644 --- a/blueprints/full-single-node-cluster/terraform/README.md +++ b/blueprints/full-single-node-cluster/terraform/README.md @@ -31,6 +31,7 @@ for a single-node cluster deployment, including observability, messaging, and da | cloud\_security\_identity | ../../../src/000-cloud/010-security-identity/terraform | n/a | | cloud\_vm\_host | ../../../src/000-cloud/051-vm-host/terraform | n/a | | cloud\_vpn\_gateway | ../../../src/000-cloud/055-vpn-gateway/terraform | n/a | +| edge\_arc\_extensions | ../../../src/100-edge/109-arc-extensions/terraform | n/a | | edge\_assets | ../../../src/100-edge/111-assets/terraform | n/a | | edge\_azureml | ../../../src/100-edge/140-azureml/terraform | n/a | | edge\_cncf\_cluster | ../../../src/100-edge/100-cncf-cluster/terraform | n/a | diff --git a/blueprints/full-single-node-cluster/terraform/main.tf b/blueprints/full-single-node-cluster/terraform/main.tf index 0ea3a4e9..1e9c412f 100644 --- a/blueprints/full-single-node-cluster/terraform/main.tf +++ b/blueprints/full-single-node-cluster/terraform/main.tf @@ -371,10 +371,18 @@ module "edge_cncf_cluster" { key_vault = module.cloud_security_identity.key_vault } +module "edge_arc_extensions" { + source = "../../../src/100-edge/109-arc-extensions/terraform" + + depends_on = [module.edge_cncf_cluster] + + arc_connected_cluster = module.edge_cncf_cluster.arc_connected_cluster +} + module "edge_iot_ops" { source = "../../../src/100-edge/110-iot-ops/terraform" - depends_on = [module.edge_cncf_cluster] + depends_on = [module.edge_arc_extensions] adr_schema_registry = module.cloud_data.schema_registry adr_namespace = module.cloud_data.adr_namespace diff --git a/blueprints/minimum-single-node-cluster/bicep/README.md b/blueprints/minimum-single-node-cluster/bicep/README.md index 1c362470..d8716323 100644 --- a/blueprints/minimum-single-node-cluster/bicep/README.md +++ b/blueprints/minimum-single-node-cluster/bicep/README.md @@ -33,6 +33,7 @@ Deploys the minimal set of resources required for Azure IoT Operations on a sing | cloudNetworking | `Microsoft.Resources/deployments` | 2025-04-01 | | cloudVmHost | `Microsoft.Resources/deployments` | 2025-04-01 | | edgeCncfCluster | `Microsoft.Resources/deployments` | 2025-04-01 | +| edgeArcExtensions | `Microsoft.Resources/deployments` | 2025-04-01 | | edgeIotOps | `Microsoft.Resources/deployments` | 2025-04-01 | | edgeAssets | `Microsoft.Resources/deployments` | 2025-04-01 | @@ -46,6 +47,7 @@ Deploys the minimal set of resources required for Azure IoT Operations on a sing | cloudNetworking | Creates virtual network, subnet, and network security group resources for Azure deployments. | | cloudVmHost | Provisions virtual machines and networking infrastructure for hosting Azure IoT Operations edge deployments. | | edgeCncfCluster | This module provisions and deploys automation scripts to a VM host that create and configure a K3s Kubernetes cluster with Arc connectivity.
The scripts handle primary and secondary node(s) setup, cluster administration, workload identity enablement, and installation of required Azure Arc extensions. | +| edgeArcExtensions | Deploys foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA). | | edgeIotOps | Deploys Azure IoT Operations extensions, instances, and configurations on Azure Arc-enabled Kubernetes clusters. | | edgeAssets | Deploys Kubernetes asset definitions to a connected cluster using the namespaced Device Registry model. This component facilitates the management of devices and assets within ADR namespaces. | @@ -261,7 +263,7 @@ Provisions virtual machines and networking infrastructure for hosting Azure IoT ### edgeCncfCluster -This module provisions and deploys automation scripts to a VM host that create and configure a K3s Kubernetes cluster with Arc connectivity. +This module provisions and deploys automation scripts to a VM host that create and configure a K3s Kubernetes cluster with Arc connectivity. The scripts handle primary and secondary node(s) setup, cluster administration, workload identity enablement, and installation of required Azure Arc extensions. #### Parameters for edgeCncfCluster @@ -317,57 +319,83 @@ The scripts handle primary and secondary node(s) setup, cluster administration, | clusterServerScriptSecretShowCommand | `string` | The AZ CLI command to get the cluster server script from Key Vault | | clusterNodeScriptSecretShowCommand | `string` | The AZ CLI command to get the cluster node script from Key Vault | +### edgeArcExtensions + +Deploys foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA). + +#### Parameters for edgeArcExtensions + +| Name | Description | Type | Default | Required | +|:------------------------|:----------------------------------------------------------------------|:------------------------------------------------------|:----------------------------------------------------|:---------| +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | +| certManagerConfig | The settings for the cert-manager Extension. | `[_1.CertManagerExtension](#user-defined-types)` | [variables('_1.certManagerExtensionDefaults')] | no | +| containerStorageConfig | The settings for the Azure Container Storage for Azure Arc Extension. | `[_1.ContainerStorageExtension](#user-defined-types)` | [variables('_1.containerStorageExtensionDefaults')] | no | + +#### Resources for edgeArcExtensions + +| Name | Type | API Version | +|:-----------------|:-----------------------------------------------|:------------| +| aioCertManager | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | +| containerStorage | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | + +#### Outputs for edgeArcExtensions + +| Name | Type | Description | +|:------------------------------|:---------|:----------------------------------------------------------| +| certManagerExtensionId | `string` | The resource ID of the cert-manager extension. | +| certManagerExtensionName | `string` | The name of the cert-manager extension. | +| containerStorageExtensionId | `string` | The resource ID of the Azure Container Storage extension. | +| containerStorageExtensionName | `string` | The name of the Azure Container Storage extension. | + ### edgeIotOps Deploys Azure IoT Operations extensions, instances, and configurations on Azure Arc-enabled Kubernetes clusters. #### Parameters for edgeIotOps -| Name | Description | Type | Default | Required | -|:------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------|:---------| -| common | The common component configuration. | `[_2.Common](#user-defined-types)` | n/a | yes | -| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | -| containerStorageConfig | The settings for the Azure Container Store for Azure Arc Extension. | `[_1.ContainerStorageExtension](#user-defined-types)` | [variables('_1.containerStorageExtensionDefaults')] | no | -| aioCertManagerConfig | The settings for the Azure IoT Operations Platform Extension. | `[_1.AioCertManagerExtension](#user-defined-types)` | [variables('_1.aioCertManagerExtensionDefaults')] | no | -| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | [variables('_1.secretStoreExtensionDefaults')] | no | -| shouldInitAio | Whether to deploy the Azure IoT Operations initial connected cluster resources, Secret Sync, ACSA, OSM, AIO Platform. | `bool` | `true` | no | -| aioIdentityName | The name of the User Assigned Managed Identity for Azure IoT Operations. | `string` | n/a | yes | -| aioExtensionConfig | The settings for the Azure IoT Operations Extension. | `[_1.AioExtension](#user-defined-types)` | [variables('_1.aioExtensionDefaults')] | no | -| aioFeatures | AIO Instance features. | `[_1.AioFeatures](#user-defined-types)` | n/a | no | -| aioInstanceName | The name for the Azure IoT Operations Instance resource. | `string` | [format('{0}-ops-instance', parameters('arcConnectedClusterName'))] | no | -| aioDataFlowInstanceConfig | The settings for Azure IoT Operations Data Flow Instances. | `[_1.AioDataFlowInstance](#user-defined-types)` | [variables('_1.aioDataFlowInstanceDefaults')] | no | -| aioMqBrokerConfig | The settings for the Azure IoT Operations MQ Broker. | `[_1.AioMqBroker](#user-defined-types)` | [variables('_1.aioMqBrokerDefaults')] | no | -| brokerListenerAnonymousConfig | Configuration for the insecure anonymous AIO MQ Broker Listener. | `[_1.AioMqBrokerAnonymous](#user-defined-types)` | [variables('_1.aioMqBrokerAnonymousDefaults')] | no | -| configurationSettingsOverride | Optional configuration settings to override default IoT Operations extension configuration. Use the same key names as the az iot ops --ops-config parameter. | `object` | {} | no | -| schemaRegistryName | The resource name for the ADR Schema Registry for Azure IoT Operations. | `string` | n/a | yes | -| adrNamespaceName | The resource name for the ADR Namespace for Azure IoT Operations. | `string` | n/a | no | -| shouldDeployAio | Whether to deploy an Azure IoT Operations Instance and all of its required components into the connected cluster. | `bool` | `true` | no | -| shouldDeployResourceSyncRules | Whether or not to deploy the Custom Locations Resource Sync Rules for the Azure IoT Operations resources. | `bool` | `true` | no | -| shouldCreateAnonymousBrokerListener | Whether to enable an insecure anonymous AIO MQ Broker Listener. (Should only be used for dev or test environments) | `bool` | `false` | no | -| shouldEnableOtelCollector | Whether or not to enable the Open Telemetry Collector for Azure IoT Operations. | `bool` | `true` | no | -| shouldEnableOpcUaSimulator | Whether or not to enable the OPC UA Simulator for Azure IoT Operations. | `bool` | `true` | no | -| shouldEnableAkriRestConnector | Deploy Akri REST HTTP Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriMediaConnector | Deploy Akri Media Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriOnvifConnector | Deploy Akri ONVIF Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriSseConnector | Deploy Akri SSE Connector template to the IoT Operations instance. | `bool` | `false` | no | -| customAkriConnectors | List of custom Akri connector templates with user-defined endpoint types and container images. | `array` | [] | no | -| akriMqttSharedConfig | Shared MQTT connection configuration for all Akri connectors. | `[_1.AkriMqttConfig](#user-defined-types)` | {'host': 'aio-broker:18883', 'audience': 'aio-internal', 'caConfigmap': 'azure-iot-operations-aio-ca-trust-bundle'} | no | -| customLocationName | The name for the Custom Locations resource. | `string` | [format('{0}-cl', parameters('arcConnectedClusterName'))] | no | -| additionalClusterExtensionIds | Additional cluster extension IDs to include in the custom location. (Appended to the default Secret Store and IoT Operations extension IDs) | `array` | [] | no | -| trustIssuerSettings | The trust issuer settings for Customer Managed Azure IoT Operations Settings. | `[_1.TrustIssuerConfig](#user-defined-types)` | {'trustSource': 'SelfSigned'} | no | -| sseKeyVaultName | The name of the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | n/a | yes | -| sseIdentityName | The name of the User Assigned Managed Identity for Secret Sync. | `string` | n/a | yes | -| sseKeyVaultResourceGroupName | The name of the Resource Group for the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | [resourceGroup().name] | no | -| shouldAssignSseKeyVaultRoles | Whether to assign roles for Key Vault to the provided Secret Sync Identity. | `bool` | `true` | no | -| shouldAssignDeployIdentityRoles | Whether to assign roles to the deploy identity. | `bool` | [not(empty(parameters('deployIdentityName')))] | no | -| deployIdentityName | The resource name for a managed identity that will be given deployment admin permissions. | `string` | n/a | no | -| shouldDeployAioDeploymentScripts | Whether to deploy DeploymentScripts for Azure IoT Operations. | `bool` | `false` | no | -| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | [parameters('sseKeyVaultName')] | no | -| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [parameters('sseKeyVaultResourceGroupName')] | no | -| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | -| deploymentScriptsSecretNamePrefix | The prefix used with constructing the secret name that will have the deployment script. | `string` | [format('{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | -| shouldAddDeployScriptsToKeyVault | Whether to add the deploy scripts for DeploymentScripts to Key Vault as secrets. (Required for DeploymentScripts) | `bool` | `false` | no | -| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | +| Name | Description | Type | Default | Required | +|:------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------|:---------| +| common | The common component configuration. | `[_2.Common](#user-defined-types)` | n/a | yes | +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | +| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | [variables('_1.secretStoreExtensionDefaults')] | no | +| shouldInitAio | Whether to deploy the Azure IoT Operations initial connected cluster resources, Secret Sync, ACSA, OSM, AIO Platform. | `bool` | `true` | no | +| aioIdentityName | The name of the User Assigned Managed Identity for Azure IoT Operations. | `string` | n/a | yes | +| aioExtensionConfig | The settings for the Azure IoT Operations Extension. | `[_1.AioExtension](#user-defined-types)` | [variables('_1.aioExtensionDefaults')] | no | +| aioFeatures | AIO Instance features. | `[_1.AioFeatures](#user-defined-types)` | n/a | no | +| aioInstanceName | The name for the Azure IoT Operations Instance resource. | `string` | [format('{0}-ops-instance', parameters('arcConnectedClusterName'))] | no | +| aioDataFlowInstanceConfig | The settings for Azure IoT Operations Data Flow Instances. | `[_1.AioDataFlowInstance](#user-defined-types)` | [variables('_1.aioDataFlowInstanceDefaults')] | no | +| aioMqBrokerConfig | The settings for the Azure IoT Operations MQ Broker. | `[_1.AioMqBroker](#user-defined-types)` | [variables('_1.aioMqBrokerDefaults')] | no | +| brokerListenerAnonymousConfig | Configuration for the insecure anonymous AIO MQ Broker Listener. | `[_1.AioMqBrokerAnonymous](#user-defined-types)` | [variables('_1.aioMqBrokerAnonymousDefaults')] | no | +| configurationSettingsOverride | Optional configuration settings to override default IoT Operations extension configuration. Use the same key names as the az iot ops --ops-config parameter. | `object` | {} | no | +| schemaRegistryName | The resource name for the ADR Schema Registry for Azure IoT Operations. | `string` | n/a | yes | +| adrNamespaceName | The resource name for the ADR Namespace for Azure IoT Operations. | `string` | n/a | no | +| shouldDeployAio | Whether to deploy an Azure IoT Operations Instance and all of its required components into the connected cluster. | `bool` | `true` | no | +| shouldDeployResourceSyncRules | Whether or not to deploy the Custom Locations Resource Sync Rules for the Azure IoT Operations resources. | `bool` | `true` | no | +| shouldCreateAnonymousBrokerListener | Whether to enable an insecure anonymous AIO MQ Broker Listener. (Should only be used for dev or test environments) | `bool` | `false` | no | +| shouldEnableOtelCollector | Whether or not to enable the Open Telemetry Collector for Azure IoT Operations. | `bool` | `true` | no | +| shouldEnableOpcUaSimulator | Whether or not to enable the OPC UA Simulator for Azure IoT Operations. | `bool` | `true` | no | +| shouldEnableAkriRestConnector | Deploy Akri REST HTTP Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriMediaConnector | Deploy Akri Media Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriOnvifConnector | Deploy Akri ONVIF Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriSseConnector | Deploy Akri SSE Connector template to the IoT Operations instance. | `bool` | `false` | no | +| customAkriConnectors | List of custom Akri connector templates with user-defined endpoint types and container images. | `array` | [] | no | +| akriMqttSharedConfig | Shared MQTT connection configuration for all Akri connectors. | `[_1.AkriMqttConfig](#user-defined-types)` | {'host': 'aio-broker:18883', 'audience': 'aio-internal', 'caConfigmap': 'azure-iot-operations-aio-ca-trust-bundle'} | no | +| customLocationName | The name for the Custom Locations resource. | `string` | [format('{0}-cl', parameters('arcConnectedClusterName'))] | no | +| additionalClusterExtensionIds | Additional cluster extension IDs to include in the custom location. (Appended to the default Secret Store and IoT Operations extension IDs) | `array` | [] | no | +| trustIssuerSettings | The trust issuer settings for Customer Managed Azure IoT Operations Settings. | `[_1.TrustIssuerConfig](#user-defined-types)` | {'trustSource': 'SelfSigned'} | no | +| sseKeyVaultName | The name of the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | n/a | yes | +| sseIdentityName | The name of the User Assigned Managed Identity for Secret Sync. | `string` | n/a | yes | +| sseKeyVaultResourceGroupName | The name of the Resource Group for the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | [resourceGroup().name] | no | +| shouldAssignSseKeyVaultRoles | Whether to assign roles for Key Vault to the provided Secret Sync Identity. | `bool` | `true` | no | +| shouldAssignDeployIdentityRoles | Whether to assign roles to the deploy identity. | `bool` | [not(empty(parameters('deployIdentityName')))] | no | +| deployIdentityName | The resource name for a managed identity that will be given deployment admin permissions. | `string` | n/a | no | +| shouldDeployAioDeploymentScripts | Whether to deploy DeploymentScripts for Azure IoT Operations. | `bool` | `false` | no | +| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | [parameters('sseKeyVaultName')] | no | +| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [parameters('sseKeyVaultResourceGroupName')] | no | +| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | +| deploymentScriptsSecretNamePrefix | The prefix used with constructing the secret name that will have the deployment script. | `string` | [format('{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | +| shouldAddDeployScriptsToKeyVault | Whether to add the deploy scripts for DeploymentScripts to Key Vault as secrets. (Required for DeploymentScripts) | `bool` | `false` | no | +| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | #### Resources for edgeIotOps @@ -386,24 +414,22 @@ Deploys Azure IoT Operations extensions, instances, and configurations on Azure #### Outputs for edgeIotOps -| Name | Type | Description | -|:------------------------------|:---------|:-------------------------------------------------------------------| -| containerStorageExtensionId | `string` | The ID of the Container Storage Extension. | -| containerStorageExtensionName | `string` | The name of the Container Storage Extension. | -| aioCertManagerExtensionId | `string` | The ID of the Azure IoT Operations Cert-Manager Extension. | -| aioCertManagerExtensionName | `string` | The name of the Azure IoT Operations Cert-Manager Extension. | -| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | -| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | -| customLocationId | `string` | The ID of the deployed Custom Location. | -| customLocationName | `string` | The name of the deployed Custom Location. | -| aioInstanceId | `string` | The ID of the deployed Azure IoT Operations instance. | -| aioInstanceName | `string` | The name of the deployed Azure IoT Operations instance. | -| dataFlowProfileId | `string` | The ID of the deployed Azure IoT Operations Data Flow Profile. | -| dataFlowProfileName | `string` | The name of the deployed Azure IoT Operations Data Flow Profile. | -| dataFlowEndpointId | `string` | The ID of the deployed Azure IoT Operations Data Flow Endpoint. | -| dataFlowEndpointName | `string` | The name of the deployed Azure IoT Operations Data Flow Endpoint. | -| akriConnectorTemplates | `array` | Map of deployed Akri connector templates by name with id and type. | -| akriConnectorTypesDeployed | `array` | List of Akri connector types that were deployed. | +| Name | Type | Description | +|:---------------------------|:---------|:-------------------------------------------------------------------| +| aioPlatformExtensionId | `string` | The ID of the Azure IoT Operations Platform Extension. | +| aioPlatformExtensionName | `string` | The name of the Azure IoT Operations Platform Extension. | +| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | +| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | +| customLocationId | `string` | The ID of the deployed Custom Location. | +| customLocationName | `string` | The name of the deployed Custom Location. | +| aioInstanceId | `string` | The ID of the deployed Azure IoT Operations instance. | +| aioInstanceName | `string` | The name of the deployed Azure IoT Operations instance. | +| dataFlowProfileId | `string` | The ID of the deployed Azure IoT Operations Data Flow Profile. | +| dataFlowProfileName | `string` | The name of the deployed Azure IoT Operations Data Flow Profile. | +| dataFlowEndpointId | `string` | The ID of the deployed Azure IoT Operations Data Flow Endpoint. | +| dataFlowEndpointName | `string` | The name of the deployed Azure IoT Operations Data Flow Endpoint. | +| akriConnectorTemplates | `array` | Map of deployed Akri connector templates by name with id and type. | +| akriConnectorTypesDeployed | `array` | List of Akri connector types that were deployed. | ### edgeAssets diff --git a/blueprints/minimum-single-node-cluster/bicep/main.bicep b/blueprints/minimum-single-node-cluster/bicep/main.bicep index 03ef86c9..1a35eb2d 100644 --- a/blueprints/minimum-single-node-cluster/bicep/main.bicep +++ b/blueprints/minimum-single-node-cluster/bicep/main.bicep @@ -91,6 +91,7 @@ param shouldCreateDefaultNamespacedAsset bool = true resource attribution 'Microsoft.Resources/deployments@2020-06-01' = if (!telemetry_opt_out) { name: 'pid-acce1e78-0375-4637-a593-86aa36dcfeac' + location: common.location properties: { mode: 'Incremental' template: { @@ -178,10 +179,19 @@ module edgeCncfCluster '../../../src/100-edge/100-cncf-cluster/bicep/main.bicep' } } +module edgeArcExtensions '../../../src/100-edge/109-arc-extensions/bicep/main.bicep' = { + name: '${deployment().name}-eae4' + scope: resourceGroup(resourceGroupName) + dependsOn: [cloudResourceGroup] + params: { + arcConnectedClusterName: edgeCncfCluster.outputs.connectedClusterName + } +} + module edgeIotOps '../../../src/100-edge/110-iot-ops/bicep/main.bicep' = { name: '${deployment().name}-eio5' scope: resourceGroup(resourceGroupName) - dependsOn: [cloudResourceGroup] + dependsOn: [edgeArcExtensions] params: { // Common Parameters common: common @@ -241,4 +251,4 @@ output vmUsername string = cloudVmHost.outputs.adminUsername output vmNames array = cloudVmHost.outputs.vmNames @description('The ID of the Azure IoT Operations Cert-Manager Extension.') -output aioCertManagerExtensionId string = edgeIotOps.outputs.aioCertManagerExtensionId +output aioCertManagerExtensionId string = edgeArcExtensions.outputs.certManagerExtensionId diff --git a/blueprints/minimum-single-node-cluster/terraform/README.md b/blueprints/minimum-single-node-cluster/terraform/README.md index 49115fd3..58e11c40 100644 --- a/blueprints/minimum-single-node-cluster/terraform/README.md +++ b/blueprints/minimum-single-node-cluster/terraform/README.md @@ -23,6 +23,7 @@ It includes only the essential components and minimizes resource usage. | cloud\_resource\_group | ../../../src/000-cloud/000-resource-group/terraform | n/a | | cloud\_security\_identity | ../../../src/000-cloud/010-security-identity/terraform | n/a | | cloud\_vm\_host | ../../../src/000-cloud/051-vm-host/terraform | n/a | +| edge\_arc\_extensions | ../../../src/100-edge/109-arc-extensions/terraform | n/a | | edge\_assets | ../../../src/100-edge/111-assets/terraform | n/a | | edge\_cncf\_cluster | ../../../src/100-edge/100-cncf-cluster/terraform | n/a | | edge\_iot\_ops | ../../../src/100-edge/110-iot-ops/terraform | n/a | diff --git a/blueprints/minimum-single-node-cluster/terraform/main.tf b/blueprints/minimum-single-node-cluster/terraform/main.tf index 5f8719bd..ec0beea8 100644 --- a/blueprints/minimum-single-node-cluster/terraform/main.tf +++ b/blueprints/minimum-single-node-cluster/terraform/main.tf @@ -105,10 +105,18 @@ module "edge_cncf_cluster" { key_vault = module.cloud_security_identity.key_vault } +module "edge_arc_extensions" { + source = "../../../src/100-edge/109-arc-extensions/terraform" + + depends_on = [module.edge_cncf_cluster] + + arc_connected_cluster = module.edge_cncf_cluster.arc_connected_cluster +} + module "edge_iot_ops" { source = "../../../src/100-edge/110-iot-ops/terraform" - depends_on = [module.edge_cncf_cluster] + depends_on = [module.edge_arc_extensions] adr_schema_registry = module.cloud_data.schema_registry adr_namespace = module.cloud_data.adr_namespace diff --git a/blueprints/only-edge-iot-ops/bicep/README.md b/blueprints/only-edge-iot-ops/bicep/README.md index 97440da4..f0fa18bc 100644 --- a/blueprints/only-edge-iot-ops/bicep/README.md +++ b/blueprints/only-edge-iot-ops/bicep/README.md @@ -39,71 +39,99 @@ Deploys Azure IoT Operations on an existing Arc-enabled Kubernetes cluster witho ## Resources -| Name | Type | API Version | -|:-----------|:----------------------------------|:------------| -| edgeIotOps | `Microsoft.Resources/deployments` | 2025-04-01 | -| edgeAssets | `Microsoft.Resources/deployments` | 2025-04-01 | +| Name | Type | API Version | +|:------------------|:----------------------------------|:------------| +| edgeArcExtensions | `Microsoft.Resources/deployments` | 2025-04-01 | +| edgeIotOps | `Microsoft.Resources/deployments` | 2025-04-01 | +| edgeAssets | `Microsoft.Resources/deployments` | 2025-04-01 | ## Modules -| Name | Description | -|:-----------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| edgeIotOps | Deploys Azure IoT Operations extensions, instances, and configurations on Azure Arc-enabled Kubernetes clusters. | -| edgeAssets | Deploys Kubernetes asset definitions to a connected cluster using the namespaced Device Registry model. This component facilitates the management of devices and assets within ADR namespaces. | +| Name | Description | +|:------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| edgeArcExtensions | Deploys foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA). | +| edgeIotOps | Deploys Azure IoT Operations extensions, instances, and configurations on Azure Arc-enabled Kubernetes clusters. | +| edgeAssets | Deploys Kubernetes asset definitions to a connected cluster using the namespaced Device Registry model. This component facilitates the management of devices and assets within ADR namespaces. | ## Module Details +### edgeArcExtensions + +Deploys foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA). + +#### Parameters for edgeArcExtensions + +| Name | Description | Type | Default | Required | +|:------------------------|:----------------------------------------------------------------------|:------------------------------------------------------|:----------------------------------------------------|:---------| +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | +| certManagerConfig | The settings for the cert-manager Extension. | `[_1.CertManagerExtension](#user-defined-types)` | [variables('_1.certManagerExtensionDefaults')] | no | +| containerStorageConfig | The settings for the Azure Container Storage for Azure Arc Extension. | `[_1.ContainerStorageExtension](#user-defined-types)` | [variables('_1.containerStorageExtensionDefaults')] | no | + +#### Resources for edgeArcExtensions + +| Name | Type | API Version | +|:-----------------|:-----------------------------------------------|:------------| +| aioCertManager | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | +| containerStorage | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | + +#### Outputs for edgeArcExtensions + +| Name | Type | Description | +|:------------------------------|:---------|:----------------------------------------------------------| +| certManagerExtensionId | `string` | The resource ID of the cert-manager extension. | +| certManagerExtensionName | `string` | The name of the cert-manager extension. | +| containerStorageExtensionId | `string` | The resource ID of the Azure Container Storage extension. | +| containerStorageExtensionName | `string` | The name of the Azure Container Storage extension. | + ### edgeIotOps Deploys Azure IoT Operations extensions, instances, and configurations on Azure Arc-enabled Kubernetes clusters. #### Parameters for edgeIotOps -| Name | Description | Type | Default | Required | -|:------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------|:---------| -| common | The common component configuration. | `[_2.Common](#user-defined-types)` | n/a | yes | -| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | -| containerStorageConfig | The settings for the Azure Container Store for Azure Arc Extension. | `[_1.ContainerStorageExtension](#user-defined-types)` | [variables('_1.containerStorageExtensionDefaults')] | no | -| aioCertManagerConfig | The settings for the Azure IoT Operations Platform Extension. | `[_1.AioCertManagerExtension](#user-defined-types)` | [variables('_1.aioCertManagerExtensionDefaults')] | no | -| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | [variables('_1.secretStoreExtensionDefaults')] | no | -| shouldInitAio | Whether to deploy the Azure IoT Operations initial connected cluster resources, Secret Sync, ACSA, OSM, AIO Platform. | `bool` | `true` | no | -| aioIdentityName | The name of the User Assigned Managed Identity for Azure IoT Operations. | `string` | n/a | yes | -| aioExtensionConfig | The settings for the Azure IoT Operations Extension. | `[_1.AioExtension](#user-defined-types)` | [variables('_1.aioExtensionDefaults')] | no | -| aioFeatures | AIO Instance features. | `[_1.AioFeatures](#user-defined-types)` | n/a | no | -| aioInstanceName | The name for the Azure IoT Operations Instance resource. | `string` | [format('{0}-ops-instance', parameters('arcConnectedClusterName'))] | no | -| aioDataFlowInstanceConfig | The settings for Azure IoT Operations Data Flow Instances. | `[_1.AioDataFlowInstance](#user-defined-types)` | [variables('_1.aioDataFlowInstanceDefaults')] | no | -| aioMqBrokerConfig | The settings for the Azure IoT Operations MQ Broker. | `[_1.AioMqBroker](#user-defined-types)` | [variables('_1.aioMqBrokerDefaults')] | no | -| brokerListenerAnonymousConfig | Configuration for the insecure anonymous AIO MQ Broker Listener. | `[_1.AioMqBrokerAnonymous](#user-defined-types)` | [variables('_1.aioMqBrokerAnonymousDefaults')] | no | -| configurationSettingsOverride | Optional configuration settings to override default IoT Operations extension configuration. Use the same key names as the az iot ops --ops-config parameter. | `object` | {} | no | -| schemaRegistryName | The resource name for the ADR Schema Registry for Azure IoT Operations. | `string` | n/a | yes | -| adrNamespaceName | The resource name for the ADR Namespace for Azure IoT Operations. | `string` | n/a | no | -| shouldDeployAio | Whether to deploy an Azure IoT Operations Instance and all of its required components into the connected cluster. | `bool` | `true` | no | -| shouldDeployResourceSyncRules | Whether or not to deploy the Custom Locations Resource Sync Rules for the Azure IoT Operations resources. | `bool` | `true` | no | -| shouldCreateAnonymousBrokerListener | Whether to enable an insecure anonymous AIO MQ Broker Listener. (Should only be used for dev or test environments) | `bool` | `false` | no | -| shouldEnableOtelCollector | Whether or not to enable the Open Telemetry Collector for Azure IoT Operations. | `bool` | `true` | no | -| shouldEnableOpcUaSimulator | Whether or not to enable the OPC UA Simulator for Azure IoT Operations. | `bool` | `true` | no | -| shouldEnableAkriRestConnector | Deploy Akri REST HTTP Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriMediaConnector | Deploy Akri Media Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriOnvifConnector | Deploy Akri ONVIF Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriSseConnector | Deploy Akri SSE Connector template to the IoT Operations instance. | `bool` | `false` | no | -| customAkriConnectors | List of custom Akri connector templates with user-defined endpoint types and container images. | `array` | [] | no | -| akriMqttSharedConfig | Shared MQTT connection configuration for all Akri connectors. | `[_1.AkriMqttConfig](#user-defined-types)` | {'host': 'aio-broker:18883', 'audience': 'aio-internal', 'caConfigmap': 'azure-iot-operations-aio-ca-trust-bundle'} | no | -| customLocationName | The name for the Custom Locations resource. | `string` | [format('{0}-cl', parameters('arcConnectedClusterName'))] | no | -| additionalClusterExtensionIds | Additional cluster extension IDs to include in the custom location. (Appended to the default Secret Store and IoT Operations extension IDs) | `array` | [] | no | -| trustIssuerSettings | The trust issuer settings for Customer Managed Azure IoT Operations Settings. | `[_1.TrustIssuerConfig](#user-defined-types)` | {'trustSource': 'SelfSigned'} | no | -| sseKeyVaultName | The name of the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | n/a | yes | -| sseIdentityName | The name of the User Assigned Managed Identity for Secret Sync. | `string` | n/a | yes | -| sseKeyVaultResourceGroupName | The name of the Resource Group for the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | [resourceGroup().name] | no | -| shouldAssignSseKeyVaultRoles | Whether to assign roles for Key Vault to the provided Secret Sync Identity. | `bool` | `true` | no | -| shouldAssignDeployIdentityRoles | Whether to assign roles to the deploy identity. | `bool` | [not(empty(parameters('deployIdentityName')))] | no | -| deployIdentityName | The resource name for a managed identity that will be given deployment admin permissions. | `string` | n/a | no | -| shouldDeployAioDeploymentScripts | Whether to deploy DeploymentScripts for Azure IoT Operations. | `bool` | `false` | no | -| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | [parameters('sseKeyVaultName')] | no | -| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [parameters('sseKeyVaultResourceGroupName')] | no | -| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | -| deploymentScriptsSecretNamePrefix | The prefix used with constructing the secret name that will have the deployment script. | `string` | [format('{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | -| shouldAddDeployScriptsToKeyVault | Whether to add the deploy scripts for DeploymentScripts to Key Vault as secrets. (Required for DeploymentScripts) | `bool` | `false` | no | -| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | +| Name | Description | Type | Default | Required | +|:------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------|:---------| +| common | The common component configuration. | `[_2.Common](#user-defined-types)` | n/a | yes | +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | +| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | [variables('_1.secretStoreExtensionDefaults')] | no | +| shouldInitAio | Whether to deploy the Azure IoT Operations initial connected cluster resources, Secret Sync, ACSA, OSM, AIO Platform. | `bool` | `true` | no | +| aioIdentityName | The name of the User Assigned Managed Identity for Azure IoT Operations. | `string` | n/a | yes | +| aioExtensionConfig | The settings for the Azure IoT Operations Extension. | `[_1.AioExtension](#user-defined-types)` | [variables('_1.aioExtensionDefaults')] | no | +| aioFeatures | AIO Instance features. | `[_1.AioFeatures](#user-defined-types)` | n/a | no | +| aioInstanceName | The name for the Azure IoT Operations Instance resource. | `string` | [format('{0}-ops-instance', parameters('arcConnectedClusterName'))] | no | +| aioDataFlowInstanceConfig | The settings for Azure IoT Operations Data Flow Instances. | `[_1.AioDataFlowInstance](#user-defined-types)` | [variables('_1.aioDataFlowInstanceDefaults')] | no | +| aioMqBrokerConfig | The settings for the Azure IoT Operations MQ Broker. | `[_1.AioMqBroker](#user-defined-types)` | [variables('_1.aioMqBrokerDefaults')] | no | +| brokerListenerAnonymousConfig | Configuration for the insecure anonymous AIO MQ Broker Listener. | `[_1.AioMqBrokerAnonymous](#user-defined-types)` | [variables('_1.aioMqBrokerAnonymousDefaults')] | no | +| configurationSettingsOverride | Optional configuration settings to override default IoT Operations extension configuration. Use the same key names as the az iot ops --ops-config parameter. | `object` | {} | no | +| schemaRegistryName | The resource name for the ADR Schema Registry for Azure IoT Operations. | `string` | n/a | yes | +| adrNamespaceName | The resource name for the ADR Namespace for Azure IoT Operations. | `string` | n/a | no | +| shouldDeployAio | Whether to deploy an Azure IoT Operations Instance and all of its required components into the connected cluster. | `bool` | `true` | no | +| shouldDeployResourceSyncRules | Whether or not to deploy the Custom Locations Resource Sync Rules for the Azure IoT Operations resources. | `bool` | `true` | no | +| shouldCreateAnonymousBrokerListener | Whether to enable an insecure anonymous AIO MQ Broker Listener. (Should only be used for dev or test environments) | `bool` | `false` | no | +| shouldEnableOtelCollector | Whether or not to enable the Open Telemetry Collector for Azure IoT Operations. | `bool` | `true` | no | +| shouldEnableOpcUaSimulator | Whether or not to enable the OPC UA Simulator for Azure IoT Operations. | `bool` | `true` | no | +| shouldEnableAkriRestConnector | Deploy Akri REST HTTP Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriMediaConnector | Deploy Akri Media Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriOnvifConnector | Deploy Akri ONVIF Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriSseConnector | Deploy Akri SSE Connector template to the IoT Operations instance. | `bool` | `false` | no | +| customAkriConnectors | List of custom Akri connector templates with user-defined endpoint types and container images. | `array` | [] | no | +| akriMqttSharedConfig | Shared MQTT connection configuration for all Akri connectors. | `[_1.AkriMqttConfig](#user-defined-types)` | {'host': 'aio-broker:18883', 'audience': 'aio-internal', 'caConfigmap': 'azure-iot-operations-aio-ca-trust-bundle'} | no | +| customLocationName | The name for the Custom Locations resource. | `string` | [format('{0}-cl', parameters('arcConnectedClusterName'))] | no | +| additionalClusterExtensionIds | Additional cluster extension IDs to include in the custom location. (Appended to the default Secret Store and IoT Operations extension IDs) | `array` | [] | no | +| trustIssuerSettings | The trust issuer settings for Customer Managed Azure IoT Operations Settings. | `[_1.TrustIssuerConfig](#user-defined-types)` | {'trustSource': 'SelfSigned'} | no | +| sseKeyVaultName | The name of the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | n/a | yes | +| sseIdentityName | The name of the User Assigned Managed Identity for Secret Sync. | `string` | n/a | yes | +| sseKeyVaultResourceGroupName | The name of the Resource Group for the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | [resourceGroup().name] | no | +| shouldAssignSseKeyVaultRoles | Whether to assign roles for Key Vault to the provided Secret Sync Identity. | `bool` | `true` | no | +| shouldAssignDeployIdentityRoles | Whether to assign roles to the deploy identity. | `bool` | [not(empty(parameters('deployIdentityName')))] | no | +| deployIdentityName | The resource name for a managed identity that will be given deployment admin permissions. | `string` | n/a | no | +| shouldDeployAioDeploymentScripts | Whether to deploy DeploymentScripts for Azure IoT Operations. | `bool` | `false` | no | +| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | [parameters('sseKeyVaultName')] | no | +| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [parameters('sseKeyVaultResourceGroupName')] | no | +| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | +| deploymentScriptsSecretNamePrefix | The prefix used with constructing the secret name that will have the deployment script. | `string` | [format('{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | +| shouldAddDeployScriptsToKeyVault | Whether to add the deploy scripts for DeploymentScripts to Key Vault as secrets. (Required for DeploymentScripts) | `bool` | `false` | no | +| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | #### Resources for edgeIotOps @@ -122,24 +150,22 @@ Deploys Azure IoT Operations extensions, instances, and configurations on Azure #### Outputs for edgeIotOps -| Name | Type | Description | -|:------------------------------|:---------|:-------------------------------------------------------------------| -| containerStorageExtensionId | `string` | The ID of the Container Storage Extension. | -| containerStorageExtensionName | `string` | The name of the Container Storage Extension. | -| aioCertManagerExtensionId | `string` | The ID of the Azure IoT Operations Cert-Manager Extension. | -| aioCertManagerExtensionName | `string` | The name of the Azure IoT Operations Cert-Manager Extension. | -| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | -| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | -| customLocationId | `string` | The ID of the deployed Custom Location. | -| customLocationName | `string` | The name of the deployed Custom Location. | -| aioInstanceId | `string` | The ID of the deployed Azure IoT Operations instance. | -| aioInstanceName | `string` | The name of the deployed Azure IoT Operations instance. | -| dataFlowProfileId | `string` | The ID of the deployed Azure IoT Operations Data Flow Profile. | -| dataFlowProfileName | `string` | The name of the deployed Azure IoT Operations Data Flow Profile. | -| dataFlowEndpointId | `string` | The ID of the deployed Azure IoT Operations Data Flow Endpoint. | -| dataFlowEndpointName | `string` | The name of the deployed Azure IoT Operations Data Flow Endpoint. | -| akriConnectorTemplates | `array` | Map of deployed Akri connector templates by name with id and type. | -| akriConnectorTypesDeployed | `array` | List of Akri connector types that were deployed. | +| Name | Type | Description | +|:---------------------------|:---------|:-------------------------------------------------------------------| +| aioPlatformExtensionId | `string` | The ID of the Azure IoT Operations Platform Extension. | +| aioPlatformExtensionName | `string` | The name of the Azure IoT Operations Platform Extension. | +| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | +| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | +| customLocationId | `string` | The ID of the deployed Custom Location. | +| customLocationName | `string` | The name of the deployed Custom Location. | +| aioInstanceId | `string` | The ID of the deployed Azure IoT Operations instance. | +| aioInstanceName | `string` | The name of the deployed Azure IoT Operations instance. | +| dataFlowProfileId | `string` | The ID of the deployed Azure IoT Operations Data Flow Profile. | +| dataFlowProfileName | `string` | The name of the deployed Azure IoT Operations Data Flow Profile. | +| dataFlowEndpointId | `string` | The ID of the deployed Azure IoT Operations Data Flow Endpoint. | +| dataFlowEndpointName | `string` | The name of the deployed Azure IoT Operations Data Flow Endpoint. | +| akriConnectorTemplates | `array` | Map of deployed Akri connector templates by name with id and type. | +| akriConnectorTypesDeployed | `array` | List of Akri connector types that were deployed. | ### edgeAssets diff --git a/blueprints/only-edge-iot-ops/bicep/main.bicep b/blueprints/only-edge-iot-ops/bicep/main.bicep index 0a4210ec..8d262504 100644 --- a/blueprints/only-edge-iot-ops/bicep/main.bicep +++ b/blueprints/only-edge-iot-ops/bicep/main.bicep @@ -168,6 +168,13 @@ resource attribution 'Microsoft.Resources/deployments@2020-06-01' = if (!telemet Modules */ +module edgeArcExtensions '../../../src/100-edge/109-arc-extensions/bicep/main.bicep' = { + name: '${deployment().name}-eae0' + params: { + arcConnectedClusterName: arcConnectedClusterName + } +} + module edgeIotOps '../../../src/100-edge/110-iot-ops/bicep/main.bicep' = { name: '${deployment().name}-eio0' params: { @@ -230,10 +237,10 @@ module edgeAssets '../../../src/100-edge/111-assets/bicep/main.bicep' = { */ @description('The ID of the Azure IoT Operations Cert-Manager Extension.') -output aioCertManagerExtensionId string = edgeIotOps.outputs.aioCertManagerExtensionId +output aioCertManagerExtensionId string = edgeArcExtensions.outputs.certManagerExtensionId @description('The name of the Azure IoT Operations Cert-Manager Extension.') -output aioCertManagerExtensionName string = edgeIotOps.outputs.aioCertManagerExtensionName +output aioCertManagerExtensionName string = edgeArcExtensions.outputs.certManagerExtensionName @description('The ID of the Secret Store Extension.') output secretStoreExtensionId string = edgeIotOps.outputs.secretStoreExtensionId diff --git a/blueprints/only-output-cncf-cluster-script/bicep/README.md b/blueprints/only-output-cncf-cluster-script/bicep/README.md index 9c839fa3..092383bc 100644 --- a/blueprints/only-output-cncf-cluster-script/bicep/README.md +++ b/blueprints/only-output-cncf-cluster-script/bicep/README.md @@ -53,7 +53,7 @@ Generates scripts for Azure IoT Operations CNCF cluster creation without deployi ### edgeCncfCluster -This module provisions and deploys automation scripts to a VM host that create and configure a K3s Kubernetes cluster with Arc connectivity. +This module provisions and deploys automation scripts to a VM host that create and configure a K3s Kubernetes cluster with Arc connectivity. The scripts handle primary and secondary node(s) setup, cluster administration, workload identity enablement, and installation of required Azure Arc extensions. #### Parameters for edgeCncfCluster diff --git a/docs/_parts/infrastructure-sidebar.md b/docs/_parts/infrastructure-sidebar.md index 89351184..9ace62eb 100644 --- a/docs/_parts/infrastructure-sidebar.md +++ b/docs/_parts/infrastructure-sidebar.md @@ -11,6 +11,7 @@ - [Data](/src/000-cloud/030-data/README) - [Fabric](/src/000-cloud/031-fabric/README) - [Fabric Rti](/src/000-cloud/032-fabric-rti/README) + - [Fabric Ontology](/src/000-cloud/033-fabric-ontology/README) - [Postgresql](/src/000-cloud/035-postgresql/README) - [Managed Redis](/src/000-cloud/036-managed-redis/README) - [Messaging](/src/000-cloud/040-messaging/README) @@ -22,6 +23,7 @@ - [Edge](/src/100-edge/README) - [Cncf Cluster](/src/100-edge/100-cncf-cluster/README) + - [Arc Extensions](/src/100-edge/109-arc-extensions/README) - [Iot Ops](/src/100-edge/110-iot-ops/README) - [Assets](/src/100-edge/111-assets/README) - [Observability](/src/100-edge/120-observability/README) diff --git a/docs/_sidebar.md b/docs/_sidebar.md index 06f70db5..22a1ac0c 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -376,6 +376,8 @@ - [Data](/src/000-cloud/030-data/README) - [Fabric](/src/000-cloud/031-fabric/README) - [Fabric Rti](/src/000-cloud/032-fabric-rti/README) + - [Fabric Ontology](/src/000-cloud/033-fabric-ontology/README) + - [Fabric Ontology Dim](/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/README) - [Postgresql](/src/000-cloud/035-postgresql/README) - [Managed Redis](/src/000-cloud/036-managed-redis/README) - [Messaging](/src/000-cloud/040-messaging/README) @@ -386,6 +388,7 @@ - [Ai Foundry](/src/000-cloud/085-ai-foundry/README) - [Edge](/src/100-edge/README) - [Cncf Cluster](/src/100-edge/100-cncf-cluster/README) + - [Arc Extensions](/src/100-edge/109-arc-extensions/README) - [Iot Ops](/src/100-edge/110-iot-ops/README) - [Assets](/src/100-edge/111-assets/README) - [Observability](/src/100-edge/120-observability/README) @@ -435,6 +438,7 @@ - [Vm Host](/src/000-cloud/073-vm-host/bicep/README) - [Ai Foundry](/src/000-cloud/085-ai-foundry/bicep/README) - [Cncf Cluster](/src/100-edge/100-cncf-cluster/bicep/README) + - [Arc Extensions](/src/100-edge/109-arc-extensions/bicep/README) - [Iot Ops](/src/100-edge/110-iot-ops/bicep/README) - [Assets](/src/100-edge/111-assets/bicep/README) - [Observability](/src/100-edge/120-observability/bicep/README) @@ -505,6 +509,9 @@ - [Ubuntu K3s](/src/100-edge/100-cncf-cluster/terraform/modules/ubuntu-k3s/README) - [Vm Script Deployment](/src/100-edge/100-cncf-cluster/terraform/modules/vm-script-deployment/README) - [Ci](/src/100-edge/100-cncf-cluster/ci/terraform/README) + - [Arc Extensions](/src/100-edge/109-arc-extensions/terraform/README) + - [Cert Manager](/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/README) + - [Container Storage](/src/100-edge/109-arc-extensions/terraform/modules/container-storage/README) - [Iot Ops](/src/100-edge/110-iot-ops/terraform/README) - [Akri Connectors](/src/100-edge/110-iot-ops/terraform/modules/akri-connectors/README) - [Apply Scripts](/src/100-edge/110-iot-ops/terraform/modules/apply-scripts/README) diff --git a/docs/build-cicd/azure-pipelines/templates/aio-version-checker-template.md b/docs/build-cicd/azure-pipelines/templates/aio-version-checker-template.md index d0bb41c8..6c04d1fb 100644 --- a/docs/build-cicd/azure-pipelines/templates/aio-version-checker-template.md +++ b/docs/build-cicd/azure-pipelines/templates/aio-version-checker-template.md @@ -101,7 +101,7 @@ The version checker verifies these components: | Component | Terraform Name | Bicep Name | Remote Manifest Source | |----------------|------------------------|---------------------------------|------------------------| -| Cert Manager | cert_manager | aioCertManagerExtensionDefaults | enablement | +| Cert Manager | cert_manager_extension | aioCertManagerExtensionDefaults | enablement | | Secret Store | secret_sync_controller | secretStoreExtensionDefaults | enablement | | IoT Operations | azure-iot-operations | aioExtensionDefaults | instance | diff --git a/docs/getting-started/general-user.md b/docs/getting-started/general-user.md index 0c749480..14f9ed11 100644 --- a/docs/getting-started/general-user.md +++ b/docs/getting-started/general-user.md @@ -225,7 +225,7 @@ blueprints/ ```bash # Set required environment variable for Terraform Azure provider by running the script - ../../../scripts/az-sub-init.sh + source ../../../scripts/az-sub-init.sh # Initialize Terraform (pulls down providers and modules) terraform init -upgrade diff --git a/scripts/az-sub-init.sh b/scripts/az-sub-init.sh index a5fb13ce..191c1bf2 100755 --- a/scripts/az-sub-init.sh +++ b/scripts/az-sub-init.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash tenant="" -help="Usage: az-sub-init.sh [--tenant your-tenant.onmicrosoft.com] [--help] +help="Usage: source az-sub-init.sh [--tenant your-tenant.onmicrosoft.com] [--help] Attempts to set the ARM_SUBSCRIPTION_ID env var to 'id' from 'az account show' in the following ways: - 'az login' if not logged in (optionally with specific tenant) diff --git a/src/000-cloud/033-fabric-ontology/README.md b/src/000-cloud/033-fabric-ontology/README.md new file mode 100644 index 00000000..2e49d9c1 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/README.md @@ -0,0 +1,529 @@ +--- +title: Fabric Ontology Deployment Component +description: Schema-driven deployment of Microsoft Fabric Ontology resources from portable YAML definitions +author: Edge AI Team +ms.date: 2026-01-09 +ms.topic: reference +keywords: + - fabric + - ontology + - lakehouse + - eventhouse + - semantic-model + - robotics + - ieee-1872 +estimated_reading_time: 10 +--- + +## Fabric Ontology Deployment Component + +Generic, schema-driven Fabric ontology deployment component that provisions Lakehouse, Eventhouse, Semantic Model, and Ontology resources from a portable YAML definition file. + +**Microsoft Documentation:** + +- [Ontology Overview](https://learn.microsoft.com/fabric/iq/ontology/overview) - Concepts and capabilities +- [Ontology Definition API](https://learn.microsoft.com/rest/api/fabric/articles/item-management/definitions/ontology-definition) - REST API reference + +### Quick Start - Deploy IEEE 1872 Robotics Ontology + +Deploy a complete IEEE 1872 CORA/CORAX robotics ontology with sample robots, environments, and position measurements: + +```bash +cd src/000-cloud/033-fabric-ontology/scripts + +# Deploy ontology with sample robotics data (19 tables) +./deploy-cora-corax-dim.sh \ + --workspace-id \ + --with-seed-data + +# Dry run (preview without changes) +./deploy-cora-corax-dim.sh \ + --workspace-id \ + --with-seed-data \ + --dry-run +``` + +This deploys: + +- **Lakehouse** `RoboticsOntologyLH` with 19 Delta tables (12 entities + 7 relationships) +- **Semantic Model** `CORA_CORAX_DimensionalModel` (Direct Lake) +- **Ontology** `CORA_CORAX_Dimensional` with IEEE 1872 entity types + +**Prerequisites**: `az login`, `yq`, `jq`, `curl`, Fabric workspace with **capacity assigned** and Ontology preview enabled. + +**Sample Questions to Test:** + +- "What robots do we have?" +- "Which robots are in the Welding Cell group?" +- "What is the current position of the KUKA KR 16?" +- "Show robots in robotic systems equipped in Factory Floor North" + +See [fabric-ontology-dim/README.md](fabric-ontology-dim/README.md) for complete documentation, seed data details, and more sample questions. + +### Quick Start - Deploy Custom Ontology + +Deploy your own ontology with local data using the generic deployment script: + +```bash +cd src/000-cloud/033-fabric-ontology + +# Deploy custom ontology with local CSV/Parquet data +./scripts/deploy.sh \ + --definition ./my-ontology.yaml \ + --workspace-id \ + --data-dir ./my-data/ +``` + +This creates everything from scratch: + +- **Lakehouse** with tables loaded from your `--data-dir` +- **Semantic Model** (Direct Lake) for Power BI +- **Ontology** with entity types and relationships + +File names in `--data-dir` must match table names in your YAML (e.g., `robot.csv` for table `robot`). + +See [definitions/examples/cora-corax-dim.yaml](definitions/examples/cora-corax-dim.yaml) for a complete example definition. + +### Overview + +This component enables deployment of complete Microsoft Fabric ontologies using a declarative YAML definition format. Rather than hardcoding specific entities and relationships, you define your ontology schema in YAML and the deployment scripts automatically: + +1. Create Lakehouse and load CSV/Parquet data as Delta tables (optional) +2. Create Eventhouse and KQL database with time-series data (optional) +3. Generate and deploy Direct Lake Semantic Model (optional) +4. Create Ontology with entity types, properties, data bindings, and relationships + +### Deployment Modes + +#### Generic Deployment (Recommended for Custom Ontologies) + +Use `deploy.sh` for one-command deployment of any ontology definition with local data: + +```bash +# Deploy everything: Lakehouse, data, semantic model, ontology +./scripts/deploy.sh \ + --definition ./my-ontology.yaml \ + --workspace-id \ + --data-dir ./my-data/ + +# Skip specific steps +./scripts/deploy.sh \ + --definition ./my-ontology.yaml \ + --workspace-id \ + --data-dir ./my-data/ \ + --skip-semantic-model # Don't create semantic model + +# Use existing Lakehouse (skip data loading) +./scripts/deploy.sh \ + --definition ./my-ontology.yaml \ + --workspace-id \ + --skip-data-sources \ + --lakehouse-id + +# Dry run to preview deployment +./scripts/deploy.sh \ + --definition ./my-ontology.yaml \ + --workspace-id \ + --data-dir ./my-data/ \ + --dry-run +``` + +#### Bind to Existing Data (Most Common) + +If your Lakehouse tables already exist, you only need to deploy the ontology with bindings. The `dataSources` section in the YAML describes which tables to bind to - it does **not** create them. + +```bash +# Deploy ontology that binds to existing Lakehouse tables +./scripts/deploy-ontology.sh \ + --definition definitions/examples/cora-corax-dim.yaml \ + --workspace-id \ + --lakehouse-id +``` + +**Requirements:** + +- Lakehouse must already exist with tables matching the `dataBinding.table` names in the definition +- Column names must match the `sourceColumn` values in entity properties + +#### Full Deployment (Create Everything) + +If you need to create data sources and load sample data: + +```bash +# 1. Deploy data sources (creates Lakehouse, uploads CSV, converts to Delta) +./scripts/deploy-data-sources.sh \ + --definition definitions/examples/cora-corax-dim.yaml \ + --workspace-id \ + --data-dir ../fabric-ontology-dim/seed/ + +# 2. Deploy semantic model (optional, for Power BI) +./scripts/deploy-semantic-model.sh \ + --definition definitions/examples/cora-corax-dim.yaml \ + --workspace-id \ + --lakehouse-id + +# 3. Deploy ontology +./scripts/deploy-ontology.sh \ + --definition definitions/examples/cora-corax-dim.yaml \ + --workspace-id \ + --lakehouse-id +``` + +### Usage + +The component can be used standalone against any Fabric workspace. + +#### Bind to Existing Data + +```bash +# Validate definition before deployment +./scripts/validate-definition.sh --definition definitions/examples/cora-corax-dim.yaml + +# Deploy ontology with Lakehouse bindings only +./scripts/deploy-ontology.sh \ + --definition definitions/examples/cora-corax-dim.yaml \ + --workspace-id \ + --lakehouse-id + +# Deploy ontology with Lakehouse + Eventhouse bindings (for time-series data) +./scripts/deploy-ontology.sh \ + --definition my-ontology.yaml \ + --workspace-id \ + --lakehouse-id \ + --eventhouse-id \ + --cluster-uri https://xyz.kusto.fabric.microsoft.com +``` + +### Tutorial: Deploying the CORA/CORAX Robotics Ontology + +This walkthrough deploys a complete IEEE 1872 robotics ontology with sample data. + +#### One-Command Deployment + +```bash +cd src/000-cloud/033-fabric-ontology/scripts + +# Deploy everything with a single command +./deploy-cora-corax-dim.sh --workspace-id --with-seed-data +``` + +This script orchestrates all steps automatically. For manual step-by-step deployment, continue reading. + +#### Step 1: Prerequisites + +1. **Azure CLI**: Install and authenticate + + ```bash + az login + ``` + +2. **Tools**: Install `yq` and `jq` + + ```bash + # Windows (winget) + winget install jqlang.jq + winget install MikeFarah.yq + + # Windows (choco) + choco install jq yq + + # Ubuntu/WSL + sudo apt-get install jq + sudo snap install yq + + # macOS + brew install jq yq + ``` + +3. **Fabric Workspace**: You need an existing Fabric workspace. Get the workspace ID from the URL: + + ```text + https://app.fabric.microsoft.com/groups//... + ``` + +4. **Fabric Capacity**: The workspace must have Fabric capacity assigned: + - Open workspace settings → License info + - Under "License mode", select **Fabric capacity** (not Pro or PPU) + - Assign an F-SKU capacity (F2 minimum for ontology features) + - Without capacity, Lakehouse and Ontology APIs will fail + +5. **Tenant Settings**: Ensure these are enabled in Fabric Admin Portal: + - Ontology (Preview) + - XMLA read-write endpoints + - Graph and Copilot for Ontology + +#### Step 2: Validate the Definition + +```bash +cd src/000-cloud/033-fabric-ontology + +# Validate the definition file +./scripts/validate-definition.sh --definition definitions/examples/cora-corax-dim.yaml +``` + +Expected output: + +```text +✓ API version valid: fabric.ontology/v1 +✓ Metadata valid: CORA_CORAX_Dimensional +✓ Entity types valid: 12 entities +✓ Relationships valid: 7 relationships +✓ Definition is valid +``` + +#### Step 3: Deploy with Seed Data + +```bash +cd scripts + +./deploy-cora-corax-dim.sh \ + --workspace-id \ + --with-seed-data +``` + +The script will: + +1. Create Lakehouse `RoboticsOntologyLH` (or use existing) +2. Upload 19 CSV files from `fabric-ontology-dim/seed/` +3. Convert to Delta tables +4. Create Direct Lake Semantic Model +5. Create Ontology with 12 entity types and 7 relationships + +#### Step 4: Verify in Fabric Portal + +1. Go to your Fabric workspace +2. Find `CORA_CORAX_Dimensional` in the items list +3. Open it to see the entity types and relationships + +#### Step 5: Create Data Agent (Manual) + +The Data Agent provides natural language Q&A over your ontology. **This step must be done manually** - Fabric Data Agents do not have REST API support. + +1. **Create the Data Agent**: + - In your workspace, click **+ New item** → Search for **Data agent** + - Name: `RoboticsOntologyAgent` + - Click **Create** + +2. **Add the Ontology as Data Source**: + - Select `CORA_CORAX_Dimensional` from OneLake data hub + - Click **Add** then wait for the agent to initialize + +3. **Add Required Instruction** (IMPORTANT): + - Click **Agent instructions** in the menu ribbon + - Add this text at the bottom: + + ```text + Support group by in GQL + ``` + + - This enables aggregation queries (known issue workaround per [Microsoft docs](https://learn.microsoft.com/fabric/iq/ontology/tutorial-4-create-data-agent#provide-agent-instructions)) + +4. **Test the Agent**: + - "What robots do we have?" + - "Which robots are in the Assembly Line group?" + - "What is the current position of the KUKA KR 16?" + +> **Tip**: If queries return "no data" errors, wait a few minutes for agent initialization then retry. +> +> **Note**: Data Agent is a preview feature requiring F2+ Fabric capacity and tenant admin settings for Fabric Data Agent, Cross-geo processing for AI, and Cross-geo storing for AI. + +#### Data Sources Reference + +The CORA/CORAX ontology uses IEEE 1872 robotics seed data: + +| Entity Type | Table | Rows | Description | +|--------------------|--------------------|------|------------------------------------------| +| Robot | robot | 3 | ABB IRB 6700, KUKA KR 16, Fanuc M-20iA | +| RobotGroup | robotgroup | 2 | Welding Cell, Assembly Line | +| RoboticSystem | roboticsystem | 2 | Production Line A, Production Line B | +| RoboticEnvironment | roboticenvironment | 2 | Factory Floor North, Factory Floor South | +| PositionMeasure | positionmeasure | 3 | X, Y, Z positions in meters | +| OrientationMeasure | orientationmeasure | 3 | Roll, Pitch, Yaw in radians | +| PoseMeasure | posemeasure | 3 | Combined position + orientation | + +See [fabric-ontology-dim/README.md](fabric-ontology-dim/README.md) for complete entity and relationship reference. + +### Environment Variables + +The deployment scripts use Azure CLI authentication. Set the following before running: + +```bash +# Authenticate to Azure +az login + +# Scripts automatically obtain tokens for: +# - https://api.fabric.microsoft.com (Fabric REST API) +# - https://storage.azure.com (OneLake DFS API) +``` + +### Prerequisites + +- Azure CLI with logged-in session (`az login`) +- [`yq`](https://github.com/mikefarah/yq) - YAML parser +- `jq` - JSON processor +- `curl` - HTTP client +- Fabric workspace with **Fabric capacity assigned** (F2 or higher SKU) + - Workspace Settings → License info → License mode: Fabric capacity +- Tenant admin settings enabled: Ontology preview, XMLA endpoints, Graph preview + +### Creating New Ontologies + +To deploy a new ontology, create a YAML definition file following the schema format. + +#### Definition Schema + +```yaml +apiVersion: fabric.ontology/v1 +kind: OntologyDefinition + +metadata: + name: "MyOntology" # Required: Ontology display name (letters, numbers, underscores only) + description: "Description" # Optional: Ontology description + version: "1.0.0" # Optional: Version string + +dataSources: + lakehouse: # Required for static data bindings + name: "MyLakehouse" + tables: + - name: "tablename" + format: "csv" # csv or parquet + eventhouse: # Optional: for time-series bindings + name: "MyEventhouse" + database: "MyKqlDatabase" + tables: + - name: "telemetry" + format: "csv" + +entityTypes: + - name: "EntityName" # Required: Entity type name + key: "IdProperty" # Required: Primary key property + displayName: "NameProp" # Optional: Display name property + dataBinding: + type: "static" # static (Lakehouse) or timeseries (Eventhouse) + source: "lakehouse" # lakehouse or eventhouse + table: "tablename" # Source table name + properties: + - name: "PropertyName" # Required: Property name + type: "string" # string, int, double, datetime, boolean + sourceColumn: "Column" # Column name in source table + +relationships: + - name: "relName" # Required: Relationship name + from: "EntityA" # Required: Source entity type + to: "EntityB" # Required: Target entity type + cardinality: "one-to-many" # one-to-one, one-to-many, many-to-many + binding: + table: "jointable" # Table containing relationship data + fromColumn: "FromKey" # Column for source entity key + toColumn: "ToKey" # Column for target entity key +``` + +> **Note**: Ontology names must start with a letter, be less than 90 characters, and contain only letters, numbers, and underscores. No hyphens allowed. + +#### Supported Property Types + +| Definition Type | Fabric Ontology Type | KQL Type | TMDL Type | +|-----------------|----------------------|------------|------------| +| `string` | `String` | `string` | `String` | +| `int` | `Int64` | `int` | `Int64` | +| `double` | `Double` | `real` | `Double` | +| `datetime` | `DateTime` | `datetime` | `DateTime` | +| `boolean` | `Boolean` | `bool` | `Boolean` | + +#### Example Definition + +See [definitions/examples/cora-corax-dim.yaml](definitions/examples/cora-corax-dim.yaml) - IEEE 1872 CORA/CORAX robotics ontology with 12 entity types and 7 relationships. + +### Troubleshooting + +#### Ontology Shows "Setting up your ontology" for Extended Period + +This is **normal behavior**. Ontology creation is an asynchronous operation that can take **10-20 minutes** to complete, even though the API returns success immediately. + +**What's happening:** + +- The Fabric API creates the ontology shell immediately +- Entity types and relationships are provisioned asynchronously in the background +- The portal shows "Setting up your ontology" until all entity types are ready + +**How to verify completion:** + +```bash +# Get token and ontology ID +TOKEN=$(az account get-access-token --resource "https://api.fabric.microsoft.com" --query accessToken -o tsv) +WORKSPACE_ID="" +ONTOLOGY_ID=$(curl -s -H "Authorization: Bearer $TOKEN" \ + "https://api.fabric.microsoft.com/v1/workspaces/$WORKSPACE_ID/ontologies" | jq -r '.value[0].id') + +# Check if entity types exist (returns "EntityNotFound" while still processing) +curl -s -H "Authorization: Bearer $TOKEN" \ + "https://api.fabric.microsoft.com/v1/workspaces/$WORKSPACE_ID/ontologies/$ONTOLOGY_ID/entityTypes" | jq . +``` + +- **Still processing**: Returns `{"errorCode": "EntityNotFound", ...}` +- **Complete**: Returns array of entity types + +**Recommendation**: Wait 10-20 minutes after deployment before checking the ontology in the Fabric portal or creating a Data Agent. + +#### Common Errors + +**BadArtifactCreateRequest - Item name must start with a letter**: + +- Ontology names cannot contain hyphens +- Use underscores instead: `My_Ontology` not `My-Ontology` + +**AADSTS50173: The provided grant has expired** (Kusto/Eventhouse operations): + +- The Kusto API requires a different token scope than Fabric API +- Run `az logout` then `az login` to refresh all tokens +- If Lakehouse was already created, skip data sources: `--skip-data-sources --lakehouse-id ` + +**ALMOperationImportFailed**: The ontology definition JSON format is incorrect. + +- Ensure entity types have `$schema` property +- Ensure `.platform` has both `$schema` and `config` blocks +- Ensure Lakehouse bindings use `sourceSchema: null` (not `"dbo"`) + +**Operation timeout**: Long-running operations may take 60-120 seconds. + +- The scripts poll with a 300-second timeout +- Check Fabric portal for operation status + +**Authentication failed**: Token acquisition issues. + +- Run `az login` to refresh credentials +- Ensure you have Fabric workspace contributor access + +**Definition validation failed**: YAML schema issues. + +- Run `./scripts/validate-definition.sh --definition ` for details +- Check entity keys reference valid properties +- Check relationships reference valid entity types + +### Component Structure + +```text +033-fabric-ontology/ +├── README.md # This documentation +├── definitions/ # Ontology definition files +│ ├── schema.json # JSON Schema for validation +│ └── examples/ # Example definitions +│ └── cora-corax-dim.yaml # IEEE 1872 CORA/CORAX robotics ontology +├── fabric-ontology-dim/ # CORA/CORAX starter kit +│ ├── README.md # Detailed documentation and sample questions +│ ├── json-schema/ # JSON Schema validation files +│ └── seed/ # Sample CSV data (19 tables) +├── scripts/ # Deployment scripts +│ ├── lib/ # Shared libraries (logging, parser, API) +│ ├── deploy.sh # Generic one-command deployment +│ ├── deploy-cora-corax-dim.sh # CORA/CORAX robotics deployment +│ ├── validate-definition.sh # Schema validation +│ ├── deploy-data-sources.sh # Lakehouse + Eventhouse creation +│ ├── deploy-semantic-model.sh # TMDL generation +│ ├── deploy-ontology.sh # Ontology creation +│ └── dump-parts.sh # Debug utility for ontology parts +└── templates/ # Template files + ├── semantic-model/ # TMDL templates + ├── kql/ # KQL templates + └── ontology/ # Ontology JSON templates +``` diff --git a/src/000-cloud/033-fabric-ontology/definitions/examples/cora-corax-dim.yaml b/src/000-cloud/033-fabric-ontology/definitions/examples/cora-corax-dim.yaml new file mode 100644 index 00000000..5609093e --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/definitions/examples/cora-corax-dim.yaml @@ -0,0 +1,469 @@ +apiVersion: fabric.ontology/v1 +kind: OntologyDefinition + +metadata: + name: "CORA_CORAX_Dimensional" + description: "IEEE 1872 CORA/CORAX Robotics Ontology - Dimensional Schema" + version: "1.0.0" + +dataSources: + lakehouse: + name: "RoboticsOntologyLH" + tables: + # 12 Entity tables (lowercase names for Lakehouse) + - name: "robot" + format: "csv" + - name: "robotgroup" + format: "csv" + - name: "roboticsystem" + format: "csv" + - name: "roboticenvironment" + format: "csv" + - name: "robotinterface" + format: "csv" + - name: "processingdevice" + format: "csv" + - name: "physicalenvironment" + format: "csv" + - name: "positioncoordinatesystem" + format: "csv" + - name: "orientationcoordinatesystem" + format: "csv" + - name: "positionmeasure" + format: "csv" + - name: "orientationmeasure" + format: "csv" + - name: "posemeasure" + format: "csv" + # 7 Relationship tables + - name: "robot_robotgroup" + format: "csv" + - name: "robot_roboticsystem" + format: "csv" + - name: "roboticenvironment_roboticsystem" + format: "csv" + - name: "coordinatesystem_transformations" + format: "csv" + - name: "robot_positionmeasure" + format: "csv" + - name: "robot_orientationmeasure" + format: "csv" + - name: "robot_posemeasure" + format: "csv" + +entityTypes: + # --- CORA Entities --- + - name: "Robot" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "robot" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + + - name: "RobotGroup" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "robotgroup" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + + - name: "RoboticSystem" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "roboticsystem" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + + - name: "RoboticEnvironment" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "roboticenvironment" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + + - name: "RobotInterface" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "robotinterface" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + + # --- CORAX Entities --- + - name: "ProcessingDevice" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "processingdevice" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + + - name: "PhysicalEnvironment" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "physicalenvironment" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + + # --- POS Entities --- + - name: "PositionCoordinateSystem" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "positioncoordinatesystem" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + + - name: "OrientationCoordinateSystem" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "orientationcoordinatesystem" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + + - name: "PositionMeasure" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "positionmeasure" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "X" + type: "double" + sourceColumn: "X" + - name: "Y" + type: "double" + sourceColumn: "Y" + - name: "Z" + type: "double" + sourceColumn: "Z" + - name: "CoordinateSystemId" + type: "string" + sourceColumn: "CoordinateSystemId" + - name: "Timestamp" + type: "string" + sourceColumn: "Timestamp" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + + - name: "OrientationMeasure" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "orientationmeasure" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "Roll" + type: "double" + sourceColumn: "Roll" + - name: "Pitch" + type: "double" + sourceColumn: "Pitch" + - name: "Yaw" + type: "double" + sourceColumn: "Yaw" + - name: "CoordinateSystemId" + type: "string" + sourceColumn: "CoordinateSystemId" + - name: "Timestamp" + type: "string" + sourceColumn: "Timestamp" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + + - name: "PoseMeasure" + key: "Id" + displayName: "Name" + dataBinding: + type: "static" + source: "lakehouse" + table: "posemeasure" + properties: + - name: "Id" + type: "string" + sourceColumn: "Id" + - name: "Name" + type: "string" + sourceColumn: "Name" + - name: "Description" + type: "string" + sourceColumn: "Description" + - name: "X" + type: "double" + sourceColumn: "X" + - name: "Y" + type: "double" + sourceColumn: "Y" + - name: "Z" + type: "double" + sourceColumn: "Z" + - name: "Roll" + type: "double" + sourceColumn: "Roll" + - name: "Pitch" + type: "double" + sourceColumn: "Pitch" + - name: "Yaw" + type: "double" + sourceColumn: "Yaw" + - name: "CoordinateSystemId" + type: "string" + sourceColumn: "CoordinateSystemId" + - name: "Timestamp" + type: "string" + sourceColumn: "Timestamp" + - name: "Namespace" + type: "string" + sourceColumn: "Namespace" + - name: "Source" + type: "string" + sourceColumn: "Source" + +relationships: + - name: "member" + description: "Robot is member of RobotGroup" + from: "Robot" + to: "RobotGroup" + cardinality: "many-to-many" + binding: + table: "robot_robotgroup" + fromColumn: "FromId" + toColumn: "ToId" + + - name: "partOf" + description: "Robot is part of RoboticSystem" + from: "Robot" + to: "RoboticSystem" + cardinality: "many-to-many" + binding: + table: "robot_roboticsystem" + fromColumn: "FromId" + toColumn: "ToId" + + - name: "equippedWith" + description: "RoboticEnvironment is equipped with RoboticSystem" + from: "RoboticEnvironment" + to: "RoboticSystem" + cardinality: "many-to-many" + binding: + table: "roboticenvironment_roboticsystem" + fromColumn: "FromId" + toColumn: "ToId" + + - name: "transform" + description: "PositionCoordinateSystem transforms to OrientationCoordinateSystem" + from: "PositionCoordinateSystem" + to: "OrientationCoordinateSystem" + cardinality: "many-to-many" + binding: + table: "coordinatesystem_transformations" + fromColumn: "FromId" + toColumn: "ToId" + + - name: "hasPosition" + description: "Robot has PositionMeasure" + from: "Robot" + to: "PositionMeasure" + cardinality: "many-to-many" + binding: + table: "robot_positionmeasure" + fromColumn: "FromId" + toColumn: "ToId" + + - name: "hasOrientation" + description: "Robot has OrientationMeasure" + from: "Robot" + to: "OrientationMeasure" + cardinality: "many-to-many" + binding: + table: "robot_orientationmeasure" + fromColumn: "FromId" + toColumn: "ToId" + + - name: "hasPose" + description: "Robot has PoseMeasure" + from: "Robot" + to: "PoseMeasure" + cardinality: "many-to-many" + binding: + table: "robot_posemeasure" + fromColumn: "FromId" + toColumn: "ToId" diff --git a/src/000-cloud/033-fabric-ontology/definitions/schema.json b/src/000-cloud/033-fabric-ontology/definitions/schema.json new file mode 100644 index 00000000..314c4e31 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/definitions/schema.json @@ -0,0 +1,411 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Fabric Ontology Definition", + "description": "Schema for defining Microsoft Fabric Ontology deployments", + "type": "object", + "required": ["apiVersion", "kind", "metadata", "entityTypes"], + "additionalProperties": false, + "properties": { + "apiVersion": { + "type": "string", + "const": "fabric.ontology/v1", + "description": "Schema version identifier" + }, + "kind": { + "type": "string", + "const": "OntologyDefinition", + "description": "Resource kind" + }, + "metadata": { + "$ref": "#/definitions/metadata" + }, + "dataSources": { + "$ref": "#/definitions/dataSources" + }, + "entityTypes": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/entityType" + }, + "description": "Entity type definitions" + }, + "relationships": { + "type": "array", + "items": { + "$ref": "#/definitions/relationship" + }, + "description": "Relationship definitions between entity types" + }, + "semanticModel": { + "$ref": "#/definitions/semanticModel" + } + }, + "definitions": { + "metadata": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "Ontology display name" + }, + "description": { + "type": "string", + "maxLength": 1024, + "description": "Ontology description" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$", + "description": "Semantic version" + } + } + }, + "dataSources": { + "type": "object", + "additionalProperties": false, + "properties": { + "lakehouse": { + "$ref": "#/definitions/lakehouseDataSource" + }, + "eventhouse": { + "$ref": "#/definitions/eventhouseDataSource" + } + }, + "description": "Data source configurations" + }, + "lakehouseDataSource": { + "type": "object", + "required": ["name", "tables"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Lakehouse display name" + }, + "tables": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/lakehouseTable" + } + } + } + }, + "lakehouseTable": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Table name in Lakehouse" + }, + "sourceFile": { + "type": "string", + "description": "Relative path to source data file" + }, + "sourceUrl": { + "type": "string", + "format": "uri", + "description": "URL to download source data file" + }, + "format": { + "type": "string", + "enum": ["csv", "parquet"], + "default": "csv", + "description": "Source file format" + }, + "options": { + "type": "object", + "additionalProperties": true, + "properties": { + "header": { + "type": "boolean", + "default": true + }, + "delimiter": { + "type": "string", + "default": "," + } + } + } + } + }, + "eventhouseDataSource": { + "type": "object", + "required": ["name", "database", "tables"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Eventhouse display name" + }, + "database": { + "type": "string", + "minLength": 1, + "description": "KQL database name" + }, + "tables": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/eventhouseTable" + } + } + } + }, + "eventhouseTable": { + "type": "object", + "required": ["name", "schema"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "KQL table name" + }, + "sourceFile": { + "type": "string", + "description": "Relative path to source data file" + }, + "sourceUrl": { + "type": "string", + "format": "uri", + "description": "URL to download source data file" + }, + "format": { + "type": "string", + "enum": ["csv", "parquet", "json"], + "default": "csv" + }, + "schema": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/kqlColumn" + }, + "description": "Explicit KQL table schema" + }, + "policies": { + "type": "object", + "additionalProperties": false, + "properties": { + "retention": { + "type": "string", + "pattern": "^[0-9]+d$", + "default": "30d", + "description": "Data retention period (e.g., 30d)" + }, + "caching": { + "type": "string", + "pattern": "^[0-9]+d$", + "default": "7d", + "description": "Hot cache period (e.g., 7d)" + } + } + } + } + }, + "kqlColumn": { + "type": "object", + "required": ["name", "type"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Column name" + }, + "type": { + "type": "string", + "enum": ["string", "int", "double", "datetime", "boolean", "object"], + "description": "Column data type" + } + } + }, + "entityType": { + "type": "object", + "required": ["name", "key", "properties"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "Entity type name" + }, + "key": { + "type": "string", + "minLength": 1, + "description": "Primary key property name (must match a property name)" + }, + "displayName": { + "type": "string", + "description": "Property used for entity display name" + }, + "dataBinding": { + "$ref": "#/definitions/entityDataBinding", + "description": "Single data binding (use dataBindings for multiple)" + }, + "dataBindings": { + "type": "array", + "items": { + "$ref": "#/definitions/entityDataBinding" + }, + "description": "Multiple data bindings (static + timeseries)" + }, + "properties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/entityProperty" + }, + "description": "Entity properties" + } + } + }, + "entityDataBinding": { + "type": "object", + "required": ["type", "source", "table"], + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": ["static", "timeseries"], + "description": "Binding type" + }, + "source": { + "type": "string", + "enum": ["lakehouse", "eventhouse"], + "description": "Data source type" + }, + "table": { + "type": "string", + "minLength": 1, + "description": "Source table name" + }, + "timestampColumn": { + "type": "string", + "description": "Timestamp column for timeseries bindings" + }, + "correlationColumn": { + "type": "string", + "description": "Column linking timeseries to entity key" + } + } + }, + "entityProperty": { + "type": "object", + "required": ["name", "type"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "Property name" + }, + "type": { + "type": "string", + "enum": ["string", "int", "double", "datetime", "boolean", "object"], + "description": "Property data type" + }, + "sourceColumn": { + "type": "string", + "description": "Source table column name (defaults to property name)" + }, + "binding": { + "type": "string", + "enum": ["static", "timeseries"], + "description": "Which binding this property comes from (for multi-binding entities)" + } + } + }, + "relationship": { + "type": "object", + "required": ["name", "from", "to"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "Relationship name" + }, + "description": { + "type": "string", + "maxLength": 1024, + "description": "Relationship description" + }, + "from": { + "type": "string", + "minLength": 1, + "description": "Source entity type name" + }, + "to": { + "type": "string", + "minLength": 1, + "description": "Target entity type name" + }, + "cardinality": { + "type": "string", + "enum": ["one-to-one", "one-to-many", "many-to-many"], + "default": "one-to-many", + "description": "Relationship cardinality" + }, + "binding": { + "type": "object", + "required": ["table", "fromColumn", "toColumn"], + "additionalProperties": false, + "properties": { + "table": { + "type": "string", + "description": "Table containing the foreign key relationship" + }, + "fromColumn": { + "type": "string", + "description": "Column matching source entity key" + }, + "toColumn": { + "type": "string", + "description": "Column matching target entity key" + } + } + } + } + }, + "semanticModel": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "Semantic model display name" + }, + "mode": { + "type": "string", + "enum": ["directLake", "import"], + "default": "directLake", + "description": "Semantic model storage mode" + }, + "generateFrom": { + "type": "string", + "enum": ["definition", "lakehouse"], + "default": "definition", + "description": "How to generate tables - from this definition or auto-detect from lakehouse" + } + } + } + } +} diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/README.md b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/README.md new file mode 100644 index 00000000..bffc3347 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/README.md @@ -0,0 +1,207 @@ + +# Fabric Ontology Dimensional Schema - CORA/CORAX Starter Kit + +IEEE 1872 CORA/CORAX Robotics Ontology implementation for Microsoft Fabric Ontology (preview). + +## Overview + +This starter kit provides a complete dimensional schema based on IEEE 1872 robotics standards: + +- **CORA** (Core Ontology for Robotics and Automation): Robot, RobotGroup, RoboticSystem, RoboticEnvironment +- **CORAX** (CORA Extensions): RobotInterface, ProcessingDevice, PhysicalEnvironment +- **POS** (Position Ontology): PositionMeasure, OrientationMeasure, PoseMeasure, CoordinateSystems + +### Schema Statistics + +| Category | Count | +|---------------------|-------| +| Entity Tables | 12 | +| Relationship Tables | 7 | +| Total Tables | 19 | + +## Quick Start Deployment + +### Prerequisites + +- Azure CLI authenticated (`az login`) +- Fabric workspace with capacity +- Bash shell (Git Bash on Windows) + +### Deploy with Seed Data + +```bash +cd src/000-cloud/033-fabric-ontology/scripts + +# Deploy ontology with sample robotics data +./deploy-cora-corax-dim.sh \ + --workspace-id \ + --with-seed-data + +# Dry run (preview without changes) +./deploy-cora-corax-dim.sh \ + --workspace-id \ + --with-seed-data \ + --dry-run +``` + +### What Gets Deployed + +1. **Lakehouse**: `RoboticsOntologyLH` with 19 Delta tables +2. **Semantic Model**: `CORA_CORAX_DimensionalModel` (Direct Lake) +3. **Ontology**: `CORA_CORAX_Dimensional` with entity types and relationships + +## Sample Questions + +Use these questions to test your deployed ontology. They range from simple lookups to complex multi-hop analytics. + +### Entity Lookups + +| Question | Tests | +|------------------------------------------------|------------------------| +| What robots do we have? | Basic entity retrieval | +| Show me all robotic systems | Entity type query | +| List the physical environments in our facility | CORAX entity query | + +### Relationship Traversal + +| Question | Tests | +|------------------------------------------------------------|-----------------------------| +| Which robots are in the Assembly Line group? | `member` relationship | +| What robotic system is the ABB IRB 6700 part of? | `partOf` relationship | +| Which robotic systems are equipped in Factory Floor North? | `equippedWith` relationship | + +### Measurement Queries + +| Question | Tests | +|-----------------------------------------------------------|---------------------------------------| +| What is the current position of the KUKA KR 16? | `hasPosition` + PositionMeasure | +| Show me the orientation (roll, pitch, yaw) for all robots | `hasOrientation` + OrientationMeasure | +| Which robot has the highest Z position? | Aggregation on measures | + +### Multi-Hop Queries + +| Question | Tests | +|-----------------------------------------------------------------------|-----------------------------------------------------| +| Show me all robots in robotic systems equipped in Factory Floor North | 3-hop: Robot → RoboticSystem → RoboticEnvironment | +| What are the positions of robots in the Welding Cell group? | 2-hop: Robot → RobotGroup + Robot → PositionMeasure | +| List robots and their coordinate systems | Traverse through measures to coordinate systems | + +### Aggregations + +| Question | Tests | +|------------------------------------------------------------|----------------------| +| What is the average X position across all robots? | Numeric aggregation | +| How many robots are in each robot group? | Group-by counting | +| What is the range of Z positions for Assembly Line robots? | Filtered aggregation | + +### Complex Analytics + +| Question | Tests | +|------------------------------------------------------------------|------------------------------------| +| Compare the pose measurements between ABB and KUKA robots | Multi-entity comparison | +| Which robots have moved more than 1 meter from origin? | Calculated field (X^2+Y^2+Z^2 > 1) | +| Show robots with roll angle > 0.5 radians and their environments | Filtered multi-hop | + +### Cross-Entity Analysis + +| Question | Tests | +|----------------------------------------------------------------------------------|----------------------------| +| For each robotic environment, show robot count and average positions | Join + aggregation | +| Which processing devices are associated with robots that have pose measurements? | Complex relationship chain | + +## Seed Data Summary + +### Robots (3 records) + +| Robot | Group | System | +|--------------|---------------|-------------------| +| ABB IRB 6700 | Welding Cell | Production Line A | +| KUKA KR 16 | Assembly Line | Production Line B | +| Fanuc M-20iA | Assembly Line | Production Line B | + +### Robotic Environments (2 records) + +| Environment | Equipped Systems | +|---------------------|-------------------| +| Factory Floor North | Production Line A | +| Factory Floor South | Production Line B | + +### Measurement Data + +Each robot has sample position, orientation, and pose measurements with realistic robotics values (X/Y/Z in meters, Roll/Pitch/Yaw in radians). + +## Directory Structure + +```plaintext +fabric-ontology-dim/ +├── README.md # This file +├── json-schema/ # JSON Schema validation files +│ ├── Robot.schema.json +│ ├── RobotGroup.schema.json +│ ├── ... +│ └── Robot_PoseMeasure.schema.json +└── seed/ # Sample CSV data + ├── Robot.csv + ├── RobotGroup.csv + ├── ... + └── Robot_PoseMeasure.csv +``` + +## Entity Type Reference + +### CORA Entities + +| Entity | Key | Description | +|--------------------|-----|----------------------------------------------------| +| Robot | Id | Agentive device performing physical actions | +| RobotGroup | Id | Collection of robots working together | +| RoboticSystem | Id | System containing robots and supporting components | +| RoboticEnvironment | Id | Environment where robotic systems operate | + +### CORAX Entities + +| Entity | Key | Description | +|---------------------|-----|----------------------------------------| +| RobotInterface | Id | Communication/interaction interface | +| ProcessingDevice | Id | Computational device for robot control | +| PhysicalEnvironment | Id | Physical space containing robots | + +### POS Entities + +| Entity | Key | Measurement Columns | +|-----------------------------|-----|---------------------------------| +| PositionCoordinateSystem | Id | Reference frame for position | +| OrientationCoordinateSystem | Id | Reference frame for orientation | +| PositionMeasure | Id | X, Y, Z (meters) | +| OrientationMeasure | Id | Roll, Pitch, Yaw (radians) | +| PoseMeasure | Id | X, Y, Z, Roll, Pitch, Yaw | + +## Relationship Reference + +| Relationship | From | To | Cardinality | +|----------------|--------------------------|-----------------------------|--------------| +| member | Robot | RobotGroup | Many-to-Many | +| partOf | Robot | RoboticSystem | Many-to-Many | +| equippedWith | RoboticEnvironment | RoboticSystem | Many-to-Many | +| transform | PositionCoordinateSystem | OrientationCoordinateSystem | Many-to-Many | +| hasPosition | Robot | PositionMeasure | Many-to-Many | +| hasOrientation | Robot | OrientationMeasure | Many-to-Many | +| hasPose | Robot | PoseMeasure | Many-to-Many | + +## Extending the Schema + +### Adding New Robots + +1. Add row to `seed/Robot.csv` +2. Add relationships in `seed/Robot_RobotGroup.csv`, `seed/Robot_RoboticSystem.csv` +3. Add measurements in `seed/Robot_PositionMeasure.csv`, etc. +4. Re-run deployment with `--with-seed-data` + +### Adding Telemetry Data + +The measure tables (PositionMeasure, OrientationMeasure, PoseMeasure) include `Timestamp` and `CoordinateSystemId` columns ready for streaming telemetry from IoT Hub → Event Hubs → Eventhouse. + +## IEEE 1872 Reference + +- [IEEE 1872-2015](https://standards.ieee.org/standard/1872-2015.html) - Core Ontology for Robotics and Automation +- [IEEE 1872.2-2021](https://standards.ieee.org/standard/1872_2-2021.html) - Autonomous Robotics Ontology diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/CoordinateSystem_Transformations.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/CoordinateSystem_Transformations.sql new file mode 100644 index 00000000..47837aa9 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/CoordinateSystem_Transformations.sql @@ -0,0 +1,10 @@ + +CREATE TABLE CoordinateSystem_Transformations ( + Id STRING NOT NULL, + FromId STRING NOT NULL, + ToId STRING NOT NULL, + RelationType STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/OrientationCoordinateSystem.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/OrientationCoordinateSystem.sql new file mode 100644 index 00000000..35605c1b --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/OrientationCoordinateSystem.sql @@ -0,0 +1,9 @@ + +CREATE TABLE OrientationCoordinateSystem ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/OrientationMeasure.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/OrientationMeasure.sql new file mode 100644 index 00000000..7fea164e --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/OrientationMeasure.sql @@ -0,0 +1,9 @@ + +CREATE TABLE OrientationMeasure ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PhysicalEnvironment.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PhysicalEnvironment.sql new file mode 100644 index 00000000..43b57103 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PhysicalEnvironment.sql @@ -0,0 +1,9 @@ + +CREATE TABLE PhysicalEnvironment ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PoseMeasure.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PoseMeasure.sql new file mode 100644 index 00000000..3a44e3c0 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PoseMeasure.sql @@ -0,0 +1,9 @@ + +CREATE TABLE PoseMeasure ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PositionCoordinateSystem.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PositionCoordinateSystem.sql new file mode 100644 index 00000000..5900432f --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PositionCoordinateSystem.sql @@ -0,0 +1,9 @@ + +CREATE TABLE PositionCoordinateSystem ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PositionMeasure.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PositionMeasure.sql new file mode 100644 index 00000000..63430da2 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/PositionMeasure.sql @@ -0,0 +1,9 @@ + +CREATE TABLE PositionMeasure ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/ProcessingDevice.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/ProcessingDevice.sql new file mode 100644 index 00000000..c1ebc0a0 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/ProcessingDevice.sql @@ -0,0 +1,9 @@ + +CREATE TABLE ProcessingDevice ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/Robot.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/Robot.sql new file mode 100644 index 00000000..adb252f6 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/Robot.sql @@ -0,0 +1,9 @@ + +CREATE TABLE Robot ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RobotGroup.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RobotGroup.sql new file mode 100644 index 00000000..99ec05ee --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RobotGroup.sql @@ -0,0 +1,9 @@ + +CREATE TABLE RobotGroup ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RobotInterface.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RobotInterface.sql new file mode 100644 index 00000000..0da08759 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RobotInterface.sql @@ -0,0 +1,9 @@ + +CREATE TABLE RobotInterface ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/Robot_RobotGroup.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/Robot_RobotGroup.sql new file mode 100644 index 00000000..585b294f --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/Robot_RobotGroup.sql @@ -0,0 +1,10 @@ + +CREATE TABLE Robot_RobotGroup ( + Id STRING NOT NULL, + FromId STRING NOT NULL, + ToId STRING NOT NULL, + RelationType STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/Robot_RoboticSystem.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/Robot_RoboticSystem.sql new file mode 100644 index 00000000..e3680514 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/Robot_RoboticSystem.sql @@ -0,0 +1,10 @@ + +CREATE TABLE Robot_RoboticSystem ( + Id STRING NOT NULL, + FromId STRING NOT NULL, + ToId STRING NOT NULL, + RelationType STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RoboticEnvironment.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RoboticEnvironment.sql new file mode 100644 index 00000000..4a8ce9da --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RoboticEnvironment.sql @@ -0,0 +1,9 @@ + +CREATE TABLE RoboticEnvironment ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RoboticSystem.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RoboticSystem.sql new file mode 100644 index 00000000..4eced157 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RoboticSystem.sql @@ -0,0 +1,9 @@ + +CREATE TABLE RoboticSystem ( + Id STRING NOT NULL, + Name STRING, + Description STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RoboticSystem_RoboticEnvironment.sql b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RoboticSystem_RoboticEnvironment.sql new file mode 100644 index 00000000..91b0dd6c --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/ddl/RoboticSystem_RoboticEnvironment.sql @@ -0,0 +1,10 @@ + +CREATE TABLE RoboticSystem_RoboticEnvironment ( + Id STRING NOT NULL, + FromId STRING NOT NULL, + ToId STRING NOT NULL, + RelationType STRING, + Namespace STRING, + Source STRING, + PRIMARY KEY (Id) +); diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/CoordinateSystem_Transformations.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/CoordinateSystem_Transformations.schema.json new file mode 100644 index 00000000..955e21a0 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/CoordinateSystem_Transformations.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "CoordinateSystem_Transformations", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "FromId": { + "type": "string" + }, + "ToId": { + "type": "string" + }, + "RelationType": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id", + "FromId", + "ToId" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/OrientationCoordinateSystem.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/OrientationCoordinateSystem.schema.json new file mode 100644 index 00000000..eaef28b7 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/OrientationCoordinateSystem.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OrientationCoordinateSystem", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/OrientationMeasure.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/OrientationMeasure.schema.json new file mode 100644 index 00000000..1d10aa9a --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/OrientationMeasure.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OrientationMeasure", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "Roll": { + "type": "number", + "description": "Roll angle in radians" + }, + "Pitch": { + "type": "number", + "description": "Pitch angle in radians" + }, + "Yaw": { + "type": "number", + "description": "Yaw angle in radians" + }, + "CoordinateSystemId": { + "type": [ + "string", + "null" + ], + "description": "Reference to OrientationCoordinateSystem" + }, + "Timestamp": { + "type": [ + "string", + "null" + ], + "description": "ISO 8601 timestamp of measurement" + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id", + "Roll", + "Pitch", + "Yaw" + ] +} diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PhysicalEnvironment.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PhysicalEnvironment.schema.json new file mode 100644 index 00000000..8b495a45 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PhysicalEnvironment.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "PhysicalEnvironment", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PoseMeasure.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PoseMeasure.schema.json new file mode 100644 index 00000000..51cd0809 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PoseMeasure.schema.json @@ -0,0 +1,78 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "PoseMeasure", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "X": { + "type": "number", + "description": "X coordinate in meters" + }, + "Y": { + "type": "number", + "description": "Y coordinate in meters" + }, + "Z": { + "type": "number", + "description": "Z coordinate in meters" + }, + "Roll": { + "type": "number", + "description": "Roll angle in radians" + }, + "Pitch": { + "type": "number", + "description": "Pitch angle in radians" + }, + "Yaw": { + "type": "number", + "description": "Yaw angle in radians" + }, + "CoordinateSystemId": { + "type": [ + "string", + "null" + ], + "description": "Reference to coordinate system" + }, + "Timestamp": { + "type": [ + "string", + "null" + ], + "description": "ISO 8601 timestamp of measurement" + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id", + "X", + "Y", + "Z", + "Roll", + "Pitch", + "Yaw" + ] +} diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PositionCoordinateSystem.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PositionCoordinateSystem.schema.json new file mode 100644 index 00000000..2b1e6ac5 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PositionCoordinateSystem.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "PositionCoordinateSystem", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PositionMeasure.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PositionMeasure.schema.json new file mode 100644 index 00000000..73cc0a51 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/PositionMeasure.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "PositionMeasure", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "X": { + "type": "number", + "description": "X coordinate in meters" + }, + "Y": { + "type": "number", + "description": "Y coordinate in meters" + }, + "Z": { + "type": "number", + "description": "Z coordinate in meters" + }, + "CoordinateSystemId": { + "type": [ + "string", + "null" + ], + "description": "Reference to PositionCoordinateSystem" + }, + "Timestamp": { + "type": [ + "string", + "null" + ], + "description": "ISO 8601 timestamp of measurement" + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id", + "X", + "Y", + "Z" + ] +} diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/ProcessingDevice.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/ProcessingDevice.schema.json new file mode 100644 index 00000000..1fecec82 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/ProcessingDevice.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ProcessingDevice", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot.schema.json new file mode 100644 index 00000000..1b9e656b --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Robot", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RobotGroup.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RobotGroup.schema.json new file mode 100644 index 00000000..4a4e1083 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RobotGroup.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "RobotGroup", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RobotInterface.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RobotInterface.schema.json new file mode 100644 index 00000000..cec49d2f --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RobotInterface.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "RobotInterface", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_OrientationMeasure.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_OrientationMeasure.schema.json new file mode 100644 index 00000000..d414b128 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_OrientationMeasure.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Robot_OrientationMeasure", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "FromId": { + "type": "string" + }, + "ToId": { + "type": "string" + }, + "RelationType": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id", + "FromId", + "ToId" + ] +} diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_PoseMeasure.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_PoseMeasure.schema.json new file mode 100644 index 00000000..9ee08e19 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_PoseMeasure.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Robot_PoseMeasure", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "FromId": { + "type": "string" + }, + "ToId": { + "type": "string" + }, + "RelationType": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id", + "FromId", + "ToId" + ] +} diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_PositionMeasure.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_PositionMeasure.schema.json new file mode 100644 index 00000000..0a8f44f3 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_PositionMeasure.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Robot_PositionMeasure", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "FromId": { + "type": "string" + }, + "ToId": { + "type": "string" + }, + "RelationType": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id", + "FromId", + "ToId" + ] +} diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_RobotGroup.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_RobotGroup.schema.json new file mode 100644 index 00000000..16023bb2 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_RobotGroup.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Robot_RobotGroup", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "FromId": { + "type": "string" + }, + "ToId": { + "type": "string" + }, + "RelationType": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id", + "FromId", + "ToId" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_RoboticSystem.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_RoboticSystem.schema.json new file mode 100644 index 00000000..154e13b3 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/Robot_RoboticSystem.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Robot_RoboticSystem", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "FromId": { + "type": "string" + }, + "ToId": { + "type": "string" + }, + "RelationType": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id", + "FromId", + "ToId" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RoboticEnvironment.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RoboticEnvironment.schema.json new file mode 100644 index 00000000..9625dfbe --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RoboticEnvironment.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "RoboticEnvironment", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RoboticSystem.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RoboticSystem.schema.json new file mode 100644 index 00000000..2875ec02 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RoboticSystem.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "RoboticSystem", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RoboticSystem_RoboticEnvironment.schema.json b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RoboticSystem_RoboticEnvironment.schema.json new file mode 100644 index 00000000..3c44e444 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/json-schema/RoboticSystem_RoboticEnvironment.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "RoboticSystem_RoboticEnvironment", + "type": "object", + "properties": { + "Id": { + "type": "string" + }, + "FromId": { + "type": "string" + }, + "ToId": { + "type": "string" + }, + "RelationType": { + "type": [ + "string", + "null" + ] + }, + "Namespace": { + "type": [ + "string", + "null" + ] + }, + "Source": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "Id", + "FromId", + "ToId" + ] +} \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/CoordinateSystem_Transformations.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/CoordinateSystem_Transformations.csv new file mode 100644 index 00000000..b5b1dab9 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/CoordinateSystem_Transformations.csv @@ -0,0 +1,4 @@ +Id,FromId,ToId,RelationType,Namespace,Source +rel_cs1,pcs1,ocs1,transform,POS,Example +rel_cs2,pcs2,ocs1,transform,POS,Example +rel_cs3,pcs1,ocs2,transform,POS,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/OrientationCoordinateSystem.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/OrientationCoordinateSystem.csv new file mode 100644 index 00000000..87273adf --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/OrientationCoordinateSystem.csv @@ -0,0 +1,3 @@ +Id,Name,Description,Namespace,Source +ocs1,Euler ZYX,Euler angles with ZYX rotation order,POS,Example +ocs2,Quaternion,Quaternion orientation representation,POS,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/OrientationMeasure.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/OrientationMeasure.csv new file mode 100644 index 00000000..98436338 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/OrientationMeasure.csv @@ -0,0 +1,4 @@ +Id,Name,Description,Roll,Pitch,Yaw,CoordinateSystemId,Timestamp,Namespace,Source +orient1,ABB Home Orientation,Home orientation for ABB IRB 6700,0.0,0.0,1.57,ocs1,2026-01-08T00:00:00Z,POS,Example +orient2,KUKA Home Orientation,Home orientation for KUKA KR 16,0.0,0.0,0.0,ocs1,2026-01-08T00:00:00Z,POS,Example +orient3,Fanuc Home Orientation,Home orientation for Fanuc M-20iA,0.0,0.0,-1.57,ocs1,2026-01-08T00:00:00Z,POS,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PhysicalEnvironment.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PhysicalEnvironment.csv new file mode 100644 index 00000000..046a2f63 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PhysicalEnvironment.csv @@ -0,0 +1,3 @@ +Id,Name,Description,Namespace,Source +penv1,Manufacturing Facility,Main factory building with climate control,CORAX,Example +penv2,Warehouse,Storage and logistics area adjacent to factory,CORAX,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PoseMeasure.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PoseMeasure.csv new file mode 100644 index 00000000..4d3c3da3 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PoseMeasure.csv @@ -0,0 +1,4 @@ +Id,Name,Description,X,Y,Z,Roll,Pitch,Yaw,CoordinateSystemId,Timestamp,Namespace,Source +pose1,ABB Home Pose,Home pose for ABB IRB 6700,1.5,2.0,0.8,0.0,0.0,1.57,pcs1,2026-01-08T00:00:00Z,POS,Example +pose2,KUKA Home Pose,Home pose for KUKA KR 16,4.5,2.0,0.6,0.0,0.0,0.0,pcs1,2026-01-08T00:00:00Z,POS,Example +pose3,Fanuc Home Pose,Home pose for Fanuc M-20iA,7.5,3.5,0.7,0.0,0.0,-1.57,pcs1,2026-01-08T00:00:00Z,POS,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PositionCoordinateSystem.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PositionCoordinateSystem.csv new file mode 100644 index 00000000..f218cf0d --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PositionCoordinateSystem.csv @@ -0,0 +1,3 @@ +Id,Name,Description,Namespace,Source +pcs1,World Frame,Global reference coordinate system for factory,POS,Example +pcs2,Base Frame,Robot base coordinate system origin,POS,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PositionMeasure.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PositionMeasure.csv new file mode 100644 index 00000000..924b55d8 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/PositionMeasure.csv @@ -0,0 +1,4 @@ +Id,Name,Description,X,Y,Z,CoordinateSystemId,Timestamp,Namespace,Source +pos1,ABB Home Position,Home position for ABB IRB 6700,1.5,2.0,0.8,pcs1,2026-01-08T00:00:00Z,POS,Example +pos2,KUKA Home Position,Home position for KUKA KR 16,4.5,2.0,0.6,pcs1,2026-01-08T00:00:00Z,POS,Example +pos3,Fanuc Home Position,Home position for Fanuc M-20iA,7.5,3.5,0.7,pcs1,2026-01-08T00:00:00Z,POS,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/ProcessingDevice.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/ProcessingDevice.csv new file mode 100644 index 00000000..f2f960f1 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/ProcessingDevice.csv @@ -0,0 +1,3 @@ +Id,Name,Description,Namespace,Source +proc1,ABB IRC5 Controller,Robot controller for ABB IRB 6700,CORAX,Example +proc2,KUKA KR C4 Controller,Robot controller for KUKA KR 16,CORAX,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot.csv new file mode 100644 index 00000000..315a45ea --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot.csv @@ -0,0 +1,4 @@ +Id,Name,Description,Namespace,Source +robot1,ABB IRB 6700,Industrial welding robot with 6-axis arm,CORA,Example +robot2,KUKA KR 16,Precision assembly robot for electronics,CORA,Example +robot3,Fanuc M-20iA,Material handling robot with payload capacity 20kg,CORA,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RobotGroup.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RobotGroup.csv new file mode 100644 index 00000000..7089a3ed --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RobotGroup.csv @@ -0,0 +1,3 @@ +Id,Name,Description,Namespace,Source +group1,Welding Cell,Group of welding robots in north factory,CORA,Example +group2,Assembly Line,Group of assembly and handling robots,CORA,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RobotInterface.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RobotInterface.csv new file mode 100644 index 00000000..791196e5 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RobotInterface.csv @@ -0,0 +1,4 @@ +Id,Name,Description,Namespace,Source +iface1,Weld Torch Interface,Welding torch end effector for ABB IRB 6700,CORA,Example +iface2,Gripper Interface,Precision gripper for KUKA KR 16,CORA,Example +iface3,Vacuum Suction Interface,Vacuum end effector for Fanuc M-20iA,CORA,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_OrientationMeasure.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_OrientationMeasure.csv new file mode 100644 index 00000000..1c234d50 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_OrientationMeasure.csv @@ -0,0 +1,4 @@ +Id,FromId,ToId,RelationType,Namespace,Source +rel_orient1,robot1,orient1,has,POS,Example +rel_orient2,robot2,orient2,has,POS,Example +rel_orient3,robot3,orient3,has,POS,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_PoseMeasure.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_PoseMeasure.csv new file mode 100644 index 00000000..1c3cfeed --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_PoseMeasure.csv @@ -0,0 +1,4 @@ +Id,FromId,ToId,RelationType,Namespace,Source +rel_pose1,robot1,pose1,has,POS,Example +rel_pose2,robot2,pose2,has,POS,Example +rel_pose3,robot3,pose3,has,POS,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_PositionMeasure.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_PositionMeasure.csv new file mode 100644 index 00000000..69d8f06a --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_PositionMeasure.csv @@ -0,0 +1,4 @@ +Id,FromId,ToId,RelationType,Namespace,Source +rel_pos1,robot1,pos1,has,POS,Example +rel_pos2,robot2,pos2,has,POS,Example +rel_pos3,robot3,pos3,has,POS,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_RobotGroup.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_RobotGroup.csv new file mode 100644 index 00000000..7d777e6f --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_RobotGroup.csv @@ -0,0 +1,4 @@ +Id,FromId,ToId,RelationType,Namespace,Source +rel_rg1,robot1,group1,member,CORA,Example +rel_rg2,robot2,group2,member,CORA,Example +rel_rg3,robot3,group2,member,CORA,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_RoboticSystem.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_RoboticSystem.csv new file mode 100644 index 00000000..df618add --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/Robot_RoboticSystem.csv @@ -0,0 +1,4 @@ +Id,FromId,ToId,RelationType,Namespace,Source +rel_rs1,robot1,system1,partOf,CORA,Example +rel_rs2,robot2,system2,partOf,CORA,Example +rel_rs3,robot3,system2,partOf,CORA,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticEnvironment.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticEnvironment.csv new file mode 100644 index 00000000..0912887c --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticEnvironment.csv @@ -0,0 +1,3 @@ +Id,Name,Description,Namespace,Source +renv1,Factory Floor North,Main production area equipped with welding systems,CORA,Example +renv2,Factory Floor South,Secondary area equipped with assembly systems,CORA,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticEnvironment_RoboticSystem.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticEnvironment_RoboticSystem.csv new file mode 100644 index 00000000..694be601 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticEnvironment_RoboticSystem.csv @@ -0,0 +1,3 @@ +Id,FromId,ToId,RelationType,Namespace,Source +rel_se1,renv1,system1,equippedWith,CORA,Example +rel_se2,renv2,system2,equippedWith,CORA,Example \ No newline at end of file diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticSystem.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticSystem.csv new file mode 100644 index 00000000..a2fffe42 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticSystem.csv @@ -0,0 +1,3 @@ +Id,Name,Description,Namespace,Source +system1,Production Line A,Primary manufacturing line with welding station,CORA,Example +system2,Production Line B,Secondary manufacturing line with assembly station,CORA,Example diff --git a/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticSystem_RoboticEnvironment.csv b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticSystem_RoboticEnvironment.csv new file mode 100644 index 00000000..501f6fcf --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/fabric-ontology-dim/seed/RoboticSystem_RoboticEnvironment.csv @@ -0,0 +1,3 @@ +Id,FromId,ToId,RelationType,Namespace,Source +rel_se1,renv1,system1,equippedWith,CORA,Example +rel_se2,renv2,system2,equippedWith,CORA,Example diff --git a/src/000-cloud/033-fabric-ontology/scripts/deploy-cora-corax-dim.sh b/src/000-cloud/033-fabric-ontology/scripts/deploy-cora-corax-dim.sh new file mode 100755 index 00000000..0f4401d2 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/scripts/deploy-cora-corax-dim.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1091 +# deploy-cora-corax-dim.sh - Deploy CORA/CORAX dimensional robotics ontology +# +# Wrapper script for deploying the IEEE 1872 CORA/CORAX robotics ontology +# with optional seed data for demonstration purposes. +# +# Dependencies: Azure CLI authenticated, deploy.sh +# +# Usage: +# ./deploy-cora-corax-dim.sh --workspace-id [--with-seed-data] + +set -e +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPONENT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" + +source "${SCRIPT_DIR}/lib/logging.sh" + +DEFINITION_FILE="${COMPONENT_DIR}/definitions/examples/cora-corax-dim.yaml" +SEED_DIR="${COMPONENT_DIR}/fabric-ontology-dim/seed" + +WORKSPACE_ID="" +LAKEHOUSE_ID="" +WITH_SEED_DATA="false" +PASSTHROUGH_ARGS=() + +usage() { + cat << EOF +Usage: $(basename "$0") [OPTIONS] + +Deploy the IEEE 1872 CORA/CORAX robotics ontology to Microsoft Fabric. + +This script deploys a dimensional schema with 12 entity types and 7 relationships +based on the CORA (Core Ontology for Robotics and Automation), CORAX, and POS +standards from IEEE 1872-2015. + +Required Arguments: + --workspace-id Fabric workspace ID (GUID) + +Optional Arguments: + --lakehouse-id Use existing Lakehouse (skips Lakehouse creation) + --with-seed-data Load sample robotics data for demonstration + --dry-run Show deployment plan without making changes + -h, --help Show this help message + +Examples: + # Deploy ontology schema only (bind to existing tables) + $(basename "$0") --workspace-id abc123 --lakehouse-id def456 + + # Deploy with sample robotics data (ABB, KUKA, Fanuc robots) + $(basename "$0") --workspace-id abc123 --with-seed-data + + # Preview deployment without making changes + $(basename "$0") --workspace-id abc123 --with-seed-data --dry-run + +Schema Contents: + Entity Types (12): + CORA: Robot, RobotGroup, RoboticSystem, RoboticEnvironment, RobotInterface + CORAX: ProcessingDevice, PhysicalEnvironment + POS: PositionCoordinateSystem, OrientationCoordinateSystem, + PositionMeasure, OrientationMeasure, PoseMeasure + + Relationships (7): + member, partOf, equippedWith, transform, + hasPosition, hasOrientation, hasPose +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --workspace-id) + WORKSPACE_ID="$2" + shift 2 + ;; + --lakehouse-id) + LAKEHOUSE_ID="$2" + shift 2 + ;; + --with-seed-data) + WITH_SEED_DATA="true" + shift + ;; + --dry-run) + PASSTHROUGH_ARGS+=("$1") + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + PASSTHROUGH_ARGS+=("$1") + shift + ;; + esac +done + +if [[ -z "${WORKSPACE_ID}" ]]; then + err "--workspace-id is required" +fi + +if [[ ! -f "${DEFINITION_FILE}" ]]; then + err "Definition file not found: ${DEFINITION_FILE}" +fi + +log "Deploying CORA/CORAX Dimensional Ontology" +info "Definition: ${DEFINITION_FILE}" +info "Workspace: ${WORKSPACE_ID}" +info "With Seed Data: ${WITH_SEED_DATA}" + +if [[ "${WITH_SEED_DATA}" == "true" ]]; then + if [[ ! -d "${SEED_DIR}" ]]; then + err "Seed directory not found: ${SEED_DIR}" + fi + + info "Seed Directory: ${SEED_DIR}" + + "${SCRIPT_DIR}/deploy.sh" \ + --definition "${DEFINITION_FILE}" \ + --workspace-id "${WORKSPACE_ID}" \ + --data-dir "${SEED_DIR}" \ + ${LAKEHOUSE_ID:+--lakehouse-id "${LAKEHOUSE_ID}"} \ + "${PASSTHROUGH_ARGS[@]}" +else + if [[ -z "${LAKEHOUSE_ID}" ]]; then + err "--lakehouse-id is required when not using --with-seed-data" + fi + + "${SCRIPT_DIR}/deploy.sh" \ + --definition "${DEFINITION_FILE}" \ + --workspace-id "${WORKSPACE_ID}" \ + --lakehouse-id "${LAKEHOUSE_ID}" \ + --skip-data-sources \ + "${PASSTHROUGH_ARGS[@]}" +fi + +ok "CORA/CORAX deployment complete" diff --git a/src/000-cloud/033-fabric-ontology/scripts/deploy-data-sources.sh b/src/000-cloud/033-fabric-ontology/scripts/deploy-data-sources.sh new file mode 100755 index 00000000..39dd627b --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/scripts/deploy-data-sources.sh @@ -0,0 +1,593 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1091 +# Deploy Data Sources - Creates Lakehouse and Eventhouse from ontology definition +# +# Creates and populates Fabric data sources (Lakehouse and Eventhouse) based on +# the dataSources section in an ontology definition YAML file. +# +# Dependencies: yq, jq, curl, az (Azure CLI) +# +# Required Arguments: +# --definition Path to ontology definition YAML file +# --workspace-id Fabric workspace ID (GUID) +# +# Optional Arguments: +# --skip-lakehouse Skip Lakehouse creation and data loading +# --skip-eventhouse Skip Eventhouse creation and data loading +# --dry-run Validate and show plan without making changes +# -d, --debug Enable debug output +# +# Environment Variables (optional): +# FABRIC_API_BASE_URL Override default Fabric API URL +# SKIP_VALIDATION Skip definition validation (not recommended) +# +# Outputs (exported as environment variables): +# LAKEHOUSE_ID Created/existing Lakehouse ID +# LAKEHOUSE_NAME Lakehouse display name +# EVENTHOUSE_ID Created/existing Eventhouse ID +# EVENTHOUSE_NAME Eventhouse display name +# KQL_DATABASE_ID Created/existing KQL Database ID +# KQL_DATABASE_NAME KQL Database display name +# +## Examples +## ./deploy-data-sources.sh --definition ../definitions/examples/lakeshore-retail.yaml --workspace-id 12345678-1234-1234-1234-123456789abc +## ./deploy-data-sources.sh --definition ontology.yaml --workspace-id $WORKSPACE_ID --skip-eventhouse +## ./deploy-data-sources.sh --definition ontology.yaml --workspace-id $WORKSPACE_ID --dry-run +### + +set -e +set -o pipefail + +# Script directory for relative paths +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source library functions +source "$SCRIPT_DIR/lib/definition-parser.sh" +source "$SCRIPT_DIR/lib/fabric-api.sh" + +# Template directory +TEMPLATE_DIR="$SCRIPT_DIR/../templates/kql" + +# Arguments +DEFINITION_FILE="" +WORKSPACE_ID="" +SKIP_LAKEHOUSE="${SKIP_LAKEHOUSE:-false}" +SKIP_EVENTHOUSE="${SKIP_EVENTHOUSE:-false}" +SKIP_VALIDATION="${SKIP_VALIDATION:-false}" +DRY_RUN="${DRY_RUN:-false}" +DEBUG="${DEBUG:-false}" + +#### +# Utility Functions +#### + +usage() { + echo "Usage: ${0##*/}" + grep -x -B99 -m 1 "^###" "$0" | + sed -E -e '/^[^#]+=/ {s/^([^ ])/ \1/ ; s/#/ / ; s/=[^ ]*$// ;}' | + sed -E -e ':x' -e '/^[^#]+=/ {s/^( [^ ]+)[^ ] /\1 / ;}' -e 'tx' | + sed -e 's/^## //' -e '/^#/d' -e '/^$/d' + exit 1 +} + +log() { + printf "========== %s ==========\n" "$1" +} + +info() { + printf "[ INFO ]: %s\n" "$1" +} + +warn() { + printf "[ WARN ]: %s\n" "$1" >&2 +} + +err() { + printf "[ ERROR ]: %s\n" "$1" >&2 + exit 1 +} + +enable_debug() { + echo "[ DEBUG ]: Enabling debug output" + set -x +} + +#### +# Argument Parsing +#### + +while [[ $# -gt 0 ]]; do + case "$1" in + --definition) + DEFINITION_FILE="$2" + shift 2 + ;; + --workspace-id) + WORKSPACE_ID="$2" + shift 2 + ;; + --skip-lakehouse) + SKIP_LAKEHOUSE="true" + shift + ;; + --skip-eventhouse) + SKIP_EVENTHOUSE="true" + shift + ;; + --skip-validation) + SKIP_VALIDATION="true" + shift + ;; + --dry-run) + DRY_RUN="true" + shift + ;; + -d|--debug) + DEBUG="true" + enable_debug + shift + ;; + -h|--help) + usage + ;; + *) + err "Unknown argument: $1" + ;; + esac +done + +#### +# Validate Arguments +#### + +if [[ -z "$DEFINITION_FILE" ]]; then + err "--definition is required" +fi + +if [[ ! -f "$DEFINITION_FILE" ]]; then + err "Definition file not found: $DEFINITION_FILE" +fi + +if [[ -z "$WORKSPACE_ID" ]]; then + err "--workspace-id is required" +fi + +#### +# Validate Definition +#### + +if [[ "$SKIP_VALIDATION" != "true" ]]; then + log "Validating Definition" + if ! "$SCRIPT_DIR/validate-definition.sh" --definition "$DEFINITION_FILE"; then + err "Definition validation failed" + fi + info "Definition validation passed" +fi + +#### +# Extract Metadata +#### + +log "Extracting Definition Metadata" +ONTOLOGY_NAME=$(get_metadata_name "$DEFINITION_FILE") +ONTOLOGY_VERSION=$(get_metadata_version "$DEFINITION_FILE") +info "Ontology: $ONTOLOGY_NAME (v$ONTOLOGY_VERSION)" + +#### +# Authenticate +#### + +log "Authenticating to Fabric API" +FABRIC_TOKEN=$(get_fabric_token) +STORAGE_TOKEN=$(get_storage_token) +info "Authentication successful" + +#### +# Verify Workspace Access +#### + +log "Verifying Workspace Access" +workspace_response=$(get_workspace "$WORKSPACE_ID" "$FABRIC_TOKEN") +workspace_name=$(echo "$workspace_response" | jq -r '.displayName // "Unknown"') +info "Workspace: $workspace_name ($WORKSPACE_ID)" + +#### +# Deploy Lakehouse +#### + +deploy_lakehouse() { + local lakehouse_name lakehouse_id lakehouse_response + + lakehouse_name=$(get_lakehouse_name "$DEFINITION_FILE") + if [[ -z "$lakehouse_name" ]]; then + info "No lakehouse defined in dataSources, skipping" + return 0 + fi + + log "Deploying Lakehouse" + info "Lakehouse name: $lakehouse_name" + + if [[ "$DRY_RUN" == "true" ]]; then + info "[DRY-RUN] Would create/get Lakehouse: $lakehouse_name" + return 0 + fi + + # Create or get existing lakehouse + lakehouse_response=$(get_or_create_lakehouse "$WORKSPACE_ID" "$lakehouse_name" "$FABRIC_TOKEN") + lakehouse_id=$(echo "$lakehouse_response" | jq -r '.id') + + if [[ -z "$lakehouse_id" || "$lakehouse_id" == "null" ]]; then + err "Failed to get Lakehouse ID" + fi + + export LAKEHOUSE_ID="$lakehouse_id" + export LAKEHOUSE_NAME="$lakehouse_name" + info "Lakehouse ID: $lakehouse_id" + + # Process lakehouse tables + process_lakehouse_tables "$lakehouse_id" +} + +process_lakehouse_tables() { + local lakehouse_id="$1" + local tables table_count table_name source_url source_file format + + tables=$(get_lakehouse_tables "$DEFINITION_FILE") + table_count=$(echo "$tables" | jq 'length') + + if [[ "$table_count" -eq 0 ]]; then + info "No tables defined in lakehouse, skipping data loading" + return 0 + fi + + info "Processing $table_count lakehouse tables" + + for i in $(seq 0 $((table_count - 1))); do + table_name=$(echo "$tables" | jq -r ".[$i].name") + source_url=$(echo "$tables" | jq -r ".[$i].sourceUrl // empty") + source_file=$(echo "$tables" | jq -r ".[$i].sourceFile // empty") + format=$(echo "$tables" | jq -r ".[$i].format // \"csv\"") + + info "Table: $table_name (format: $format)" + + if [[ "$DRY_RUN" == "true" ]]; then + info "[DRY-RUN] Would process table: $table_name" + continue + fi + + # Download source data if URL provided + local local_file="" + if [[ -n "$source_url" ]]; then + local_file=$(download_source_file "$source_url" "$table_name") + elif [[ -n "$source_file" ]]; then + local_file="$source_file" + if [[ ! -f "$local_file" ]]; then + warn "Source file not found: $local_file, skipping table $table_name" + continue + fi + else + warn "No sourceUrl or sourceFile for table $table_name, skipping" + continue + fi + + # Upload to OneLake Files + upload_to_onelake "$WORKSPACE_ID" "$lakehouse_id" "raw/${table_name}.${format}" "$local_file" "$STORAGE_TOKEN" + + # Convert to Delta table + load_lakehouse_table "$WORKSPACE_ID" "$lakehouse_id" "$table_name" "raw/${table_name}.${format}" "$format" "$FABRIC_TOKEN" + + info "Table $table_name loaded successfully" + + # Clean up downloaded file + if [[ -n "$source_url" && -f "$local_file" ]]; then + rm -f "$local_file" + fi + done +} + +download_source_file() { + local url="$1" + local table_name="$2" + local tmp_file + + tmp_file=$(mktemp "/tmp/${table_name}.XXXXXX.csv") + + info "Downloading: $url" >&2 + if ! curl -sSL "$url" -o "$tmp_file"; then + err "Failed to download: $url" + fi + + echo "$tmp_file" +} + +#### +# Deploy Eventhouse +#### + +deploy_eventhouse() { + local eventhouse_name database_name eventhouse_id eventhouse_response database_id database_response + + eventhouse_name=$(get_eventhouse_name "$DEFINITION_FILE") + if [[ -z "$eventhouse_name" ]]; then + info "No eventhouse defined in dataSources, skipping" + return 0 + fi + + database_name=$(get_eventhouse_database "$DEFINITION_FILE") + if [[ -z "$database_name" ]]; then + database_name="${eventhouse_name}DB" + warn "No database name specified, using default: $database_name" + fi + + log "Deploying Eventhouse" + info "Eventhouse name: $eventhouse_name" + info "Database name: $database_name" + + if [[ "$DRY_RUN" == "true" ]]; then + info "[DRY-RUN] Would create/get Eventhouse: $eventhouse_name" + info "[DRY-RUN] Would create/get KQL Database: $database_name" + return 0 + fi + + # Create or get existing eventhouse + eventhouse_response=$(get_or_create_eventhouse "$WORKSPACE_ID" "$eventhouse_name" "$FABRIC_TOKEN") + eventhouse_id=$(echo "$eventhouse_response" | jq -r '.id') + + if [[ -z "$eventhouse_id" || "$eventhouse_id" == "null" ]]; then + err "Failed to get Eventhouse ID" + fi + + export EVENTHOUSE_ID="$eventhouse_id" + export EVENTHOUSE_NAME="$eventhouse_name" + info "Eventhouse ID: $eventhouse_id" + + # Get Eventhouse query URI for KQL operations + local query_uri + query_uri=$(get_eventhouse_query_uri "$WORKSPACE_ID" "$eventhouse_id" "$FABRIC_TOKEN") + if [[ -z "$query_uri" ]]; then + err "Failed to get Eventhouse query URI" + fi + export EVENTHOUSE_QUERY_URI="$query_uri" + info "Eventhouse Query URI: $query_uri" + + # Create or get existing KQL database + database_response=$(get_or_create_kql_database "$WORKSPACE_ID" "$database_name" "$eventhouse_id" "$FABRIC_TOKEN") + database_id=$(echo "$database_response" | jq -r '.id') + + if [[ -z "$database_id" || "$database_id" == "null" ]]; then + err "Failed to get KQL Database ID" + fi + + export KQL_DATABASE_ID="$database_id" + export KQL_DATABASE_NAME="$database_name" + info "KQL Database ID: $database_id" + + # Process eventhouse tables + process_eventhouse_tables "$database_name" +} + +process_eventhouse_tables() { + local database_name="$1" + local tables table_count table_name source_url format schema + + tables=$(get_eventhouse_tables "$DEFINITION_FILE") + table_count=$(echo "$tables" | jq 'length') + + if [[ "$table_count" -eq 0 ]]; then + info "No tables defined in eventhouse, skipping" + return 0 + fi + + info "Processing $table_count eventhouse tables" + + for i in $(seq 0 $((table_count - 1))); do + table_name=$(echo "$tables" | jq -r ".[$i].name") + source_url=$(echo "$tables" | jq -r ".[$i].sourceUrl // empty") + format=$(echo "$tables" | jq -r ".[$i].format // \"csv\"") + schema=$(echo "$tables" | jq ".[$i].schema // []") + + info "Table: $table_name" + + if [[ "$DRY_RUN" == "true" ]]; then + info "[DRY-RUN] Would create KQL table: $table_name" + continue + fi + + # Generate KQL schema from definition + create_kql_table "$database_name" "$table_name" "$schema" + + # Create CSV mapping + create_kql_csv_mapping "$database_name" "$table_name" "$schema" + + # Set retention/caching policies + local policies + policies=$(echo "$tables" | jq ".[$i].policies // {}") + local retention caching + retention=$(echo "$policies" | jq -r '.retention // "30d"' | sed 's/d$//') + caching=$(echo "$policies" | jq -r '.caching // "7d"' | sed 's/d$//') + + set_kql_retention_policy "$database_name" "$table_name" "$retention" "$caching" + + # Ingest data if source URL provided + if [[ -n "$source_url" ]]; then + ingest_kql_data "$database_name" "$table_name" "$source_url" "$format" + fi + + info "Table $table_name created successfully" + done +} + +# Strip KQL comments and empty lines from template output +strip_kql_comments() { + grep -v '^[[:space:]]*//\|^[[:space:]]*$' | tr '\n' ' ' | sed 's/[[:space:]]*$//' +} + +create_kql_table() { + local database_name="$1" + local table_name="$2" + local schema="$3" + local column_schema="" col_name col_type kql_type + + # Build column schema from definition + local schema_count + schema_count=$(echo "$schema" | jq 'length') + + for j in $(seq 0 $((schema_count - 1))); do + col_name=$(echo "$schema" | jq -r ".[$j].name") + col_type=$(echo "$schema" | jq -r ".[$j].type") + kql_type=$(map_kql_type "$col_type") + + if [[ -n "$column_schema" ]]; then + column_schema="$column_schema, " + fi + column_schema="${column_schema}${col_name}: ${kql_type}" + done + + # Generate KQL command from template (strip comments) + local kql_command + kql_command=$(TABLE_NAME="$table_name" COLUMN_SCHEMA="$column_schema" envsubst < "$TEMPLATE_DIR/create-table.kql.tmpl" | strip_kql_comments) + + info "Creating KQL table: $table_name" + execute_kql "$EVENTHOUSE_QUERY_URI" "$database_name" "$kql_command" +} + +create_kql_csv_mapping() { + local database_name="$1" + local table_name="$2" + local schema="$3" + local mapping_name="${table_name}CsvMapping" + local mapping_json="[" + + # Build JSON mapping array + local schema_count + schema_count=$(echo "$schema" | jq 'length') + + for j in $(seq 0 $((schema_count - 1))); do + local col_name col_type kql_type + col_name=$(echo "$schema" | jq -r ".[$j].name") + col_type=$(echo "$schema" | jq -r ".[$j].type") + kql_type=$(map_kql_type "$col_type") + + if [[ "$j" -gt 0 ]]; then + mapping_json="$mapping_json," + fi + mapping_json="${mapping_json}{\"Name\":\"${col_name}\",\"DataType\":\"${kql_type}\",\"Ordinal\":${j}}" + done + + mapping_json="$mapping_json]" + + # Generate KQL command from template (strip comments) + local kql_command + kql_command=$(TABLE_NAME="$table_name" MAPPING_NAME="$mapping_name" MAPPING_JSON="$mapping_json" envsubst < "$TEMPLATE_DIR/create-mapping.kql.tmpl" | strip_kql_comments) + + info "Creating CSV mapping: $mapping_name" + execute_kql "$EVENTHOUSE_QUERY_URI" "$database_name" "$kql_command" +} + +set_kql_retention_policy() { + local database_name="$1" + local table_name="$2" + local retention_days="$3" + local caching_days="$4" + + # Generate KQL commands from template + local kql_commands + kql_commands=$(TABLE_NAME="$table_name" RETENTION_DAYS="$retention_days" CACHING_DAYS="$caching_days" envsubst < "$TEMPLATE_DIR/retention-policy.kql.tmpl") + + info "Setting retention policy: ${retention_days}d retention, ${caching_days}d caching" + + # Execute each command separately (retention and caching are separate commands) + while IFS= read -r command; do + # Skip comments and empty lines - trim leading whitespace + command="${command#"${command%%[![:space:]]*}"}" + if [[ -n "$command" && ! "$command" =~ ^// ]]; then + execute_kql "$EVENTHOUSE_QUERY_URI" "$database_name" "$command" + fi + done <<< "$kql_commands" +} + +ingest_kql_data() { + local database_name="$1" + local table_name="$2" + local source_url="$3" + local format="$4" + local mapping_name="${table_name}CsvMapping" + + info "Ingesting data from: $source_url" + + local kql_command + kql_command=".ingest into table ${table_name} (h\"${source_url}\") with (format=\"${format}\", ingestionMappingReference=\"${mapping_name}\")" + + execute_kql "$EVENTHOUSE_QUERY_URI" "$database_name" "$kql_command" +} + +#### +# Main Execution +#### + +log "Starting Data Sources Deployment" +info "Definition: $DEFINITION_FILE" +info "Workspace: $WORKSPACE_ID" +info "Dry run: $DRY_RUN" + +# Deploy Lakehouse +if [[ "$SKIP_LAKEHOUSE" != "true" ]]; then + deploy_lakehouse +else + info "Skipping Lakehouse deployment (--skip-lakehouse)" +fi + +# Deploy Eventhouse +if [[ "$SKIP_EVENTHOUSE" != "true" ]]; then + deploy_eventhouse +else + info "Skipping Eventhouse deployment (--skip-eventhouse)" +fi + +#### +# Output Summary +#### + +log "Deployment Complete" + +echo "" +echo "=== Data Sources Summary ===" +echo "" + +if [[ -n "$LAKEHOUSE_ID" ]]; then + echo "Lakehouse:" + echo " Name: ${LAKEHOUSE_NAME:-N/A}" + echo " ID: ${LAKEHOUSE_ID:-N/A}" + echo "" +fi + +if [[ -n "$EVENTHOUSE_ID" ]]; then + echo "Eventhouse:" + echo " Name: ${EVENTHOUSE_NAME:-N/A}" + echo " ID: ${EVENTHOUSE_ID:-N/A}" + echo "" + echo "KQL Database:" + echo " Name: ${KQL_DATABASE_NAME:-N/A}" + echo " ID: ${KQL_DATABASE_ID:-N/A}" + echo "" +fi + +# Output JSON for programmatic consumption +if [[ "$DRY_RUN" != "true" ]]; then + echo "" + echo "=== JSON Output ===" + jq -n \ + --arg lh_id "${LAKEHOUSE_ID:-}" \ + --arg lh_name "${LAKEHOUSE_NAME:-}" \ + --arg eh_id "${EVENTHOUSE_ID:-}" \ + --arg eh_name "${EVENTHOUSE_NAME:-}" \ + --arg db_id "${KQL_DATABASE_ID:-}" \ + --arg db_name "${KQL_DATABASE_NAME:-}" \ + '{ + lakehouse: {id: $lh_id, name: $lh_name}, + eventhouse: {id: $eh_id, name: $eh_name}, + kqlDatabase: {id: $db_id, name: $db_name} + }' +fi + +info "Data sources deployment complete" diff --git a/src/000-cloud/033-fabric-ontology/scripts/deploy-ontology.sh b/src/000-cloud/033-fabric-ontology/scripts/deploy-ontology.sh new file mode 100755 index 00000000..da16c706 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/scripts/deploy-ontology.sh @@ -0,0 +1,850 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1091 +# deploy-ontology.sh - Deploy Fabric Ontology from ontology definition +# +# Creates entity types, properties, data bindings, relationships, and contextualizations +# using the Microsoft Fabric Ontology REST API. +# +# Dependencies: curl, jq, yq, az (Azure CLI), uuidgen +# +# Usage: +# ./deploy-ontology.sh --definition --workspace-id --lakehouse-id \ +# --eventhouse-id --cluster-uri + +set -e +set -o pipefail + +# Script directory for relative paths +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source libraries +source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/definition-parser.sh" +source "$SCRIPT_DIR/lib/fabric-api.sh" + +#### +# Configuration +#### + +DEFINITION_FILE="" +WORKSPACE_ID="" +LAKEHOUSE_ID="" +EVENTHOUSE_ID="" +KQL_DATABASE_ID="" +CLUSTER_URI="" +DRY_RUN="false" + +# Associative arrays for ID tracking (entity name -> generated ID) +declare -A ENTITY_TYPE_IDS +declare -A PROPERTY_IDS +declare -A RELATIONSHIP_IDS + +#### +# Usage and Argument Parsing +#### + +usage() { + cat << EOF +Usage: $(basename "$0") [OPTIONS] + +Deploy Fabric Ontology from ontology definition. + +Required Arguments: + --definition Path to ontology definition YAML file + --workspace-id Fabric workspace ID (GUID) + --lakehouse-id Lakehouse ID for static data bindings (GUID) + +Conditional Arguments (required if eventhouse tables exist): + --eventhouse-id Eventhouse ID for time-series bindings (GUID) + --cluster-uri Kusto cluster URI (e.g., https://xxx.kusto.fabric.microsoft.com) + +Optional Arguments: + --kql-database-id KQL Database ID (for reference) + +Options: + --dry-run Show what would be created without making changes + -h, --help Show this help message + +Examples: + # Static data only (lakehouse) + $(basename "$0") --definition ./definitions/examples/lakeshore-retail.yaml \\ + --workspace-id abc123 --lakehouse-id def456 + + # With time-series data (lakehouse + eventhouse) + $(basename "$0") --definition ./definitions/examples/lakeshore-retail.yaml \\ + --workspace-id abc123 --lakehouse-id def456 \\ + --eventhouse-id ghi789 --cluster-uri https://xyz.kusto.fabric.microsoft.com +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --definition) + DEFINITION_FILE="$2" + shift 2 + ;; + --workspace-id) + WORKSPACE_ID="$2" + shift 2 + ;; + --lakehouse-id) + LAKEHOUSE_ID="$2" + shift 2 + ;; + --eventhouse-id) + EVENTHOUSE_ID="$2" + shift 2 + ;; + --kql-database-id) + KQL_DATABASE_ID="$2" + shift 2 + ;; + --cluster-uri) + CLUSTER_URI="$2" + shift 2 + ;; + --dry-run) + DRY_RUN="true" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + err "Unknown argument: $1" + ;; + esac +done + +# Validate required arguments +if [[ -z "$DEFINITION_FILE" ]]; then + err "Missing required argument: --definition" +fi + +if [[ ! -f "$DEFINITION_FILE" ]]; then + err "Definition file not found: $DEFINITION_FILE" +fi + +if [[ -z "$WORKSPACE_ID" ]]; then + err "Missing required argument: --workspace-id" +fi + +if [[ -z "$LAKEHOUSE_ID" ]]; then + err "Missing required argument: --lakehouse-id" +fi + +# Check for eventhouse requirements if time-series data exists +HAS_EVENTHOUSE=$(get_eventhouse_name "$DEFINITION_FILE") +if [[ -n "$HAS_EVENTHOUSE" && "$HAS_EVENTHOUSE" != "null" ]]; then + if [[ -z "$EVENTHOUSE_ID" ]]; then + err "Eventhouse ID required when definition contains eventhouse data sources (--eventhouse-id)" + fi + if [[ -z "$CLUSTER_URI" ]]; then + err "Cluster URI required when definition contains eventhouse data sources (--cluster-uri)" + fi +fi + +#### +# Validation +#### + +log "Validating Definition" +info "Validating definition: $DEFINITION_FILE" + +if ! "$SCRIPT_DIR/validate-definition.sh" --definition "$DEFINITION_FILE"; then + err "Definition validation failed" +fi + +info "Definition validation passed" + +#### +# Extract Metadata +#### + +log "Extracting Definition Metadata" + +ONTOLOGY_NAME=$(get_metadata_name "$DEFINITION_FILE") +ONTOLOGY_DESC=$(get_metadata_description "$DEFINITION_FILE") +DATABASE_NAME=$(get_eventhouse_database "$DEFINITION_FILE") + +info "Ontology: $ONTOLOGY_NAME" +info "Description: ${ONTOLOGY_DESC:-N/A}" + +#### +# Authentication +#### + +log "Authenticating to Fabric API" +FABRIC_TOKEN=$(get_fabric_token) +info "Authentication successful" + +#### +# Verify Workspace Access +#### + +log "Verifying Workspace Access" +workspace_response=$(get_workspace "$WORKSPACE_ID" "$FABRIC_TOKEN") +workspace_name=$(echo "$workspace_response" | jq -r '.displayName') +info "Workspace: $workspace_name ($WORKSPACE_ID)" + +#### +# ID Generation Functions +#### + +# Generate unique 64-bit ID (BigInt as string) +generate_bigint_id() { + local timestamp random_part + timestamp=$(date +%s%N | cut -c1-13) + random_part=$((RANDOM % 10000)) + printf "%s%04d" "$timestamp" "$random_part" +} + +# Generate UUID v4 +generate_uuid() { + if command -v uuidgen >/dev/null 2>&1; then + uuidgen | tr '[:upper:]' '[:lower:]' + else + # Fallback using /dev/urandom + od -x /dev/urandom | head -1 | awk '{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}' + fi +} + +# Get or generate entity type ID (uses pre-generated ID if available) +get_entity_type_id() { + local entity_name="$1" + if [[ -z "${ENTITY_TYPE_IDS[$entity_name]:-}" ]]; then + # This should not happen if pre_generate_ids was called + warn "Entity type ID not pre-generated for: $entity_name" + ENTITY_TYPE_IDS[$entity_name]=$(generate_bigint_id) + fi + echo "${ENTITY_TYPE_IDS[$entity_name]}" +} + +# Get or generate property ID (uses pre-generated ID if available) +get_property_id() { + local entity_name="$1" + local property_name="$2" + local key="${entity_name}:${property_name}" + if [[ -z "${PROPERTY_IDS[$key]:-}" ]]; then + # This should not happen if pre_generate_ids was called + warn "Property ID not pre-generated for: $key" + PROPERTY_IDS[$key]=$(generate_bigint_id) + fi + echo "${PROPERTY_IDS[$key]}" +} + +# Get or generate relationship ID (uses pre-generated ID if available) +get_relationship_id() { + local rel_name="$1" + if [[ -z "${RELATIONSHIP_IDS[$rel_name]:-}" ]]; then + # This should not happen if pre_generate_ids was called + warn "Relationship ID not pre-generated for: $rel_name" + RELATIONSHIP_IDS[$rel_name]=$(generate_bigint_id) + fi + echo "${RELATIONSHIP_IDS[$rel_name]}" +} + +#### +# JSON Generation Functions +#### + +# Build property JSON object +build_property_json() { + local prop_id="$1" + local prop_name="$2" + local prop_type="$3" + + local fabric_type + fabric_type=$(map_property_type "$prop_type") + + jq -n \ + --arg id "$prop_id" \ + --arg name "$prop_name" \ + --arg valueType "$fabric_type" \ + '{ + "id": $id, + "name": $name, + "redefines": null, + "baseTypeNamespaceType": null, + "valueType": $valueType + }' +} + +# Build property binding JSON object +build_property_binding() { + local source_column="$1" + local target_prop_id="$2" + + jq -n \ + --arg col "$source_column" \ + --arg propId "$target_prop_id" \ + '{ + "sourceColumnName": $col, + "targetPropertyId": $propId + }' +} + +# Build entity type definition +build_entity_type_definition() { + local entity_name="$1" + local entity_json="$2" + + local entity_id key_name display_name_prop + entity_id=$(get_entity_type_id "$entity_name") + key_name=$(echo "$entity_json" | jq -r '.key') + display_name_prop=$(echo "$entity_json" | jq -r '.displayName // .key') + + # Get key property ID + local key_prop_id + key_prop_id=$(get_property_id "$entity_name" "$key_name") + + # Get display name property ID + local display_prop_id + display_prop_id=$(get_property_id "$entity_name" "$display_name_prop") + + # Build properties array (static properties only) + local properties_array="[]" + local static_props + static_props=$(get_entity_static_properties "$DEFINITION_FILE" "$entity_name") + local prop_count + prop_count=$(echo "$static_props" | jq 'length') + + for i in $(seq 0 $((prop_count - 1))); do + local prop_name prop_type prop_id prop_json + prop_name=$(echo "$static_props" | jq -r ".[$i].name") + prop_type=$(echo "$static_props" | jq -r ".[$i].type") + prop_id=$(get_property_id "$entity_name" "$prop_name") + prop_json=$(build_property_json "$prop_id" "$prop_name" "$prop_type") + properties_array=$(echo "$properties_array" | jq --argjson prop "$prop_json" '. += [$prop]') + done + + # Build timeseries properties array + local timeseries_array="[]" + local ts_props + ts_props=$(get_entity_timeseries_properties "$DEFINITION_FILE" "$entity_name") + local ts_count + ts_count=$(echo "$ts_props" | jq 'length') + + for i in $(seq 0 $((ts_count - 1))); do + local prop_name prop_type prop_id prop_json + prop_name=$(echo "$ts_props" | jq -r ".[$i].name") + prop_type=$(echo "$ts_props" | jq -r ".[$i].type") + prop_id=$(get_property_id "$entity_name" "$prop_name") + prop_json=$(build_property_json "$prop_id" "$prop_name" "$prop_type") + timeseries_array=$(echo "$timeseries_array" | jq --argjson prop "$prop_json" '. += [$prop]') + done + + # Build entity ID parts (key property IDs) + local entity_id_parts + entity_id_parts=$(jq -n --arg id "$key_prop_id" '[$id]') + + # Build entity type JSON + jq -n \ + --arg entityId "$entity_id" \ + --arg entityName "$entity_name" \ + --argjson entityIdParts "$entity_id_parts" \ + --arg displayNamePropId "$display_prop_id" \ + --argjson properties "$properties_array" \ + --argjson timeseriesProps "$timeseries_array" \ + '{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/entityType/1.0.0/schema.json", + "id": $entityId, + "namespace": "usertypes", + "baseEntityTypeId": null, + "name": $entityName, + "entityIdParts": $entityIdParts, + "displayNamePropertyId": $displayNamePropId, + "namespaceType": "Custom", + "visibility": "Visible", + "properties": $properties, + "timeseriesProperties": $timeseriesProps + }' +} + +# Build Lakehouse data binding +build_lakehouse_binding() { + local entity_name="$1" + local binding_json="$2" + + local table_name binding_id + table_name=$(echo "$binding_json" | jq -r '.table') + binding_id=$(generate_uuid) + + # Build property bindings from entity properties + local property_bindings="[]" + local static_props + static_props=$(get_entity_static_properties "$DEFINITION_FILE" "$entity_name") + local prop_count + prop_count=$(echo "$static_props" | jq 'length') + + for i in $(seq 0 $((prop_count - 1))); do + local prop_name source_col prop_id binding + prop_name=$(echo "$static_props" | jq -r ".[$i].name") + source_col=$(echo "$static_props" | jq -r ".[$i].sourceColumn // .[$i].name") + prop_id=$(get_property_id "$entity_name" "$prop_name") + binding=$(build_property_binding "$source_col" "$prop_id") + property_bindings=$(echo "$property_bindings" | jq --argjson b "$binding" '. += [$b]') + done + + jq -n \ + --arg bindingId "$binding_id" \ + --argjson propBindings "$property_bindings" \ + --arg wsId "$WORKSPACE_ID" \ + --arg lhId "$LAKEHOUSE_ID" \ + --arg tableName "$table_name" \ + '{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/dataBinding/1.0.0/schema.json", + "id": $bindingId, + "dataBindingConfiguration": { + "dataBindingType": "NonTimeSeries", + "propertyBindings": $propBindings, + "sourceTableProperties": { + "sourceType": "LakehouseTable", + "workspaceId": $wsId, + "itemId": $lhId, + "sourceTableName": $tableName, + "sourceSchema": null + } + } + }' +} + +# Build Eventhouse data binding +build_eventhouse_binding() { + local entity_name="$1" + local binding_json="$2" + + local table_name timestamp_col binding_id + table_name=$(echo "$binding_json" | jq -r '.table') + timestamp_col=$(echo "$binding_json" | jq -r '.timestampColumn // "timestamp"') + binding_id=$(generate_uuid) + + # Build property bindings from timeseries properties + local property_bindings="[]" + local ts_props + ts_props=$(get_entity_timeseries_properties "$DEFINITION_FILE" "$entity_name") + local prop_count + prop_count=$(echo "$ts_props" | jq 'length') + + # Add correlation column binding (typically the entity key) + local key_name correlation_col key_prop_id key_binding + key_name=$(get_entity_key "$DEFINITION_FILE" "$entity_name") + correlation_col=$(echo "$binding_json" | jq -r '.correlationColumn // empty') + if [[ -n "$correlation_col" ]]; then + key_prop_id=$(get_property_id "$entity_name" "$key_name") + key_binding=$(build_property_binding "$correlation_col" "$key_prop_id") + property_bindings=$(echo "$property_bindings" | jq --argjson b "$key_binding" '. += [$b]') + fi + + for i in $(seq 0 $((prop_count - 1))); do + local prop_name source_col prop_id binding + prop_name=$(echo "$ts_props" | jq -r ".[$i].name") + source_col=$(echo "$ts_props" | jq -r ".[$i].sourceColumn // .[$i].name") + prop_id=$(get_property_id "$entity_name" "$prop_name") + binding=$(build_property_binding "$source_col" "$prop_id") + property_bindings=$(echo "$property_bindings" | jq --argjson b "$binding" '. += [$b]') + done + + # For KustoTable bindings, itemId should be the KQL Database ID + local kql_db_id="${KQL_DATABASE_ID:-$EVENTHOUSE_ID}" + + jq -n \ + --arg bindingId "$binding_id" \ + --arg tsCol "$timestamp_col" \ + --argjson propBindings "$property_bindings" \ + --arg wsId "$WORKSPACE_ID" \ + --arg kqlDbId "$kql_db_id" \ + --arg clusterUri "$CLUSTER_URI" \ + --arg dbName "$DATABASE_NAME" \ + --arg tableName "$table_name" \ + '{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/ontology/dataBinding/1.0.0/schema.json", + "id": $bindingId, + "dataBindingConfiguration": { + "dataBindingType": "TimeSeries", + "timestampColumnName": $tsCol, + "propertyBindings": $propBindings, + "sourceTableProperties": { + "sourceType": "KustoTable", + "workspaceId": $wsId, + "itemId": $kqlDbId, + "clusterUri": $clusterUri, + "databaseName": $dbName, + "sourceTableName": $tableName + } + } + }' +} + +# Build relationship type definition +build_relationship_definition() { + local rel_json="$1" + + local rel_name from_entity to_entity rel_id source_entity_id target_entity_id + rel_name=$(echo "$rel_json" | jq -r '.name') + from_entity=$(echo "$rel_json" | jq -r '.from') + to_entity=$(echo "$rel_json" | jq -r '.to') + + rel_id=$(get_relationship_id "$rel_name") + source_entity_id=$(get_entity_type_id "$from_entity") + target_entity_id=$(get_entity_type_id "$to_entity") + + jq -n \ + --arg relId "$rel_id" \ + --arg relName "$rel_name" \ + --arg srcId "$source_entity_id" \ + --arg tgtId "$target_entity_id" \ + '{ + "id": $relId, + "namespace": "usertypes", + "name": $relName, + "namespaceType": "Custom", + "source": {"entityTypeId": $srcId}, + "target": {"entityTypeId": $tgtId} + }' +} + +# Build contextualization (relationship data binding) +build_contextualization() { + local rel_json="$1" + + local rel_name from_entity to_entity binding ctx_id + rel_name=$(echo "$rel_json" | jq -r '.name') + from_entity=$(echo "$rel_json" | jq -r '.from') + to_entity=$(echo "$rel_json" | jq -r '.to') + binding=$(echo "$rel_json" | jq '.binding // null') + + if [[ "$binding" == "null" ]]; then + return 0 + fi + + ctx_id=$(generate_uuid) + local table_name from_col to_col + table_name=$(echo "$binding" | jq -r '.table') + from_col=$(echo "$binding" | jq -r '.fromColumn') + to_col=$(echo "$binding" | jq -r '.toColumn') + + # Get source entity key property ID + local from_key from_key_prop_id + from_key=$(get_entity_key "$DEFINITION_FILE" "$from_entity") + from_key_prop_id=$(get_property_id "$from_entity" "$from_key") + + # Get target entity key property ID + local to_key to_key_prop_id + to_key=$(get_entity_key "$DEFINITION_FILE" "$to_entity") + to_key_prop_id=$(get_property_id "$to_entity" "$to_key") + + # Build key ref bindings + local source_bindings target_bindings + source_bindings=$(jq -n \ + --arg col "$from_col" \ + --arg propId "$from_key_prop_id" \ + '[{"sourceColumnName": $col, "targetPropertyId": $propId}]') + + target_bindings=$(jq -n \ + --arg col "$to_col" \ + --arg propId "$to_key_prop_id" \ + '[{"sourceColumnName": $col, "targetPropertyId": $propId}]') + + jq -n \ + --arg ctxId "$ctx_id" \ + --arg wsId "$WORKSPACE_ID" \ + --arg lhId "$LAKEHOUSE_ID" \ + --arg tableName "$table_name" \ + --argjson srcBindings "$source_bindings" \ + --argjson tgtBindings "$target_bindings" \ + '{ + "id": $ctxId, + "dataBindingTable": { + "workspaceId": $wsId, + "itemId": $lhId, + "sourceTableName": $tableName, + "sourceSchema": null, + "sourceType": "LakehouseTable" + }, + "sourceKeyRefBindings": $srcBindings, + "targetKeyRefBindings": $tgtBindings + }' +} + +#### +# Pre-generate IDs +#### + +# Pre-generate all entity type IDs to avoid subshell issues with associative arrays +# Must be called before build_ontology_definition +pre_generate_ids() { + local entity_types entity_count + + entity_types=$(get_entity_types "$DEFINITION_FILE") + entity_count=$(echo "$entity_types" | jq 'length') + + for i in $(seq 0 $((entity_count - 1))); do + local entity_name + entity_name=$(echo "$entity_types" | jq -r ".[$i].name") + # Generate and cache the entity type ID + ENTITY_TYPE_IDS[$entity_name]=$(generate_bigint_id) + + # Pre-generate property IDs for this entity + local static_props ts_props prop_count + static_props=$(get_entity_static_properties "$DEFINITION_FILE" "$entity_name") + prop_count=$(echo "$static_props" | jq 'length') + for j in $(seq 0 $((prop_count - 1))); do + local prop_name + prop_name=$(echo "$static_props" | jq -r ".[$j].name") + PROPERTY_IDS["${entity_name}:${prop_name}"]=$(generate_bigint_id) + done + + ts_props=$(get_entity_timeseries_properties "$DEFINITION_FILE" "$entity_name") + prop_count=$(echo "$ts_props" | jq 'length') + for j in $(seq 0 $((prop_count - 1))); do + local prop_name + prop_name=$(echo "$ts_props" | jq -r ".[$j].name") + PROPERTY_IDS["${entity_name}:${prop_name}"]=$(generate_bigint_id) + done + done + + # Pre-generate relationship IDs + local relationships rel_count + relationships=$(get_relationships "$DEFINITION_FILE") + rel_count=$(echo "$relationships" | jq 'length') + + for i in $(seq 0 $((rel_count - 1))); do + local rel_name + rel_name=$(echo "$relationships" | jq -r ".[$i].name") + RELATIONSHIP_IDS[$rel_name]=$(generate_bigint_id) + done +} + +#### +# Build Ontology Definition Parts +#### + +build_ontology_definition() { + local parts_array="[]" + + log "Building Ontology Definition Parts" + + # 1. Platform metadata + local platform_json + platform_json=$(jq -n \ + --arg name "$ONTOLOGY_NAME" \ + '{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": {"type": "Ontology", "displayName": $name}, + "config": {"version": "2.0", "logicalId": "00000000-0000-0000-0000-000000000000"} + }') + local platform_part + platform_part=$(build_definition_part ".platform" "$platform_json") + parts_array=$(echo "$parts_array" | jq --argjson p "$platform_part" '. += [$p]') + info "Added: .platform" + + # 2. Root definition.json (empty object) + local root_def_part + root_def_part=$(build_definition_part "definition.json" "{}") + parts_array=$(echo "$parts_array" | jq --argjson p "$root_def_part" '. += [$p]') + info "Added: definition.json" + + # 3. Entity types and their data bindings + local entity_types entity_count + entity_types=$(get_entity_types "$DEFINITION_FILE") + entity_count=$(echo "$entity_types" | jq 'length') + info "Processing $entity_count entity types" + + for i in $(seq 0 $((entity_count - 1))); do + local entity_name entity_json entity_id entity_def entity_def_part + entity_name=$(echo "$entity_types" | jq -r ".[$i].name") + entity_json=$(echo "$entity_types" | jq ".[$i]") + # Use pre-generated ID from associative array directly + entity_id="${ENTITY_TYPE_IDS[$entity_name]}" + + # Build entity type definition + entity_def=$(build_entity_type_definition "$entity_name" "$entity_json") + entity_def_part=$(build_definition_part "EntityTypes/${entity_id}/definition.json" "$entity_def") + parts_array=$(echo "$parts_array" | jq --argjson p "$entity_def_part" '. += [$p]') + info "Added: EntityTypes/${entity_id}/definition.json ($entity_name)" + + # Add static (lakehouse) data binding + local static_binding + static_binding=$(get_entity_static_binding "$DEFINITION_FILE" "$entity_name") + if [[ -n "$static_binding" && "$static_binding" != "null" ]]; then + local lh_binding binding_id lh_binding_part + lh_binding=$(build_lakehouse_binding "$entity_name" "$static_binding") + binding_id=$(echo "$lh_binding" | jq -r '.id') + lh_binding_part=$(build_definition_part "EntityTypes/${entity_id}/DataBindings/${binding_id}.json" "$lh_binding") + parts_array=$(echo "$parts_array" | jq --argjson p "$lh_binding_part" '. += [$p]') + info "Added: EntityTypes/${entity_id}/DataBindings/${binding_id}.json (Lakehouse)" + fi + + # Add timeseries (eventhouse) data binding + local ts_binding + ts_binding=$(get_entity_timeseries_binding "$DEFINITION_FILE" "$entity_name") + if [[ -n "$ts_binding" && "$ts_binding" != "null" ]]; then + local eh_binding binding_id eh_binding_part + eh_binding=$(build_eventhouse_binding "$entity_name" "$ts_binding") + binding_id=$(echo "$eh_binding" | jq -r '.id') + eh_binding_part=$(build_definition_part "EntityTypes/${entity_id}/DataBindings/${binding_id}.json" "$eh_binding") + parts_array=$(echo "$parts_array" | jq --argjson p "$eh_binding_part" '. += [$p]') + info "Added: EntityTypes/${entity_id}/DataBindings/${binding_id}.json (Eventhouse)" + fi + done + + # 4. Relationship types and contextualizations + local relationships rel_count + relationships=$(get_relationships "$DEFINITION_FILE") + rel_count=$(echo "$relationships" | jq 'length') + info "Processing $rel_count relationships" + + for i in $(seq 0 $((rel_count - 1))); do + local rel_json rel_name rel_id rel_def rel_def_part + rel_json=$(echo "$relationships" | jq ".[$i]") + rel_name=$(echo "$rel_json" | jq -r '.name') + rel_id=$(get_relationship_id "$rel_name") + + # Build relationship type definition + rel_def=$(build_relationship_definition "$rel_json") + rel_def_part=$(build_definition_part "RelationshipTypes/${rel_id}/definition.json" "$rel_def") + parts_array=$(echo "$parts_array" | jq --argjson p "$rel_def_part" '. += [$p]') + info "Added: RelationshipTypes/${rel_id}/definition.json ($rel_name)" + + # Add contextualization if binding exists + local ctx_def + ctx_def=$(build_contextualization "$rel_json") + if [[ -n "$ctx_def" ]]; then + local ctx_id ctx_part + ctx_id=$(echo "$ctx_def" | jq -r '.id') + ctx_part=$(build_definition_part "RelationshipTypes/${rel_id}/Contextualizations/${ctx_id}.json" "$ctx_def") + parts_array=$(echo "$parts_array" | jq --argjson p "$ctx_part" '. += [$p]') + info "Added: RelationshipTypes/${rel_id}/Contextualizations/${ctx_id}.json" + fi + done + + echo "$parts_array" +} + +#### +# Create or Update Ontology +#### + +create_ontology() { + local definition_parts="$1" + + log "Creating Ontology" + + # Check if ontology already exists + local existing_response ontology_id + existing_response=$(fabric_api_call "GET" "/workspaces/$WORKSPACE_ID/ontologies" "" "$FABRIC_TOKEN" 2>/dev/null || echo '{"value":[]}') + ontology_id=$(echo "$existing_response" | jq -r ".value[] | select(.displayName == \"$ONTOLOGY_NAME\") | .id") + + if [[ -n "$ontology_id" ]]; then + info "Ontology '$ONTOLOGY_NAME' already exists: $ontology_id" + info "Updating definition..." + + if [[ "$DRY_RUN" == "true" ]]; then + info "[DRY-RUN] Would update ontology definition" + echo "$ontology_id" + return 0 + fi + + # Update existing ontology definition + local update_body + update_body=$(jq -n --argjson parts "$definition_parts" '{"definition": {"parts": $parts}}') + + fabric_api_call "POST" "/workspaces/$WORKSPACE_ID/ontologies/$ontology_id/updateDefinition" "$update_body" "$FABRIC_TOKEN" + ok "Ontology definition updated" + echo "$ontology_id" + return 0 + fi + + # Create new ontology with definition + info "Creating ontology: $ONTOLOGY_NAME" + + if [[ "$DRY_RUN" == "true" ]]; then + info "[DRY-RUN] Would create ontology: $ONTOLOGY_NAME" + local parts_count + parts_count=$(echo "$definition_parts" | jq 'length') + info "[DRY-RUN] Definition parts count: $parts_count" + echo "dry-run-ontology-id" + return 0 + fi + + # Write parts to temp file to avoid shell argument length limits + local parts_file request_body_file response + parts_file=$(mktemp) + request_body_file=$(mktemp) + echo "$definition_parts" > "$parts_file" + + # Build request body using file-based approach + jq -n \ + --arg name "$ONTOLOGY_NAME" \ + --arg desc "${ONTOLOGY_DESC:-}" \ + --slurpfile parts "$parts_file" \ + '{ + "displayName": $name, + "description": $desc, + "definition": {"parts": $parts[0]} + }' > "$request_body_file" + + rm -f "$parts_file" + + # Save request body for debugging + cp "$request_body_file" /tmp/ontology-request.json + info "Request body saved to /tmp/ontology-request.json" + + response=$(fabric_api_call_file "POST" "/workspaces/$WORKSPACE_ID/ontologies" "$request_body_file" "$FABRIC_TOKEN") + rm -f "$request_body_file" + + ontology_id=$(echo "$response" | jq -r '.id // empty') + if [[ -z "$ontology_id" ]]; then + # May be in createdItem for LRO + ontology_id=$(echo "$response" | jq -r '.createdItem.id // empty') + fi + + if [[ -n "$ontology_id" ]]; then + ok "Ontology created: $ontology_id" + echo "$ontology_id" + else + err "Failed to create ontology - no ID returned" + fi +} + +#### +# Main +#### + +log "Deploying Fabric Ontology" +info "Ontology: $ONTOLOGY_NAME" +info "Workspace: $WORKSPACE_ID" +info "Lakehouse: $LAKEHOUSE_ID" +if [[ -n "$EVENTHOUSE_ID" ]]; then + info "Eventhouse: $EVENTHOUSE_ID" + info "Cluster URI: $CLUSTER_URI" +fi +if [[ "$DRY_RUN" == "true" ]]; then + warn "DRY-RUN mode enabled" +fi + +# Pre-generate all IDs to avoid subshell issues with associative arrays +pre_generate_ids + +# Build ontology definition parts +DEFINITION_PARTS=$(build_ontology_definition) + +parts_count=$(echo "$DEFINITION_PARTS" | jq 'length') +info "Total definition parts: $parts_count" + +# Create or update ontology +ONTOLOGY_ID=$(create_ontology "$DEFINITION_PARTS") + +log "Deployment Complete" +ok "Ontology ID: $ONTOLOGY_ID" +warn "Ontology setup is async - entity types take 10-20 minutes to fully provision" +info "The portal will show 'Setting up your ontology' until complete" + +# Output for scripting +if [[ "$DRY_RUN" != "true" ]]; then + echo "" + echo "# Environment variables for downstream scripts:" + echo "export ONTOLOGY_ID=\"$ONTOLOGY_ID\"" +fi diff --git a/src/000-cloud/033-fabric-ontology/scripts/deploy-semantic-model.sh b/src/000-cloud/033-fabric-ontology/scripts/deploy-semantic-model.sh new file mode 100755 index 00000000..64818188 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/scripts/deploy-semantic-model.sh @@ -0,0 +1,621 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1091 +# deploy-semantic-model.sh - Deploy Direct Lake Semantic Model from ontology definition +# +# Dependencies: curl, jq, yq, az (Azure CLI), uuidgen +# +# Usage: +# ./deploy-semantic-model.sh --definition --workspace-id --lakehouse-id +# ./deploy-semantic-model.sh --definition ./definitions/examples/lakeshore-retail.yaml \ +# --workspace-id abc123 --lakehouse-id def456 + +set -e +set -o pipefail + +# Script directory for relative paths +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEMPLATE_DIR="$SCRIPT_DIR/../templates/semantic-model" + +# Source libraries +source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/definition-parser.sh" +source "$SCRIPT_DIR/lib/fabric-api.sh" + +#### +# Configuration +#### + +DEFINITION_FILE="" +WORKSPACE_ID="" +LAKEHOUSE_ID="" +DRY_RUN="false" + +#### +# Usage and Argument Parsing +#### + +usage() { + cat << EOF +Usage: $(basename "$0") [OPTIONS] + +Deploy Direct Lake Semantic Model from ontology definition. + +Required Arguments: + --definition Path to ontology definition YAML file + --workspace-id Fabric workspace ID (GUID) + --lakehouse-id Lakehouse ID for Direct Lake binding (GUID) + +Options: + --dry-run Show what would be created without making changes + -h, --help Show this help message + +Examples: + $(basename "$0") --definition ./definitions/examples/lakeshore-retail.yaml \\ + --workspace-id abc123 --lakehouse-id def456 + + $(basename "$0") --definition ./my-ontology.yaml \\ + --workspace-id abc123 --lakehouse-id def456 --dry-run +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --definition) + DEFINITION_FILE="$2" + shift 2 + ;; + --workspace-id) + WORKSPACE_ID="$2" + shift 2 + ;; + --lakehouse-id) + LAKEHOUSE_ID="$2" + shift 2 + ;; + --dry-run) + DRY_RUN="true" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + err "Unknown argument: $1" + ;; + esac +done + +# Validate required arguments +if [[ -z "$DEFINITION_FILE" ]]; then + err "Missing required argument: --definition" +fi + +if [[ ! -f "$DEFINITION_FILE" ]]; then + err "Definition file not found: $DEFINITION_FILE" +fi + +if [[ -z "$WORKSPACE_ID" ]]; then + err "Missing required argument: --workspace-id" +fi + +if [[ -z "$LAKEHOUSE_ID" ]]; then + err "Missing required argument: --lakehouse-id" +fi + +#### +# Validation +#### + +log "Validating Definition" +info "Validating definition: $DEFINITION_FILE" + +if ! "$SCRIPT_DIR/validate-definition.sh" --definition "$DEFINITION_FILE"; then + err "Definition validation failed" +fi + +info "Definition validation passed" + +#### +# Extract Metadata +#### + +log "Extracting Definition Metadata" + +ONTOLOGY_NAME=$(get_metadata_name "$DEFINITION_FILE") +MODEL_NAME=$(get_semantic_model_name "$DEFINITION_FILE") + +if [[ -z "$MODEL_NAME" ]]; then + MODEL_NAME="${ONTOLOGY_NAME}Model" + warn "No semantic model name specified, using default: $MODEL_NAME" +fi + +info "Ontology: $ONTOLOGY_NAME" +info "Semantic Model: $MODEL_NAME" + +#### +# Authentication +#### + +log "Authenticating to Fabric API" +FABRIC_TOKEN=$(get_fabric_token) +info "Authentication successful" + +#### +# Verify Workspace Access +#### + +log "Verifying Workspace Access" +workspace_response=$(get_workspace "$WORKSPACE_ID" "$FABRIC_TOKEN") +workspace_name=$(echo "$workspace_response" | jq -r '.displayName') +info "Workspace: $workspace_name ($WORKSPACE_ID)" + +#### +# Type Mapping Functions +#### + +map_tmdl_type() { + local def_type="$1" + case "$def_type" in + string) echo "string" ;; + int|integer) echo "int64" ;; + double|float|decimal) echo "double" ;; + datetime) echo "dateTime" ;; + boolean|bool) echo "boolean" ;; + *) echo "string" ;; + esac +} + +#### +# TMDL Generation Functions +#### + +generate_uuid() { + # Generate UUID using available method (portable across platforms) + if command -v uuidgen &>/dev/null; then + uuidgen | tr '[:upper:]' '[:lower:]' + elif [[ -r /proc/sys/kernel/random/uuid ]]; then + cat /proc/sys/kernel/random/uuid + else + # Fallback: generate pseudo-UUID from random bytes + printf '%04x%04x-%04x-%04x-%04x-%04x%04x%04x' \ + $((RANDOM)) $((RANDOM)) $((RANDOM)) \ + $(((RANDOM & 0x0fff) | 0x4000)) \ + $(((RANDOM & 0x3fff) | 0x8000)) \ + $((RANDOM)) $((RANDOM)) $((RANDOM)) + fi +} + +generate_database_tmdl() { + local model_name="$1" + MODEL_NAME="$model_name" envsubst < "$TEMPLATE_DIR/database.tmdl.tmpl" +} + +generate_expressions_tmdl() { + local workspace_id="$1" + local lakehouse_id="$2" + WORKSPACE_ID="$workspace_id" LAKEHOUSE_ID="$lakehouse_id" envsubst < "$TEMPLATE_DIR/expressions.tmdl.tmpl" +} + +generate_table_refs() { + local entity_types entity_count entity_name + entity_types=$(get_entity_types "$DEFINITION_FILE") + entity_count=$(echo "$entity_types" | jq 'length') + + for i in $(seq 0 $((entity_count - 1))); do + entity_name=$(echo "$entity_types" | jq -r ".[$i].name") + echo "ref table '$entity_name'" + done +} + +generate_model_tmdl() { + local table_refs + table_refs=$(generate_table_refs) + TABLE_REFS="$table_refs" envsubst < "$TEMPLATE_DIR/model.tmdl.tmpl" +} + +generate_table_tmdl() { + local entity_name="$1" + local entity_json="$2" + local output_file="$3" + local key_prop lineage_tag source_table + + key_prop=$(echo "$entity_json" | jq -r '.key') + lineage_tag=$(generate_uuid) + + # Get data binding source table + local data_binding + data_binding=$(echo "$entity_json" | jq -r '.dataBinding.table // .dataBindings[0].table // empty') + if [[ -z "$data_binding" ]]; then + source_table=$(echo "$entity_name" | tr '[:upper:]' '[:lower:]') + else + source_table="$data_binding" + fi + + # Write table header directly to file + { + echo "table '$entity_name'" + echo " lineageTag: $lineage_tag" + echo "" + echo " partition '$entity_name-Partition' = entity" + echo " mode: directLake" + echo " entityName: $source_table" + echo " schemaName: dbo" + echo " expressionSource: DatabaseQuery" + echo "" + } > "$output_file" + + # Write columns directly to file + local properties prop_count prop_name prop_type source_col is_key binding tmdl_type summarize_by + properties=$(echo "$entity_json" | jq '.properties // []') + prop_count=$(echo "$properties" | jq 'length') + + for j in $(seq 0 $((prop_count - 1))); do + prop_name=$(echo "$properties" | jq -r ".[$j].name") + prop_type=$(echo "$properties" | jq -r ".[$j].type") + source_col=$(echo "$properties" | jq -r ".[$j].sourceColumn // .[$j].name") + + # Check if this property is the key + if [[ "$prop_name" == "$key_prop" ]]; then + is_key="true" + else + is_key="false" + fi + + # Only include static/lakehouse-bound properties in semantic model + binding=$(echo "$properties" | jq -r ".[$j].binding // \"static\"") + if [[ "$binding" == "timeseries" ]]; then + continue + fi + + tmdl_type=$(map_tmdl_type "$prop_type") + + # Determine summarizeBy based on type and key status + case "$tmdl_type" in + int64|double) + if [[ "$is_key" == "true" ]]; then + summarize_by="none" + else + summarize_by="sum" + fi + ;; + *) + summarize_by="none" + ;; + esac + + # Write column directly to file + { + echo " column '$prop_name'" + echo " dataType: $tmdl_type" + if [[ "$is_key" == "true" ]]; then + echo " isKey" + fi + echo " sourceColumn: $source_col" + echo " summarizeBy: $summarize_by" + echo "" + } >> "$output_file" + done +} + +generate_relationships_tmdl() { + local relationships rel_count from_entity to_entity rel_guid + relationships=$(get_relationships "$DEFINITION_FILE") + rel_count=$(echo "$relationships" | jq 'length') + + if [[ "$rel_count" -eq 0 ]]; then + echo "// No relationships defined" + return + fi + + local entity_types from_key to_key + entity_types=$(get_entity_types "$DEFINITION_FILE") + + for i in $(seq 0 $((rel_count - 1))); do + from_entity=$(echo "$relationships" | jq -r ".[$i].from") + to_entity=$(echo "$relationships" | jq -r ".[$i].to") + rel_guid=$(generate_uuid) + + # Get primary key columns from entity definitions for semantic model relationships + # Note: binding.fromColumn/toColumn are for bridge tables, not semantic model relationships + from_key=$(echo "$entity_types" | jq -r ".[] | select(.name == \"$from_entity\") | .key") + to_key=$(echo "$entity_types" | jq -r ".[] | select(.name == \"$to_entity\") | .key") + + # TMDL relationships connect entity tables via their primary keys + # fromColumn references the "many" side (from entity's key) + # toColumn references the "one" side (to entity's key) + echo "relationship $rel_guid" + echo " fromColumn: '$from_entity'.'$from_key'" + echo " toColumn: '$to_entity'.'$to_key'" + echo "" + done +} + +#### +# Build Semantic Model Definition +#### + +build_semantic_model_definition() { + local temp_dir database_tmdl model_tmdl expressions_tmdl relationships_tmdl pbism_content + + temp_dir=$(mktemp -d) + mkdir -p "$temp_dir/definition/tables" + + info "Generating TMDL files in: $temp_dir" >&2 + + # Generate database.tmdl + database_tmdl=$(generate_database_tmdl "$MODEL_NAME") + echo "$database_tmdl" > "$temp_dir/definition/database.tmdl" + info "Generated: database.tmdl" >&2 + + # Generate model.tmdl + model_tmdl=$(generate_model_tmdl) + echo "$model_tmdl" > "$temp_dir/definition/model.tmdl" + info "Generated: model.tmdl" >&2 + + # Generate expressions.tmdl + expressions_tmdl=$(generate_expressions_tmdl "$WORKSPACE_ID" "$LAKEHOUSE_ID") + echo "$expressions_tmdl" > "$temp_dir/definition/expressions.tmdl" + info "Generated: expressions.tmdl" >&2 + + # Generate table TMDL files + local entity_types entity_count entity_name entity_json + entity_types=$(get_entity_types "$DEFINITION_FILE") + entity_count=$(echo "$entity_types" | jq 'length') + + for i in $(seq 0 $((entity_count - 1))); do + entity_name=$(echo "$entity_types" | jq -r ".[$i].name") + entity_json=$(echo "$entity_types" | jq ".[$i]") + + # Skip entities that only have timeseries binding (no lakehouse table) + local has_static_binding + has_static_binding=$(echo "$entity_json" | jq -r ' + if .dataBinding then + .dataBinding.type == "static" + elif .dataBindings then + [.dataBindings[] | select(.type == "static")] | length > 0 + else + false + end + ') + + if [[ "$has_static_binding" != "true" ]]; then + info "Skipping entity $entity_name (no static data binding)" >&2 + continue + fi + + generate_table_tmdl "$entity_name" "$entity_json" "$temp_dir/definition/tables/$entity_name.tmdl" + info "Generated: tables/$entity_name.tmdl" >&2 + done + + # Generate relationships.tmdl + relationships_tmdl=$(generate_relationships_tmdl) + echo "$relationships_tmdl" > "$temp_dir/definition/relationships.tmdl" + info "Generated: relationships.tmdl" >&2 + + # Generate definition.pbism (required for TMDL format, version 4.0+) + local pbism_content + pbism_content=$(cat "$TEMPLATE_DIR/definition.pbism.tmpl") + echo "$pbism_content" > "$temp_dir/definition.pbism" + info "Generated: definition.pbism" >&2 + + echo "$temp_dir" +} + +#### +# Create Semantic Model via API +#### + +create_semantic_model() { + local temp_dir="$1" + local parts_file + parts_file=$(mktemp) + echo "[]" > "$parts_file" + + # Build definition parts from generated files using find for recursive traversal + # Store file list first to avoid subshell issues with while loop + local file_list + file_list=$(find "$temp_dir" -type f) + + while IFS= read -r file; do + [[ -z "$file" ]] && continue + local rel_path content_b64 + # Get path relative to temp_dir + rel_path="${file#"$temp_dir"/}" + + # Base64 encode + content_b64=$(base64 < "$file" | tr -d '\n\r') + + # Build part object and append to array + local current_parts new_parts + current_parts=$(cat "$parts_file") + new_parts=$(echo "$current_parts" | jq \ + --arg path "$rel_path" \ + --arg payload "$content_b64" \ + '. += [{"path": $path, "payload": $payload, "payloadType": "InlineBase64"}]') + echo "$new_parts" > "$parts_file" + done <<< "$file_list" + + local parts_array + parts_array=$(cat "$parts_file") + rm -f "$parts_file" + + # Verify we have parts + local parts_count + parts_count=$(echo "$parts_array" | jq 'length') + if [[ "$parts_count" -eq 0 ]]; then + err "No definition parts generated" + fi + info "Built $parts_count definition parts" >&2 + + # Build request body using file to avoid shell quoting issues + local request_body_file + request_body_file=$(mktemp) + echo "$parts_array" > "${request_body_file}.parts" + + if ! jq -n \ + --arg name "$MODEL_NAME" \ + --slurpfile parts "${request_body_file}.parts" \ + '{ + "displayName": $name, + "definition": { + "format": "TMDL", + "parts": $parts[0] + } + }' > "$request_body_file" 2>&1; then + # Alternative: read file content directly + jq -n \ + --arg name "$MODEL_NAME" \ + --argjson parts "$(cat "${request_body_file}.parts")" \ + '{ + "displayName": $name, + "definition": { + "format": "TMDL", + "parts": $parts + } + }' > "$request_body_file" + fi + + rm -f "${request_body_file}.parts" + local request_body + request_body=$(cat "$request_body_file") + + if [[ "$DRY_RUN" == "true" ]]; then + info "[DRY-RUN] Would create semantic model: $MODEL_NAME" >&2 + info "[DRY-RUN] Definition parts count: $parts_count" >&2 + rm -f "$request_body_file" + echo '{"id": "dry-run-id", "displayName": "'"$MODEL_NAME"'"}' + return 0 + fi + + # Check if semantic model already exists + local list_response existing_model + list_response=$(fabric_api_call "GET" "/workspaces/$WORKSPACE_ID/semanticModels" "" "$FABRIC_TOKEN") + existing_model=$(echo "$list_response" | jq -r ".value[] | select(.displayName == \"$MODEL_NAME\")") + + if [[ -n "$existing_model" ]]; then + local existing_id + existing_id=$(echo "$existing_model" | jq -r '.id') + info "Semantic model '$MODEL_NAME' already exists: $existing_id" >&2 + + # Update definition using file-based approach + local update_body_file + update_body_file=$(mktemp) + echo "$parts_array" > "${update_body_file}.parts" + + jq -n \ + --slurpfile parts "${update_body_file}.parts" \ + '{ + "definition": { + "format": "TMDL", + "parts": $parts[0] + } + }' > "$update_body_file" + + rm -f "${update_body_file}.parts" + local update_body + update_body=$(cat "$update_body_file") + rm -f "$update_body_file" + + info "Updating semantic model definition..." >&2 + fabric_api_call "POST" "/workspaces/$WORKSPACE_ID/semanticModels/$existing_id/updateDefinition" "$update_body" "$FABRIC_TOKEN" || true + + rm -f "$request_body_file" + echo "$existing_model" + return 0 + fi + + # Create new semantic model + info "Creating semantic model: $MODEL_NAME" >&2 + local response + if ! response=$(fabric_api_call "POST" "/workspaces/$WORKSPACE_ID/semanticModels" "$request_body" "$FABRIC_TOKEN"); then + err "Failed to create semantic model" + fi + rm -f "$request_body_file" + + echo "$response" +} + +#### +# Main Execution +#### + +log "Starting Semantic Model Deployment" +info "Definition: $DEFINITION_FILE" +info "Workspace: $WORKSPACE_ID" +info "Lakehouse: $LAKEHOUSE_ID" +info "Dry run: $DRY_RUN" + +# Build TMDL definition +log "Generating TMDL Definition" +TEMP_DIR=$(build_semantic_model_definition) + +# Create semantic model +log "Deploying Semantic Model" +if ! response=$(create_semantic_model "$TEMP_DIR"); then + rm -rf "$TEMP_DIR" + exit 1 +fi + +# Handle long-running operation success response (status: Succeeded but no id) +# Need to look up the created semantic model by name +if echo "$response" | jq -e '.status == "Succeeded"' >/dev/null 2>&1; then + info "Operation succeeded, looking up semantic model by name..." + list_response=$(fabric_api_call "GET" "/workspaces/$WORKSPACE_ID/semanticModels" "" "$FABRIC_TOKEN") + response=$(echo "$list_response" | jq -r ".value[] | select(.displayName == \"$MODEL_NAME\")") +fi + +# Handle null or empty response +if [[ -z "$response" || "$response" == "null" ]]; then + rm -rf "$TEMP_DIR" + err "Received empty or null response from API" +fi + +# Validate response is JSON before parsing +if ! echo "$response" | jq -e . >/dev/null 2>&1; then + rm -rf "$TEMP_DIR" + err "Invalid JSON response: $response" +fi + +SEMANTIC_MODEL_ID=$(echo "$response" | jq -r '.id // empty') +SEMANTIC_MODEL_NAME=$(echo "$response" | jq -r '.displayName // empty') + +if [[ -z "$SEMANTIC_MODEL_ID" || "$SEMANTIC_MODEL_ID" == "null" ]]; then + err "Failed to get Semantic Model ID" +fi + +export SEMANTIC_MODEL_ID +export SEMANTIC_MODEL_NAME +info "Semantic Model ID: $SEMANTIC_MODEL_ID" + +# Cleanup +rm -rf "$TEMP_DIR" + +#### +# Output Summary +#### + +log "Deployment Complete" + +cat << EOF + +=== Semantic Model Summary === + +Semantic Model: + Name: $SEMANTIC_MODEL_NAME + ID: $SEMANTIC_MODEL_ID + +Data Source: + Workspace: $WORKSPACE_ID + Lakehouse: $LAKEHOUSE_ID + +=== JSON Output === +{ + "semanticModel": { + "id": "$SEMANTIC_MODEL_ID", + "name": "$SEMANTIC_MODEL_NAME" + } +} +EOF + +info "Semantic model deployment complete" diff --git a/src/000-cloud/033-fabric-ontology/scripts/deploy.sh b/src/000-cloud/033-fabric-ontology/scripts/deploy.sh new file mode 100755 index 00000000..24df4f94 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/scripts/deploy.sh @@ -0,0 +1,517 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1091 +# deploy.sh - Deploy complete ontology with local data +# +# Generic deployment script that deploys any ontology definition along with +# local CSV/Parquet data files. This is the recommended entry point for +# deploying custom ontologies. +# +# Deployment sequence: +# 1. Validate definition +# 2. Create Lakehouse and load data from --data-dir +# 3. Create Semantic Model (Direct Lake) +# 4. Create Ontology (entity types, relationships) +# +# Dependencies: curl, jq, yq, az (Azure CLI) +# +# Usage: +# ./deploy.sh --definition --workspace-id [--data-dir ] +# +# Examples: +# # Deploy with local data directory +# ./deploy.sh --definition ./scratchpad/smart-building/smart-building.yaml \ +# --workspace-id abc123 --data-dir ./scratchpad/smart-building/data +# +# # Deploy using sourceUrl/sourceFile from YAML (no --data-dir) +# ./deploy.sh --definition ./definitions/examples/lakeshore-retail.yaml \ +# --workspace-id abc123 + +set -e +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/definition-parser.sh" +source "$SCRIPT_DIR/lib/fabric-api.sh" + +#### +# Configuration +#### + +DEFINITION_FILE="" +WORKSPACE_ID="" +DATA_DIR="" +SKIP_DATA_SOURCES="false" +SKIP_SEMANTIC_MODEL="false" +SKIP_ONTOLOGY="false" +DRY_RUN="false" + +# Resource IDs populated during deployment +LAKEHOUSE_ID="" +LAKEHOUSE_NAME="" +EVENTHOUSE_ID="" +KQL_DATABASE_ID="" +CLUSTER_URI="" + +#### +# Usage +#### + +usage() { + cat << EOF +Usage: $(basename "$0") [OPTIONS] + +Deploy a complete ontology with data to Microsoft Fabric. + +This script orchestrates the full deployment: + 1. Data Sources: Creates Lakehouse, uploads data, loads Delta tables + 2. Semantic Model: Creates Direct Lake semantic model + 3. Ontology: Creates entity types, relationships, and data bindings + +Required Arguments: + --definition Path to ontology definition YAML file + --workspace-id Fabric workspace ID (GUID) + +Data Source Arguments (one of): + --data-dir Directory containing CSV/Parquet files to upload + File names must match table names in definition + (e.g., buildings.csv for table "buildings") + (or) Definition YAML contains sourceUrl/sourceFile entries + +Optional Arguments: + --skip-data-sources Skip data source creation (use existing Lakehouse) + --skip-semantic-model Skip semantic model creation + --skip-ontology Skip ontology creation + --lakehouse-id Use existing Lakehouse (required with --skip-data-sources) + --dry-run Show deployment plan without making changes + -h, --help Show this help message + +Examples: + # Deploy custom ontology with local data + $(basename "$0") --definition ./my-ontology.yaml \\ + --workspace-id abc123 --data-dir ./my-data/ + + # Deploy example with remote data (sourceUrl in YAML) + $(basename "$0") --definition ./definitions/examples/lakeshore-retail.yaml \\ + --workspace-id abc123 + + # Skip data loading, deploy to existing Lakehouse + $(basename "$0") --definition ./my-ontology.yaml \\ + --workspace-id abc123 --skip-data-sources --lakehouse-id def456 + + # Dry run to preview deployment + $(basename "$0") --definition ./my-ontology.yaml \\ + --workspace-id abc123 --data-dir ./my-data/ --dry-run +EOF +} + +#### +# Argument Parsing +#### + +while [[ $# -gt 0 ]]; do + case "$1" in + --definition) + DEFINITION_FILE="$2" + shift 2 + ;; + --workspace-id) + WORKSPACE_ID="$2" + shift 2 + ;; + --data-dir) + DATA_DIR="$2" + shift 2 + ;; + --lakehouse-id) + LAKEHOUSE_ID="$2" + shift 2 + ;; + --skip-data-sources) + SKIP_DATA_SOURCES="true" + shift + ;; + --skip-semantic-model) + SKIP_SEMANTIC_MODEL="true" + shift + ;; + --skip-ontology) + SKIP_ONTOLOGY="true" + shift + ;; + --dry-run) + DRY_RUN="true" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + err "Unknown option: $1" + ;; + esac +done + +#### +# Validation +#### + +log "Validating Prerequisites" + +if [[ -z "$DEFINITION_FILE" ]]; then + err "--definition is required" +fi + +if [[ ! -f "$DEFINITION_FILE" ]]; then + err "Definition file not found: $DEFINITION_FILE" +fi + +if [[ -z "$WORKSPACE_ID" ]]; then + err "--workspace-id is required" +fi + +if [[ "$SKIP_DATA_SOURCES" == "true" && -z "$LAKEHOUSE_ID" ]]; then + err "--lakehouse-id is required when using --skip-data-sources" +fi + +if [[ -n "$DATA_DIR" && ! -d "$DATA_DIR" ]]; then + err "Data directory not found: $DATA_DIR" +fi + +# Check required tools +for tool in az curl jq yq; do + if ! command -v "$tool" &>/dev/null; then + err "Required tool not found: $tool" + fi +done + +# Check Azure CLI authentication +if ! az account show &>/dev/null; then + err "Azure CLI not authenticated. Run 'az login' first." +fi + +ok "Prerequisites validated" + +#### +# Validate Definition +#### + +log "Validating Definition" +info "Definition: $DEFINITION_FILE" + +if ! "$SCRIPT_DIR/validate-definition.sh" --definition "$DEFINITION_FILE"; then + err "Definition validation failed" +fi + +ok "Definition validation passed" + +#### +# Extract Metadata +#### + +log "Extracting Metadata" + +ONTOLOGY_NAME=$(get_metadata_name "$DEFINITION_FILE") +ONTOLOGY_DESC=$(get_metadata_description "$DEFINITION_FILE") +LAKEHOUSE_NAME=$(get_lakehouse_name "$DEFINITION_FILE") + +info "Ontology: $ONTOLOGY_NAME" +info "Description: $ONTOLOGY_DESC" +info "Lakehouse: $LAKEHOUSE_NAME" + +#### +# Display Configuration +#### + +log "Deployment Configuration" + +info "Workspace ID: $WORKSPACE_ID" +info "Definition: $DEFINITION_FILE" +if [[ -n "$DATA_DIR" ]]; then + info "Data Directory: $DATA_DIR" +fi +info "Deploy Data Sources: $(if [[ "$SKIP_DATA_SOURCES" == "true" ]]; then echo "No (skipped)"; else echo "Yes"; fi)" +info "Deploy Semantic Model: $(if [[ "$SKIP_SEMANTIC_MODEL" == "true" ]]; then echo "No (skipped)"; else echo "Yes"; fi)" +info "Deploy Ontology: $(if [[ "$SKIP_ONTOLOGY" == "true" ]]; then echo "No (skipped)"; else echo "Yes"; fi)" + +if [[ -n "$LAKEHOUSE_ID" ]]; then + info "Lakehouse ID: $LAKEHOUSE_ID (provided)" +fi + +if [[ "$DRY_RUN" == "true" ]]; then + warn "DRY RUN MODE - No changes will be made" +fi + +#### +# Authenticate +#### + +log "Authenticating to Fabric API" +FABRIC_TOKEN=$(get_fabric_token) +STORAGE_TOKEN=$(get_storage_token) +ok "Authentication successful" + +#### +# Verify Workspace +#### + +log "Verifying Workspace Access" +workspace_response=$(get_workspace "$WORKSPACE_ID" "$FABRIC_TOKEN") +workspace_name=$(echo "$workspace_response" | jq -r '.displayName // "Unknown"') +info "Workspace: $workspace_name ($WORKSPACE_ID)" + +#### +# Step 1: Deploy Data Sources +#### + +if [[ "$SKIP_DATA_SOURCES" != "true" ]]; then + log "Step 1: Deploying Data Sources" + + if [[ "$DRY_RUN" == "true" ]]; then + info "[DRY-RUN] Would create Lakehouse: $LAKEHOUSE_NAME" + + # Show what tables would be created + tables=$(get_lakehouse_tables "$DEFINITION_FILE") + table_count=$(echo "$tables" | jq 'length') + + for i in $(seq 0 $((table_count - 1))); do + table_name=$(echo "$tables" | jq -r ".[$i].name") + + # Check for local data file + if [[ -n "$DATA_DIR" ]]; then + for ext in csv parquet; do + if [[ -f "$DATA_DIR/${table_name}.${ext}" ]]; then + info "[DRY-RUN] Would upload: ${table_name}.${ext} -> table '$table_name'" + break + fi + done + else + source_url=$(echo "$tables" | jq -r ".[$i].sourceUrl // empty") + source_file=$(echo "$tables" | jq -r ".[$i].sourceFile // empty") + if [[ -n "$source_url" ]]; then + info "[DRY-RUN] Would download: $source_url -> table '$table_name'" + elif [[ -n "$source_file" ]]; then + info "[DRY-RUN] Would upload: $source_file -> table '$table_name'" + fi + fi + done + + # Set placeholder ID for dry-run mode + LAKEHOUSE_ID="dry-run-lakehouse-id" + else + # Create or get Lakehouse + info "Creating Lakehouse: $LAKEHOUSE_NAME" + lakehouse_response=$(get_or_create_lakehouse "$WORKSPACE_ID" "$LAKEHOUSE_NAME" "$FABRIC_TOKEN") + LAKEHOUSE_ID=$(echo "$lakehouse_response" | jq -r '.id') + + if [[ -z "$LAKEHOUSE_ID" || "$LAKEHOUSE_ID" == "null" ]]; then + err "Failed to get Lakehouse ID" + fi + + ok "Lakehouse ID: $LAKEHOUSE_ID" + + # Process tables + tables=$(get_lakehouse_tables "$DEFINITION_FILE") + table_count=$(echo "$tables" | jq 'length') + + info "Processing $table_count tables" + + for i in $(seq 0 $((table_count - 1))); do + table_name=$(echo "$tables" | jq -r ".[$i].name") + format=$(echo "$tables" | jq -r ".[$i].format // \"csv\"") + source_url=$(echo "$tables" | jq -r ".[$i].sourceUrl // empty") + source_file=$(echo "$tables" | jq -r ".[$i].sourceFile // empty") + + info "Table: $table_name" + + local_file="" + + # Priority 1: Local data directory + if [[ -n "$DATA_DIR" ]]; then + for ext in csv parquet; do + if [[ -f "$DATA_DIR/${table_name}.${ext}" ]]; then + local_file="$DATA_DIR/${table_name}.${ext}" + format="$ext" + info "Found local file: ${table_name}.${ext}" + break + fi + done + fi + + # Priority 2: sourceUrl from YAML + if [[ -z "$local_file" && -n "$source_url" ]]; then + info "Downloading from: $source_url" + local_file=$(mktemp "/tmp/${table_name}.XXXXXX.${format}") + if ! curl -sSL "$source_url" -o "$local_file"; then + err "Failed to download: $source_url" + fi + fi + + # Priority 3: sourceFile from YAML + if [[ -z "$local_file" && -n "$source_file" ]]; then + # Resolve relative paths from definition file location + if [[ ! "$source_file" = /* ]]; then + source_file="$(dirname "$DEFINITION_FILE")/$source_file" + fi + if [[ -f "$source_file" ]]; then + local_file="$source_file" + info "Using source file: $source_file" + fi + fi + + if [[ -z "$local_file" || ! -f "$local_file" ]]; then + warn "No data source found for table '$table_name', skipping" + continue + fi + + # Upload to OneLake Files + info "Uploading to OneLake: raw/${table_name}.${format}" + upload_to_onelake "$WORKSPACE_ID" "$LAKEHOUSE_ID" "raw/${table_name}.${format}" "$local_file" "$STORAGE_TOKEN" + + # Load as Delta table + info "Loading Delta table: $table_name" + load_lakehouse_table "$WORKSPACE_ID" "$LAKEHOUSE_ID" "$table_name" "raw/${table_name}.${format}" "$format" "$FABRIC_TOKEN" + + ok "Table '$table_name' loaded" + + # Cleanup temp files from URL downloads + if [[ -n "$source_url" && "$local_file" == /tmp/* ]]; then + rm -f "$local_file" + fi + done + + # Handle Eventhouse if defined + eventhouse_name=$(get_eventhouse_name "$DEFINITION_FILE") + if [[ -n "$eventhouse_name" && "$eventhouse_name" != "null" ]]; then + info "Eventhouse deployment delegated to deploy-data-sources.sh" + "$SCRIPT_DIR/deploy-data-sources.sh" \ + --definition "$DEFINITION_FILE" \ + --workspace-id "$WORKSPACE_ID" \ + --skip-lakehouse + + # Capture Eventhouse IDs from environment + EVENTHOUSE_ID="${EVENTHOUSE_ID:-}" + KQL_DATABASE_ID="${KQL_DATABASE_ID:-}" + CLUSTER_URI="${EVENTHOUSE_QUERY_URI:-}" + fi + + ok "Data sources deployed" + fi +else + log "Step 1: Skipping Data Sources" + info "Using existing Lakehouse: $LAKEHOUSE_ID" +fi + +#### +# Step 2: Deploy Semantic Model +#### + +if [[ "$SKIP_SEMANTIC_MODEL" != "true" ]]; then + log "Step 2: Deploying Semantic Model" + + if [[ -z "$LAKEHOUSE_ID" ]]; then + err "Lakehouse ID is required for semantic model deployment" + fi + + deploy_args=( + "--definition" "$DEFINITION_FILE" + "--workspace-id" "$WORKSPACE_ID" + "--lakehouse-id" "$LAKEHOUSE_ID" + ) + + if [[ "$DRY_RUN" == "true" ]]; then + deploy_args+=("--dry-run") + fi + + "$SCRIPT_DIR/deploy-semantic-model.sh" "${deploy_args[@]}" + ok "Semantic model deployed" +else + log "Step 2: Skipping Semantic Model" +fi + +#### +# Step 3: Deploy Ontology +#### + +if [[ "$SKIP_ONTOLOGY" != "true" ]]; then + log "Step 3: Deploying Ontology" + + if [[ -z "$LAKEHOUSE_ID" ]]; then + err "Lakehouse ID is required for ontology deployment" + fi + + deploy_args=( + "--definition" "$DEFINITION_FILE" + "--workspace-id" "$WORKSPACE_ID" + "--lakehouse-id" "$LAKEHOUSE_ID" + ) + + if [[ -n "$EVENTHOUSE_ID" ]]; then + deploy_args+=("--eventhouse-id" "$EVENTHOUSE_ID") + fi + if [[ -n "$CLUSTER_URI" ]]; then + deploy_args+=("--cluster-uri" "$CLUSTER_URI") + fi + if [[ -n "$KQL_DATABASE_ID" ]]; then + deploy_args+=("--kql-database-id" "$KQL_DATABASE_ID") + fi + if [[ "$DRY_RUN" == "true" ]]; then + deploy_args+=("--dry-run") + fi + + "$SCRIPT_DIR/deploy-ontology.sh" "${deploy_args[@]}" + ok "Ontology deployed" + warn "Ontology setup is async - entity types take 10-20 minutes to fully provision" + info "The portal will show 'Setting up your ontology' until complete" +else + log "Step 3: Skipping Ontology" +fi + +#### +# Summary +#### + +log "Deployment Complete" + +if [[ "$DRY_RUN" == "true" ]]; then + warn "DRY RUN - No changes were made" + info "Remove --dry-run to perform actual deployment" + exit 0 +fi + +cat << EOF + +=== Deployment Summary === + +Ontology: $ONTOLOGY_NAME +Workspace: $workspace_name ($WORKSPACE_ID) + +Resources Created: + Lakehouse: $LAKEHOUSE_NAME ($LAKEHOUSE_ID) +EOF + +if [[ -n "$EVENTHOUSE_ID" ]]; then + echo " Eventhouse: $EVENTHOUSE_ID" +fi + +cat << EOF + +=== Next Steps === + +1. Open your workspace: + https://app.fabric.microsoft.com/groups/$WORKSPACE_ID + +2. Verify the Lakehouse tables were created correctly + +3. Open the Semantic Model and verify Direct Lake connection + +4. Create a Data Agent (manual step - no API available): + - Click '+ New item' → Search for 'Data agent' + - Add the ontology as a data source + - Add instruction: "Support group by in GQL" + +EOF + +info "Deployment complete" diff --git a/src/000-cloud/033-fabric-ontology/scripts/lib/definition-parser.sh b/src/000-cloud/033-fabric-ontology/scripts/lib/definition-parser.sh new file mode 100755 index 00000000..af6fdb73 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/scripts/lib/definition-parser.sh @@ -0,0 +1,332 @@ +#!/usr/bin/env bash +# Definition Parser Library - Utilities for parsing ontology definition YAML +# +# Dependencies: yq (https://github.com/mikefarah/yq) +# +# Usage: +# source ./lib/definition-parser.sh +# name=$(get_metadata_name "ontology.yaml") +# +# shellcheck disable=SC2034 + +set -e +set -o pipefail + +# Verify yq is available +command -v yq >/dev/null 2>&1 || { + echo "[ ERROR ]: yq is required but not installed. Install from https://github.com/mikefarah/yq" >&2 + exit 1 +} + +# Get metadata.name from definition +get_metadata_name() { + local definition_file="$1" + yq -r '.metadata.name' "$definition_file" +} + +# Get metadata.description from definition +get_metadata_description() { + local definition_file="$1" + yq -r '.metadata.description // ""' "$definition_file" +} + +# Get metadata.version from definition +get_metadata_version() { + local definition_file="$1" + yq -r '.metadata.version // "1.0.0"' "$definition_file" +} + +# Get entityTypes as JSON array +get_entity_types() { + local definition_file="$1" + yq -o=json '.entityTypes // []' "$definition_file" +} + +# Get list of entity type names (one per line) +get_entity_type_names() { + local definition_file="$1" + yq -r '.entityTypes[].name' "$definition_file" +} + +# Get entity type count +get_entity_type_count() { + local definition_file="$1" + yq '.entityTypes | length' "$definition_file" +} + +# Get specific entity type by name as JSON +get_entity_type() { + local definition_file="$1" + local entity_name="$2" + yq -o=json ".entityTypes[] | select(.name == \"$entity_name\")" "$definition_file" +} + +# Get entity type key property name +get_entity_key() { + local definition_file="$1" + local entity_name="$2" + yq -r ".entityTypes[] | select(.name == \"$entity_name\") | .key" "$definition_file" +} + +# Get entity type display name property +get_entity_display_name() { + local definition_file="$1" + local entity_name="$2" + local display_name + display_name=$(yq -r ".entityTypes[] | select(.name == \"$entity_name\") | .displayName // \"\"" "$definition_file") + if [[ -z "$display_name" ]]; then + get_entity_key "$definition_file" "$entity_name" + else + echo "$display_name" + fi +} + +# Get properties for specific entity as JSON array +get_entity_properties() { + local definition_file="$1" + local entity_name="$2" + yq -o=json ".entityTypes[] | select(.name == \"$entity_name\") | .properties // []" "$definition_file" +} + +# Get entity property names (one per line) +get_entity_property_names() { + local definition_file="$1" + local entity_name="$2" + yq -r ".entityTypes[] | select(.name == \"$entity_name\") | .properties[].name" "$definition_file" +} + +# Get static properties for an entity (binding == "static" or binding is null) +get_entity_static_properties() { + local definition_file="$1" + local entity_name="$2" + yq -o=json ".entityTypes[] | select(.name == \"$entity_name\") | .properties | map(select(.binding == \"static\" or .binding == null))" "$definition_file" +} + +# Get timeseries properties for an entity +get_entity_timeseries_properties() { + local definition_file="$1" + local entity_name="$2" + yq -o=json ".entityTypes[] | select(.name == \"$entity_name\") | .properties | map(select(.binding == \"timeseries\"))" "$definition_file" +} + +# Get entity data binding (single binding) +get_entity_data_binding() { + local definition_file="$1" + local entity_name="$2" + yq -o=json ".entityTypes[] | select(.name == \"$entity_name\") | .dataBinding // null" "$definition_file" +} + +# Get entity data bindings (multiple bindings) +get_entity_data_bindings() { + local definition_file="$1" + local entity_name="$2" + yq -o=json ".entityTypes[] | select(.name == \"$entity_name\") | .dataBindings // []" "$definition_file" +} + +# Get static data binding for entity (searches both dataBinding and dataBindings) +get_entity_static_binding() { + local definition_file="$1" + local entity_name="$2" + local binding + + binding=$(yq -o=json ".entityTypes[] | select(.name == \"$entity_name\") | .dataBinding | select(.type == \"static\")" "$definition_file" 2>/dev/null) + if [[ -n "$binding" && "$binding" != "null" ]]; then + echo "$binding" + return + fi + + yq -o=json ".entityTypes[] | select(.name == \"$entity_name\") | .dataBindings[] | select(.type == \"static\")" "$definition_file" 2>/dev/null || echo "null" +} + +# Get timeseries data binding for entity +get_entity_timeseries_binding() { + local definition_file="$1" + local entity_name="$2" + yq -o=json ".entityTypes[] | select(.name == \"$entity_name\") | .dataBindings[] | select(.type == \"timeseries\")" "$definition_file" 2>/dev/null || echo "null" +} + +# Get lakehouse data source configuration +get_lakehouse_config() { + local definition_file="$1" + yq -o=json '.dataSources.lakehouse // null' "$definition_file" +} + +# Get lakehouse name +get_lakehouse_name() { + local definition_file="$1" + yq -r '.dataSources.lakehouse.name // ""' "$definition_file" +} + +# Get lakehouse tables as JSON array +get_lakehouse_tables() { + local definition_file="$1" + yq -o=json '.dataSources.lakehouse.tables // []' "$definition_file" +} + +# Get lakehouse table names (one per line) +get_lakehouse_table_names() { + local definition_file="$1" + yq -r '.dataSources.lakehouse.tables[].name // empty' "$definition_file" +} + +# Get specific lakehouse table by name +get_lakehouse_table() { + local definition_file="$1" + local table_name="$2" + yq -o=json ".dataSources.lakehouse.tables[] | select(.name == \"$table_name\")" "$definition_file" +} + +# Get eventhouse data source configuration +get_eventhouse_config() { + local definition_file="$1" + yq -o=json '.dataSources.eventhouse // null' "$definition_file" +} + +# Get eventhouse name +get_eventhouse_name() { + local definition_file="$1" + yq -r '.dataSources.eventhouse.name // ""' "$definition_file" +} + +# Get eventhouse database name +get_eventhouse_database() { + local definition_file="$1" + yq -r '.dataSources.eventhouse.database // ""' "$definition_file" +} + +# Get eventhouse tables as JSON array +get_eventhouse_tables() { + local definition_file="$1" + yq -o=json '.dataSources.eventhouse.tables // []' "$definition_file" +} + +# Get eventhouse table names (one per line) +get_eventhouse_table_names() { + local definition_file="$1" + yq -r '.dataSources.eventhouse.tables[].name // empty' "$definition_file" +} + +# Get specific eventhouse table by name +get_eventhouse_table() { + local definition_file="$1" + local table_name="$2" + yq -o=json ".dataSources.eventhouse.tables[] | select(.name == \"$table_name\")" "$definition_file" +} + +# Get relationships as JSON array +get_relationships() { + local definition_file="$1" + yq -o=json '.relationships // []' "$definition_file" +} + +# Get relationship names (one per line) +get_relationship_names() { + local definition_file="$1" + yq -r '.relationships[].name // empty' "$definition_file" +} + +# Get relationship count +get_relationship_count() { + local definition_file="$1" + yq '.relationships | length // 0' "$definition_file" +} + +# Get specific relationship by name +get_relationship() { + local definition_file="$1" + local rel_name="$2" + yq -o=json ".relationships[] | select(.name == \"$rel_name\")" "$definition_file" +} + +# Get semantic model configuration +get_semantic_model_config() { + local definition_file="$1" + yq -o=json '.semanticModel // null' "$definition_file" +} + +# Get semantic model name +get_semantic_model_name() { + local definition_file="$1" + yq -r '.semanticModel.name // ""' "$definition_file" +} + +# Get semantic model mode (directLake or import) +get_semantic_model_mode() { + local definition_file="$1" + yq -r '.semanticModel.mode // "directLake"' "$definition_file" +} + +# Map definition property type to Fabric ontology type +map_property_type() { + local def_type="$1" + case "$def_type" in + "string") echo "String" ;; + "int") echo "BigInt" ;; + "double") echo "Double" ;; + "datetime") echo "DateTime" ;; + "boolean") echo "Boolean" ;; + "object") echo "Object" ;; + *) echo "String" ;; + esac +} + +# Map definition property type to KQL type +map_kql_type() { + local def_type="$1" + case "$def_type" in + "string") echo "string" ;; + "int") echo "int" ;; + "double") echo "real" ;; + "datetime") echo "datetime" ;; + "boolean") echo "bool" ;; + "object") echo "dynamic" ;; + *) echo "string" ;; + esac +} + +# Map definition property type to TMDL type +map_tmdl_type() { + local def_type="$1" + case "$def_type" in + "string") echo "string" ;; + "int") echo "int64" ;; + "double") echo "double" ;; + "datetime") echo "dateTime" ;; + "boolean") echo "boolean" ;; + "object") echo "string" ;; + *) echo "string" ;; + esac +} + +# Check if definition has lakehouse data source +has_lakehouse() { + local definition_file="$1" + local name + name=$(get_lakehouse_name "$definition_file") + [[ -n "$name" ]] +} + +# Check if definition has eventhouse data source +has_eventhouse() { + local definition_file="$1" + local name + name=$(get_eventhouse_name "$definition_file") + [[ -n "$name" ]] +} + +# Check if definition has semantic model configuration +has_semantic_model() { + local definition_file="$1" + local name + name=$(get_semantic_model_name "$definition_file") + [[ -n "$name" ]] +} + +# Check if entity has timeseries binding +entity_has_timeseries() { + local definition_file="$1" + local entity_name="$2" + local binding + binding=$(get_entity_timeseries_binding "$definition_file" "$entity_name") + [[ -n "$binding" && "$binding" != "null" ]] +} diff --git a/src/000-cloud/033-fabric-ontology/scripts/lib/fabric-api.sh b/src/000-cloud/033-fabric-ontology/scripts/lib/fabric-api.sh new file mode 100755 index 00000000..a677d5d8 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/scripts/lib/fabric-api.sh @@ -0,0 +1,744 @@ +#!/usr/bin/env bash +# Fabric API Library - Common functions for Microsoft Fabric REST API operations +# +# Dependencies: curl, jq, az (Azure CLI) +# +# Usage: +# source ./lib/fabric-api.sh +# token=$(get_fabric_token) +# fabric_api_call "GET" "/workspaces" "" "$token" +# +# Environment Variables (optional): +# FABRIC_API_BASE_URL - Override default API base URL + +set -e +set -o pipefail + +# API Configuration +readonly FABRIC_API_BASE_URL="${FABRIC_API_BASE_URL:-https://api.fabric.microsoft.com/v1}" +readonly FABRIC_RESOURCE="https://api.fabric.microsoft.com" +readonly STORAGE_RESOURCE="https://storage.azure.com" +readonly ONELAKE_DFS_URL="https://onelake.dfs.fabric.microsoft.com" +readonly KUSTO_RESOURCE="https://kusto.kusto.windows.net" + +# Verify required tools +for cmd in curl jq az; do + command -v "$cmd" >/dev/null 2>&1 || { + echo "[ ERROR ]: $cmd is required but not installed." >&2 + exit 1 + } +done + +# Get Azure AD token for Fabric REST API +get_fabric_token() { + az account get-access-token \ + --resource "$FABRIC_RESOURCE" \ + --query accessToken \ + --output tsv +} + +# Get Azure AD token for OneLake/Storage operations +get_storage_token() { + az account get-access-token \ + --resource "$STORAGE_RESOURCE" \ + --query accessToken \ + --output tsv +} + +# Get Azure AD token for Kusto/KQL operations +get_kusto_token() { + az account get-access-token \ + --resource "$KUSTO_RESOURCE" \ + --query accessToken \ + --output tsv +} + +# Generic Fabric API call with error handling (file-based for large payloads) +# Arguments: +# $1 - HTTP method (GET, POST, PUT, PATCH, DELETE) +# $2 - API endpoint (relative to base URL, e.g., "/workspaces") +# $3 - Path to file containing request body JSON, or empty +# $4 - Bearer token (optional, will fetch if not provided) +# Returns: Response body on success, exits on error +fabric_api_call_file() { + local method="$1" + local endpoint="$2" + local body_file="${3:-}" + local token="${4:-}" + + if [[ -z "$token" ]]; then + token=$(get_fabric_token) + fi + + local url="${FABRIC_API_BASE_URL}${endpoint}" + local headers_file response http_code response_body + + headers_file=$(mktemp) + + if [[ -n "$body_file" && -f "$body_file" ]]; then + response=$(curl -s -w "\n%{http_code}" -D "$headers_file" -X "$method" "$url" \ + -H "Authorization: Bearer $token" \ + -H "Content-Type: application/json" \ + -d @"$body_file") + else + response=$(curl -s -w "\n%{http_code}" -D "$headers_file" -X "$method" "$url" \ + -H "Authorization: Bearer $token" \ + -H "Content-Type: application/json") + fi + + http_code=$(echo "$response" | tail -c 4) + response_body=$(echo "$response" | sed '$d') + + # Handle different response codes + case "$http_code" in + 200|201) + rm -f "$headers_file" + echo "$response_body" + return 0 + ;; + 204) + rm -f "$headers_file" + echo "{}" + return 0 + ;; + 202) + # Long-running operation - check for Location header and poll + local location operation_id + location=$(grep -i "^Location:" "$headers_file" | sed 's/^[Ll]ocation: *//' | tr -d '\r') + operation_id=$(grep -i "^x-ms-operation-id:" "$headers_file" | sed 's/^x-ms-operation-id: *//' | tr -d '\r') + rm -f "$headers_file" + + if [[ -n "$location" ]]; then + echo "[ INFO ]: Long-running operation, polling for completion..." >&2 + poll_operation "$location" "$token" 300 + return $? + elif [[ -n "$operation_id" ]]; then + echo "[ INFO ]: Long-running operation ID: $operation_id, polling..." >&2 + poll_operation "${FABRIC_API_BASE_URL}/operations/${operation_id}" "$token" 300 + return $? + else + # No location header, return body if any + echo "$response_body" + return 0 + fi + ;; + *) + rm -f "$headers_file" + echo "[ ERROR ]: API call failed with HTTP $http_code" >&2 + echo "[ ERROR ]: Endpoint: $method $url" >&2 + echo "[ ERROR ]: Response: $response_body" >&2 + return 1 + ;; + esac +} + +# Generic Fabric API call with error handling +# Arguments: +# $1 - HTTP method (GET, POST, PUT, PATCH, DELETE) +# $2 - API endpoint (relative to base URL, e.g., "/workspaces") +# $3 - Request body (JSON string or empty) +# $4 - Bearer token (optional, will fetch if not provided) +# Returns: Response body on success, exits on error +fabric_api_call() { + local method="$1" + local endpoint="$2" + local body="${3:-}" + local token="${4:-}" + + if [[ -z "$token" ]]; then + token=$(get_fabric_token) + fi + + local url="${FABRIC_API_BASE_URL}${endpoint}" + local headers_file response http_code response_body + + headers_file=$(mktemp) + + if [[ -n "$body" ]]; then + # Use file-based approach to avoid shell argument length limits + local body_file + body_file=$(mktemp) + echo "$body" > "$body_file" + response=$(curl -s -w "\n%{http_code}" -D "$headers_file" -X "$method" "$url" \ + -H "Authorization: Bearer $token" \ + -H "Content-Type: application/json" \ + -d @"$body_file") + rm -f "$body_file" + else + response=$(curl -s -w "\n%{http_code}" -D "$headers_file" -X "$method" "$url" \ + -H "Authorization: Bearer $token" \ + -H "Content-Type: application/json") + fi + + http_code=$(echo "$response" | tail -c 4) + response_body=$(echo "$response" | sed '$d') + + # Handle different response codes + case "$http_code" in + 200|201) + rm -f "$headers_file" + echo "$response_body" + return 0 + ;; + 204) + rm -f "$headers_file" + echo "{}" + return 0 + ;; + 202) + # Long-running operation - check for Location header and poll + local location operation_id + location=$(grep -i "^Location:" "$headers_file" | sed 's/^[Ll]ocation: *//' | tr -d '\r') + operation_id=$(grep -i "^x-ms-operation-id:" "$headers_file" | sed 's/^x-ms-operation-id: *//' | tr -d '\r') + rm -f "$headers_file" + + if [[ -n "$location" ]]; then + echo "[ INFO ]: Long-running operation, polling for completion..." >&2 + poll_operation "$location" "$token" 300 + return $? + elif [[ -n "$operation_id" ]]; then + echo "[ INFO ]: Long-running operation ID: $operation_id, polling..." >&2 + poll_operation "${FABRIC_API_BASE_URL}/operations/${operation_id}" "$token" 300 + return $? + else + # No location header, return body if any + echo "$response_body" + return 0 + fi + ;; + *) + rm -f "$headers_file" + echo "[ ERROR ]: API call failed with HTTP $http_code" >&2 + echo "[ ERROR ]: Endpoint: $method $url" >&2 + echo "[ ERROR ]: Response: $response_body" >&2 + return 1 + ;; + esac +} + +# Poll long-running operation until completion +# Arguments: +# $1 - Operation URL (from Location header or x-ms-operation-id) +# $2 - Bearer token (optional) +# $3 - Max wait time in seconds (default: 300) +# Returns: Final operation result JSON (includes createdItem for create operations) +poll_operation() { + local operation_url="$1" + local token="${2:-}" + local max_wait="${3:-300}" + + if [[ -z "$token" ]]; then + token=$(get_fabric_token) + fi + + local elapsed=0 + local sleep_interval=5 + + while [[ $elapsed -lt $max_wait ]]; do + local response + response=$(curl -s -X GET "$operation_url" \ + -H "Authorization: Bearer $token") + + local status + status=$(echo "$response" | jq -r '.status // .Status // "Unknown"') + + case "$status" in + "Succeeded"|"succeeded") + # Fetch the result endpoint to get the created item + local result_url="${operation_url}/result" + local result_response + result_response=$(curl -s -X GET "$result_url" \ + -H "Authorization: Bearer $token") + + # Return result if valid, otherwise check for createdItem in status response + if [[ -n "$result_response" && "$result_response" != "null" ]]; then + local result_id + result_id=$(echo "$result_response" | jq -r '.id // empty') + if [[ -n "$result_id" ]]; then + echo "$result_response" + return 0 + fi + fi + + # Fallback: check createdItem in status response + local created_item + created_item=$(echo "$response" | jq -r '.createdItem // empty') + if [[ -n "$created_item" && "$created_item" != "null" ]]; then + echo "$created_item" + else + echo "$response" + fi + return 0 + ;; + "Failed"|"failed") + echo "[ ERROR ]: Operation failed" >&2 + echo "$response" >&2 + return 1 + ;; + "Running"|"running"|"InProgress"|"inProgress"|"NotStarted"|"notStarted") + echo "[ INFO ]: Operation status: $status (${elapsed}s/${max_wait}s)" >&2 + sleep "$sleep_interval" + ((elapsed += sleep_interval)) + ;; + *) + echo "[ WARN ]: Unknown operation status: $status" >&2 + sleep "$sleep_interval" + ((elapsed += sleep_interval)) + ;; + esac + done + + echo "[ ERROR ]: Operation timed out after ${max_wait}s" >&2 + return 1 +} + +# Get workspace by ID +get_workspace() { + local workspace_id="$1" + local token="${2:-}" + fabric_api_call "GET" "/workspaces/$workspace_id" "" "$token" +} + +# List items in workspace by type +list_workspace_items() { + local workspace_id="$1" + local item_type="$2" + local token="${3:-}" + fabric_api_call "GET" "/workspaces/$workspace_id/${item_type}s" "" "$token" +} + +# Get or create Lakehouse (idempotent) +# Arguments: +# $1 - Workspace ID +# $2 - Lakehouse display name +# $3 - Bearer token (optional) +# Returns: Lakehouse JSON (id, displayName) +get_or_create_lakehouse() { + local workspace_id="$1" + local lakehouse_name="$2" + local token="${3:-}" + + if [[ -z "$token" ]]; then + token=$(get_fabric_token) + fi + + # Check if lakehouse exists + local existing + existing=$(fabric_api_call "GET" "/workspaces/$workspace_id/lakehouses" "" "$token") + + local lakehouse_id + lakehouse_id=$(echo "$existing" | jq -r ".value[] | select(.displayName == \"$lakehouse_name\") | .id") + + if [[ -n "$lakehouse_id" ]]; then + echo "[ INFO ]: Lakehouse '$lakehouse_name' already exists: $lakehouse_id" >&2 + echo "$existing" | jq ".value[] | select(.id == \"$lakehouse_id\")" + return 0 + fi + + # Create new lakehouse + echo "[ INFO ]: Creating Lakehouse '$lakehouse_name'..." >&2 + local body + body=$(jq -n --arg name "$lakehouse_name" '{"displayName": $name}') + + local response + response=$(fabric_api_call "POST" "/workspaces/$workspace_id/lakehouses" "$body" "$token") + echo "$response" +} + +# Get or create Eventhouse (idempotent) +get_or_create_eventhouse() { + local workspace_id="$1" + local eventhouse_name="$2" + local token="${3:-}" + + if [[ -z "$token" ]]; then + token=$(get_fabric_token) + fi + + # Check if eventhouse exists + local existing + existing=$(fabric_api_call "GET" "/workspaces/$workspace_id/eventhouses" "" "$token") + + local eventhouse_id + eventhouse_id=$(echo "$existing" | jq -r ".value[] | select(.displayName == \"$eventhouse_name\") | .id") + + if [[ -n "$eventhouse_id" ]]; then + echo "[ INFO ]: Eventhouse '$eventhouse_name' already exists: $eventhouse_id" >&2 + echo "$existing" | jq ".value[] | select(.id == \"$eventhouse_id\")" + return 0 + fi + + # Create new eventhouse + echo "[ INFO ]: Creating Eventhouse '$eventhouse_name'..." >&2 + local body + body=$(jq -n --arg name "$eventhouse_name" '{"displayName": $name}') + + local response + response=$(fabric_api_call "POST" "/workspaces/$workspace_id/eventhouses" "$body" "$token") + echo "$response" +} + +# Get or create KQL database (idempotent) +get_or_create_kql_database() { + local workspace_id="$1" + local database_name="$2" + local eventhouse_id="$3" + local token="${4:-}" + + if [[ -z "$token" ]]; then + token=$(get_fabric_token) + fi + + # Check if database exists + local existing + existing=$(fabric_api_call "GET" "/workspaces/$workspace_id/kqlDatabases" "" "$token") + + local database_id + database_id=$(echo "$existing" | jq -r ".value[] | select(.displayName == \"$database_name\") | .id") + + if [[ -n "$database_id" ]]; then + echo "[ INFO ]: KQL Database '$database_name' already exists: $database_id" >&2 + echo "$existing" | jq ".value[] | select(.id == \"$database_id\")" + return 0 + fi + + # Create new KQL database + echo "[ INFO ]: Creating KQL Database '$database_name'..." >&2 + local body + body=$(jq -n \ + --arg name "$database_name" \ + --arg ehId "$eventhouse_id" \ + '{"displayName": $name, "creationPayload": {"databaseType": "ReadWrite", "parentEventhouseItemId": $ehId}}') + + local response + response=$(fabric_api_call "POST" "/workspaces/$workspace_id/kqlDatabases" "$body" "$token") + + # KQL database creation is a long-running operation - wait for it + echo "[ INFO ]: Waiting for KQL Database creation..." >&2 + sleep 10 + + # Re-fetch the database list to get the ID + existing=$(fabric_api_call "GET" "/workspaces/$workspace_id/kqlDatabases" "" "$token") + database_id=$(echo "$existing" | jq -r ".value[] | select(.displayName == \"$database_name\") | .id") + + if [[ -n "$database_id" ]]; then + echo "$existing" | jq ".value[] | select(.id == \"$database_id\")" + return 0 + fi + + echo "$response" +} + +# Get or create Semantic Model (idempotent) +get_or_create_semantic_model() { + local workspace_id="$1" + local model_name="$2" + local definition_parts="$3" + local token="${4:-}" + + if [[ -z "$token" ]]; then + token=$(get_fabric_token) + fi + + # Check if semantic model exists + local existing + existing=$(fabric_api_call "GET" "/workspaces/$workspace_id/semanticModels" "" "$token") + + local model_id + model_id=$(echo "$existing" | jq -r ".value[] | select(.displayName == \"$model_name\") | .id") + + if [[ -n "$model_id" ]]; then + echo "[ INFO ]: Semantic Model '$model_name' already exists: $model_id" >&2 + echo "$existing" | jq ".value[] | select(.id == \"$model_id\")" + return 0 + fi + + # Create new semantic model with definition + echo "[ INFO ]: Creating Semantic Model '$model_name'..." >&2 + local body + body=$(jq -n \ + --arg name "$model_name" \ + --argjson parts "$definition_parts" \ + '{"displayName": $name, "definition": {"parts": $parts}}') + + local response + response=$(fabric_api_call "POST" "/workspaces/$workspace_id/semanticModels" "$body" "$token") + echo "$response" +} + +# Get or create generic Fabric item (idempotent) +get_or_create_item() { + local workspace_id="$1" + local item_type="$2" + local item_name="$3" + local token="${4:-}" + + if [[ -z "$token" ]]; then + token=$(get_fabric_token) + fi + + # Check if item exists + local existing + existing=$(fabric_api_call "GET" "/workspaces/$workspace_id/items?type=$item_type" "" "$token") + + local item_id + item_id=$(echo "$existing" | jq -r ".value[] | select(.displayName == \"$item_name\") | .id") + + if [[ -n "$item_id" ]]; then + echo "[ INFO ]: $item_type '$item_name' already exists: $item_id" >&2 + echo "$existing" | jq ".value[] | select(.id == \"$item_id\")" + return 0 + fi + + # Create new item + echo "[ INFO ]: Creating $item_type '$item_name'..." >&2 + local body + body=$(jq -n \ + --arg name "$item_name" \ + --arg type "$item_type" \ + '{"displayName": $name, "type": $type}') + + local response + response=$(fabric_api_call "POST" "/workspaces/$workspace_id/items" "$body" "$token") + echo "$response" +} + +# Get or create Ontology item (idempotent) +get_or_create_ontology() { + local workspace_id="$1" + local ontology_name="$2" + local token="${3:-}" + get_or_create_item "$workspace_id" "Ontology" "$ontology_name" "$token" +} + +# Update item definition +update_item_definition() { + local workspace_id="$1" + local item_id="$2" + local definition_parts="$3" + local token="${4:-}" + + if [[ -z "$token" ]]; then + token=$(get_fabric_token) + fi + + local body + body=$(jq -n --argjson parts "$definition_parts" '{"definition": {"parts": $parts}}') + + fabric_api_call "POST" "/workspaces/$workspace_id/items/$item_id/updateDefinition" "$body" "$token" +} + +# Upload file to OneLake via DFS API +# Arguments: +# $1 - Workspace ID +# $2 - Lakehouse ID +# $3 - Remote file path (relative to Files/) +# $4 - Local file path +# $5 - Bearer token (optional) +upload_to_onelake() { + local workspace_id="$1" + local lakehouse_id="$2" + local remote_path="$3" + local local_file="$4" + local token="${5:-}" + + if [[ -z "$token" ]]; then + token=$(get_storage_token) + fi + + # When using GUIDs, no .lakehouse suffix needed + local base_url="${ONELAKE_DFS_URL}/${workspace_id}/${lakehouse_id}/Files" + + echo "[ INFO ]: Uploading to OneLake: $remote_path" >&2 + + # Create parent directory if path contains subdirectories + local dir_path + dir_path=$(dirname "$remote_path") + if [[ "$dir_path" != "." ]]; then + local dir_url="${base_url}/${dir_path}?resource=directory" + curl -s -X PUT "$dir_url" \ + -H "Authorization: Bearer $token" \ + -H "Content-Length: 0" >/dev/null 2>&1 || true + fi + + # Create file (requires Content-Length: 0) + local url="${base_url}/${remote_path}?resource=file" + local response http_code + response=$(curl -s -w "\n%{http_code}" -X PUT "$url" \ + -H "Authorization: Bearer $token" \ + -H "Content-Length: 0") + http_code=$(echo "$response" | tail -c 4) + + if [[ "$http_code" != "201" && "$http_code" != "200" ]]; then + echo "[ ERROR ]: Failed to create file: HTTP $http_code" >&2 + echo "[ ERROR ]: Response: $(echo "$response" | sed '$d')" >&2 + return 1 + fi + + # Upload content + local file_size + file_size=$(wc -c < "$local_file") + local append_url="${base_url}/${remote_path}?action=append&position=0" + + response=$(curl -s -w "\n%{http_code}" -X PATCH "$append_url" \ + -H "Authorization: Bearer $token" \ + -H "Content-Type: application/octet-stream" \ + --data-binary "@$local_file") + http_code=$(echo "$response" | tail -c 4) + + if [[ "$http_code" != "202" && "$http_code" != "200" ]]; then + echo "[ ERROR ]: Failed to upload content: HTTP $http_code" >&2 + return 1 + fi + + # Flush file + local flush_url="${base_url}/${remote_path}?action=flush&position=$file_size" + + response=$(curl -s -w "\n%{http_code}" -X PATCH "$flush_url" \ + -H "Authorization: Bearer $token" \ + -H "Content-Length: 0") + http_code=$(echo "$response" | tail -c 4) + + if [[ "$http_code" != "200" ]]; then + echo "[ ERROR ]: Failed to flush file: HTTP $http_code" >&2 + return 1 + fi + + echo "[ INFO ]: Upload complete: $remote_path ($file_size bytes)" >&2 + return 0 +} + +# Load table from file in Lakehouse (CSV → Delta conversion) +load_lakehouse_table() { + local workspace_id="$1" + local lakehouse_id="$2" + local table_name="$3" + local file_path="$4" + local file_format="${5:-Csv}" + local token="${6:-}" + + if [[ -z "$token" ]]; then + token=$(get_fabric_token) + fi + + echo "[ INFO ]: Loading table '$table_name' from $file_path..." >&2 + + # Capitalize format for API (Csv, Parquet) + local api_format + api_format=$(echo "$file_format" | sed 's/csv/Csv/i; s/parquet/Parquet/i') + + local body + body=$(jq -n \ + --arg path "Files/$file_path" \ + --arg format "$api_format" \ + '{ + "relativePath": $path, + "pathType": "File", + "mode": "Overwrite", + "formatOptions": { + "format": $format, + "header": true, + "delimiter": "," + } + }') + + local response + response=$(fabric_api_call "POST" "/workspaces/$workspace_id/lakehouses/$lakehouse_id/tables/$table_name/load" "$body" "$token") + + # Check if long-running operation + local operation_id + operation_id=$(echo "$response" | jq -r '.operationId // empty') + + if [[ -n "$operation_id" ]]; then + echo "[ INFO ]: Waiting for table load operation..." >&2 + local operation_url="${FABRIC_API_BASE_URL}/operations/$operation_id" + poll_operation "$operation_url" "$token" 300 + else + echo "$response" + fi +} + +# Execute KQL management command against database +# Arguments: +# $1 - Eventhouse query URI (e.g., https://.kusto.fabric.microsoft.com) +# $2 - Database name +# $3 - KQL command +# $4 - Bearer token (optional, will use Kusto token if not provided) +execute_kql() { + local query_uri="$1" + local database_name="$2" + local kql_command="$3" + local token="${4:-}" + + if [[ -z "$token" ]]; then + token=$(get_kusto_token) + fi + + local mgmt_url="${query_uri}/v1/rest/mgmt" + + local body + body=$(jq -n \ + --arg db "$database_name" \ + --arg csl "$kql_command" \ + '{"db": $db, "csl": $csl}') + + local response http_code + response=$(curl -s -w "\n%{http_code}" -X POST "$mgmt_url" \ + -H "Authorization: Bearer $token" \ + -H "Content-Type: application/json" \ + -d "$body") + + http_code=$(echo "$response" | tail -c 4) + local response_body + response_body=$(echo "$response" | sed '$d') + + if [[ "$http_code" != "200" ]]; then + echo "[ ERROR ]: KQL command failed with HTTP $http_code" >&2 + echo "[ ERROR ]: Command: $kql_command" >&2 + echo "[ ERROR ]: Response: $response_body" >&2 + return 1 + fi + + echo "$response_body" +} + +# Get Eventhouse query URI +get_eventhouse_query_uri() { + local workspace_id="$1" + local eventhouse_id="$2" + local token="${3:-}" + + if [[ -z "$token" ]]; then + token=$(get_fabric_token) + fi + + local response + response=$(fabric_api_call "GET" "/workspaces/$workspace_id/eventhouses/$eventhouse_id" "" "$token") + echo "$response" | jq -r '.properties.queryServiceUri // empty' +} + +# Generate a unique 64-bit ID (using timestamp and random) +generate_unique_id() { + local timestamp random_part + timestamp=$(date +%s%N | cut -c1-13) + random_part=$((RANDOM % 10000)) + echo "${timestamp}${random_part}" +} + +# Encode string to Base64 +encode_base64() { + local input="$1" + echo -n "$input" | base64 -w 0 +} + +# Build definition part JSON for API +# Arguments: +# $1 - Path (e.g., "definition.json", "EntityTypes/123/definition.json") +# $2 - Content (JSON string) +build_definition_part() { + local path="$1" + local content="$2" + local payload + payload=$(encode_base64 "$content") + jq -n --arg path "$path" --arg payload "$payload" '{"path": $path, "payload": $payload, "payloadType": "InlineBase64"}' +} diff --git a/src/000-cloud/033-fabric-ontology/scripts/lib/logging.sh b/src/000-cloud/033-fabric-ontology/scripts/lib/logging.sh new file mode 100755 index 00000000..8fd5ade3 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/scripts/lib/logging.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Logging Library - Consistent logging utilities for deployment scripts +# +# Usage: +# source ./lib/logging.sh +# log "Section Header" +# info "Informational message" +# warn "Warning message" +# err "Error message" # exits with code 1 + +# Colors (if terminal supports it) +if [[ -t 2 ]]; then + readonly RED='\033[0;31m' + readonly YELLOW='\033[0;33m' + readonly GREEN='\033[0;32m' + readonly BLUE='\033[0;34m' + readonly NC='\033[0m' # No Color +else + readonly RED='' + readonly YELLOW='' + readonly GREEN='' + readonly BLUE='' + readonly NC='' +fi + +# Log a section header +log() { + echo -e "${BLUE}========== $1 ==========${NC}" >&2 +} + +# Log informational message +info() { + echo -e "[ ${GREEN}INFO${NC} ]: $1" >&2 +} + +# Log warning message +warn() { + echo -e "[ ${YELLOW}WARN${NC} ]: $1" >&2 +} + +# Log error message and exit +err() { + echo -e "[ ${RED}ERROR${NC} ]: $1" >&2 + exit 1 +} + +# Log success message +ok() { + echo -e "[ ${GREEN}OK${NC} ]: $1" >&2 +} + +# Log debug message (only if DEBUG is set) +debug() { + if [[ -n "${DEBUG:-}" ]]; then + echo -e "[ DEBUG ]: $1" >&2 + fi +} diff --git a/src/000-cloud/033-fabric-ontology/scripts/validate-definition.sh b/src/000-cloud/033-fabric-ontology/scripts/validate-definition.sh new file mode 100755 index 00000000..0631311e --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/scripts/validate-definition.sh @@ -0,0 +1,583 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1091 +#=============================================================================== +# Ontology Definition Validation Script +#=============================================================================== +# Validates ontology definition YAML files against schema and semantic rules. +# +# This script performs two levels of validation: +# 1. Structural validation - Required fields, types, allowed values +# 2. Semantic validation - Cross-references, consistency checks +# +# USAGE: +# ./validate-definition.sh --definition +# ./validate-definition.sh -d +# ./validate-definition.sh --help +# +# ARGUMENTS: +# -d, --definition Path to ontology definition YAML file (required) +# -v, --verbose Enable verbose output +# -h, --help Show this help message +# +# EXIT CODES: +# 0 - Definition is valid +# 1 - Validation failed (see error messages) +# 2 - Invalid arguments or missing dependencies +# +# EXAMPLES: +# # Validate the Lakeshore Retail example +# ./validate-definition.sh --definition ../definitions/examples/lakeshore-retail.yaml +# +# # Validate with verbose output +# ./validate-definition.sh -d my-ontology.yaml --verbose +# +# DEPENDENCIES: +# - yq (https://github.com/mikefarah/yq) - YAML parser +# - jq - JSON processor +# +# SEE ALSO: +# - definitions/schema.json - JSON Schema for structural validation +# - lib/definition-parser.sh - YAML parsing utilities +#=============================================================================== + +set -e +set -o pipefail + +# Script location for relative imports +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${SCRIPT_DIR}/lib/definition-parser.sh" + +#=============================================================================== +# Configuration +#=============================================================================== +readonly SUPPORTED_TYPES=("string" "int" "double" "datetime" "boolean" "object") +readonly SUPPORTED_BINDINGS=("static" "timeseries") +readonly SUPPORTED_SOURCES=("lakehouse" "eventhouse") + +#=============================================================================== +# Logging Functions +#=============================================================================== +VERBOSE=${VERBOSE:-false} + +log() { + printf "[ INFO ]: %s\n" "$1" +} + +warn() { + printf "[ WARN ]: %s\n" "$1" >&2 +} + +err() { + printf "[ ERROR ]: %s\n" "$1" >&2 +} + +debug() { + if [[ "$VERBOSE" == "true" ]]; then + printf "[ DEBUG ]: %s\n" "$1" + fi +} + +success() { + printf "[ OK ]: %s\n" "$1" +} + +#=============================================================================== +# Usage +#=============================================================================== +usage() { + cat << 'EOF' +Ontology Definition Validation Script + +Validates ontology definition YAML files before deployment. + +USAGE: + validate-definition.sh --definition [OPTIONS] + +ARGUMENTS: + -d, --definition Path to ontology definition YAML file (required) + +OPTIONS: + -v, --verbose Enable verbose output + -h, --help Show this help message + +EXAMPLES: + # Validate the Lakeshore Retail example + ./validate-definition.sh -d definitions/examples/lakeshore-retail.yaml + + # Validate with verbose output + ./validate-definition.sh -d my-ontology.yaml --verbose + +EXIT CODES: + 0 - Definition is valid + 1 - Validation failed + 2 - Invalid arguments or missing dependencies +EOF +} + +#=============================================================================== +# Argument Parsing +#=============================================================================== +DEFINITION_FILE="" + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + -d|--definition) + DEFINITION_FILE="$2" + shift 2 + ;; + -v|--verbose) + VERBOSE=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + err "Unknown argument: $1" + usage + exit 2 + ;; + esac + done + + if [[ -z "$DEFINITION_FILE" ]]; then + err "Missing required argument: --definition" + usage + exit 2 + fi + + if [[ ! -f "$DEFINITION_FILE" ]]; then + err "Definition file not found: $DEFINITION_FILE" + exit 2 + fi +} + +#=============================================================================== +# Validation Functions +#=============================================================================== +ERRORS=() +WARNINGS=() + +add_error() { + ERRORS+=("$1") + err "$1" +} + +add_warning() { + WARNINGS+=("$1") + warn "$1" +} + +# Check if value is in array +in_array() { + local needle="$1" + shift + local item + for item in "$@"; do + [[ "$item" == "$needle" ]] && return 0 + done + return 1 +} + +#------------------------------------------------------------------------------- +# Validate API version and kind +#------------------------------------------------------------------------------- +validate_api_version() { + debug "Checking apiVersion and kind..." + + local api_version + api_version=$(yq -r '.apiVersion // ""' "$DEFINITION_FILE") + + if [[ -z "$api_version" ]]; then + add_error "Missing required field: apiVersion" + elif [[ "$api_version" != "fabric.ontology/v1" ]]; then + add_error "Invalid apiVersion: '$api_version' (expected 'fabric.ontology/v1')" + fi + + local kind + kind=$(yq -r '.kind // ""' "$DEFINITION_FILE") + + if [[ -z "$kind" ]]; then + add_error "Missing required field: kind" + elif [[ "$kind" != "OntologyDefinition" ]]; then + add_error "Invalid kind: '$kind' (expected 'OntologyDefinition')" + fi +} + +#------------------------------------------------------------------------------- +# Validate metadata section +#------------------------------------------------------------------------------- +validate_metadata() { + debug "Checking metadata..." + + local name + name=$(get_metadata_name "$DEFINITION_FILE") + + if [[ -z "$name" || "$name" == "null" ]]; then + add_error "Missing required field: metadata.name" + else + debug " metadata.name: $name" + fi +} + +#------------------------------------------------------------------------------- +# Validate entity types +#------------------------------------------------------------------------------- +validate_entity_types() { + debug "Checking entityTypes..." + + local count + count=$(get_entity_type_count "$DEFINITION_FILE") + + if [[ "$count" -eq 0 ]]; then + add_error "At least one entityType is required" + return + fi + + debug " Found $count entity type(s)" + + # Collect all entity names for relationship validation + local entity_names=() + while IFS= read -r name; do + entity_names+=("$name") + done < <(get_entity_type_names "$DEFINITION_FILE") + + # Validate each entity type + for entity_name in "${entity_names[@]}"; do + validate_entity_type "$entity_name" + done +} + +validate_entity_type() { + local entity_name="$1" + debug " Validating entity: $entity_name" + + # Get entity key + local key + key=$(get_entity_key "$DEFINITION_FILE" "$entity_name") + + if [[ -z "$key" || "$key" == "null" ]]; then + add_error "Entity '$entity_name': Missing required field 'key'" + return + fi + + # Get property names + local prop_names=() + while IFS= read -r prop_name; do + prop_names+=("$prop_name") + done < <(get_entity_property_names "$DEFINITION_FILE" "$entity_name") + + if [[ ${#prop_names[@]} -eq 0 ]]; then + add_error "Entity '$entity_name': At least one property is required" + return + fi + + # Validate key references a valid property + if ! in_array "$key" "${prop_names[@]}"; then + add_error "Entity '$entity_name': Key '$key' does not reference a valid property. Available: ${prop_names[*]}" + fi + + # Validate each property + local properties + properties=$(get_entity_properties "$DEFINITION_FILE" "$entity_name") + + echo "$properties" | jq -c '.[]' | while read -r prop; do + local prop_name prop_type prop_binding + prop_name=$(echo "$prop" | jq -r '.name') + prop_type=$(echo "$prop" | jq -r '.type') + prop_binding=$(echo "$prop" | jq -r '.binding // "static"') + + # Validate property type + if ! in_array "$prop_type" "${SUPPORTED_TYPES[@]}"; then + add_error "Entity '$entity_name', property '$prop_name': Invalid type '$prop_type'. Supported: ${SUPPORTED_TYPES[*]}" + fi + + # Validate binding type if specified + if [[ "$prop_binding" != "null" ]] && ! in_array "$prop_binding" "${SUPPORTED_BINDINGS[@]}"; then + add_error "Entity '$entity_name', property '$prop_name': Invalid binding '$prop_binding'. Supported: ${SUPPORTED_BINDINGS[*]}" + fi + done + + # Validate data bindings + validate_entity_bindings "$entity_name" +} + +validate_entity_bindings() { + local entity_name="$1" + + # Check for single dataBinding + local single_binding + single_binding=$(get_entity_data_binding "$DEFINITION_FILE" "$entity_name") + + # Check for multiple dataBindings + local multi_bindings + multi_bindings=$(get_entity_data_bindings "$DEFINITION_FILE" "$entity_name") + + local has_single has_multi + has_single=$([[ "$single_binding" != "null" && -n "$single_binding" ]] && echo "true" || echo "false") + has_multi=$([[ $(echo "$multi_bindings" | jq 'length') -gt 0 ]] && echo "true" || echo "false") + + if [[ "$has_single" == "false" && "$has_multi" == "false" ]]; then + add_warning "Entity '$entity_name': No dataBinding or dataBindings defined" + return + fi + + # Validate single binding + if [[ "$has_single" == "true" ]]; then + validate_binding "$entity_name" "$single_binding" "dataBinding" + fi + + # Validate multiple bindings + if [[ "$has_multi" == "true" ]]; then + echo "$multi_bindings" | jq -c '.[]' | while read -r binding; do + local binding_type + binding_type=$(echo "$binding" | jq -r '.type') + validate_binding "$entity_name" "$binding" "dataBindings[$binding_type]" + done + fi +} + +validate_binding() { + local entity_name="$1" + local binding="$2" + local binding_path="$3" + + local binding_type source table + binding_type=$(echo "$binding" | jq -r '.type') + source=$(echo "$binding" | jq -r '.source') + table=$(echo "$binding" | jq -r '.table') + + # Validate binding type + if ! in_array "$binding_type" "${SUPPORTED_BINDINGS[@]}"; then + add_error "Entity '$entity_name', $binding_path: Invalid type '$binding_type'. Supported: ${SUPPORTED_BINDINGS[*]}" + fi + + # Validate source + if ! in_array "$source" "${SUPPORTED_SOURCES[@]}"; then + add_error "Entity '$entity_name', $binding_path: Invalid source '$source'. Supported: ${SUPPORTED_SOURCES[*]}" + fi + + # Validate table is specified + if [[ -z "$table" || "$table" == "null" ]]; then + add_error "Entity '$entity_name', $binding_path: Missing required field 'table'" + fi + + # Validate source is defined in dataSources + if [[ "$source" == "lakehouse" ]]; then + local lakehouse_name + lakehouse_name=$(get_lakehouse_name "$DEFINITION_FILE") + if [[ -z "$lakehouse_name" || "$lakehouse_name" == "null" ]]; then + add_error "Entity '$entity_name', $binding_path: References lakehouse but dataSources.lakehouse is not defined" + else + # Validate table exists in lakehouse + local table_exists + table_exists=$(yq ".dataSources.lakehouse.tables[] | select(.name == \"$table\") | .name" "$DEFINITION_FILE") + if [[ -z "$table_exists" ]]; then + add_error "Entity '$entity_name', $binding_path: Table '$table' not found in dataSources.lakehouse.tables" + fi + fi + elif [[ "$source" == "eventhouse" ]]; then + local eventhouse_name + eventhouse_name=$(get_eventhouse_name "$DEFINITION_FILE") + if [[ -z "$eventhouse_name" || "$eventhouse_name" == "null" ]]; then + add_error "Entity '$entity_name', $binding_path: References eventhouse but dataSources.eventhouse is not defined" + else + # Validate table exists in eventhouse + local table_exists + table_exists=$(yq ".dataSources.eventhouse.tables[] | select(.name == \"$table\") | .name" "$DEFINITION_FILE") + if [[ -z "$table_exists" ]]; then + add_error "Entity '$entity_name', $binding_path: Table '$table' not found in dataSources.eventhouse.tables" + fi + fi + fi + + # Validate timeseries-specific fields + if [[ "$binding_type" == "timeseries" ]]; then + local timestamp_col + timestamp_col=$(echo "$binding" | jq -r '.timestampColumn // ""') + if [[ -z "$timestamp_col" ]]; then + add_error "Entity '$entity_name', $binding_path: Timeseries binding requires 'timestampColumn'" + fi + fi +} + +#------------------------------------------------------------------------------- +# Validate relationships +#------------------------------------------------------------------------------- +validate_relationships() { + debug "Checking relationships..." + + local count + count=$(get_relationship_count "$DEFINITION_FILE") + + if [[ "$count" -eq 0 ]]; then + debug " No relationships defined (optional)" + return + fi + + debug " Found $count relationship(s)" + + # Collect all entity names + local entity_names=() + while IFS= read -r name; do + entity_names+=("$name") + done < <(get_entity_type_names "$DEFINITION_FILE") + + # Validate each relationship + while IFS= read -r rel_name; do + validate_relationship "$rel_name" "${entity_names[@]}" + done < <(get_relationship_names "$DEFINITION_FILE") +} + +validate_relationship() { + local rel_name="$1" + shift + local entity_names=("$@") + + debug " Validating relationship: $rel_name" + + local rel + rel=$(get_relationship "$DEFINITION_FILE" "$rel_name") + + local from_entity to_entity + from_entity=$(echo "$rel" | jq -r '.from') + to_entity=$(echo "$rel" | jq -r '.to') + + # Validate from entity exists + if ! in_array "$from_entity" "${entity_names[@]}"; then + add_error "Relationship '$rel_name': 'from' entity '$from_entity' not found. Available: ${entity_names[*]}" + fi + + # Validate to entity exists + if ! in_array "$to_entity" "${entity_names[@]}"; then + add_error "Relationship '$rel_name': 'to' entity '$to_entity' not found. Available: ${entity_names[*]}" + fi +} + +#------------------------------------------------------------------------------- +# Validate data sources +#------------------------------------------------------------------------------- +validate_data_sources() { + debug "Checking dataSources..." + + local has_lakehouse has_eventhouse + has_lakehouse=$(has_lakehouse "$DEFINITION_FILE" && echo "true" || echo "false") + has_eventhouse=$(has_eventhouse "$DEFINITION_FILE" && echo "true" || echo "false") + + if [[ "$has_lakehouse" == "false" && "$has_eventhouse" == "false" ]]; then + add_warning "No data sources defined (dataSources.lakehouse or dataSources.eventhouse)" + fi + + if [[ "$has_lakehouse" == "true" ]]; then + validate_lakehouse_config + fi + + if [[ "$has_eventhouse" == "true" ]]; then + validate_eventhouse_config + fi +} + +validate_lakehouse_config() { + debug " Validating lakehouse configuration..." + + local name + name=$(get_lakehouse_name "$DEFINITION_FILE") + debug " name: $name" + + local tables + tables=$(get_lakehouse_tables "$DEFINITION_FILE") + local table_count + table_count=$(echo "$tables" | jq 'length') + + if [[ "$table_count" -eq 0 ]]; then + add_error "dataSources.lakehouse: At least one table is required" + fi + + # Validate each table has name + echo "$tables" | jq -c '.[]' | while read -r table; do + local table_name + table_name=$(echo "$table" | jq -r '.name // ""') + if [[ -z "$table_name" ]]; then + add_error "dataSources.lakehouse.tables: Table missing required field 'name'" + fi + done +} + +validate_eventhouse_config() { + debug " Validating eventhouse configuration..." + + local name database + name=$(get_eventhouse_name "$DEFINITION_FILE") + database=$(get_eventhouse_database "$DEFINITION_FILE") + + debug " name: $name" + debug " database: $database" + + if [[ -z "$database" || "$database" == "null" ]]; then + add_error "dataSources.eventhouse: Missing required field 'database'" + fi + + local tables + tables=$(get_eventhouse_tables "$DEFINITION_FILE") + local table_count + table_count=$(echo "$tables" | jq 'length') + + if [[ "$table_count" -eq 0 ]]; then + add_error "dataSources.eventhouse: At least one table is required" + fi + + # Validate each table has name and schema + echo "$tables" | jq -c '.[]' | while read -r table; do + local table_name schema_count + table_name=$(echo "$table" | jq -r '.name // ""') + schema_count=$(echo "$table" | jq '.schema | length // 0') + + if [[ -z "$table_name" ]]; then + add_error "dataSources.eventhouse.tables: Table missing required field 'name'" + elif [[ "$schema_count" -eq 0 ]]; then + add_error "dataSources.eventhouse.tables[$table_name]: Missing required field 'schema'" + fi + done +} + +#=============================================================================== +# Main +#=============================================================================== +main() { + parse_args "$@" + + log "Validating definition: $DEFINITION_FILE" + echo + + # Run all validations + validate_api_version + validate_metadata + validate_data_sources + validate_entity_types + validate_relationships + + echo + + # Summary + local error_count=${#ERRORS[@]} + local warning_count=${#WARNINGS[@]} + + if [[ $error_count -eq 0 ]]; then + success "Definition is valid" + if [[ $warning_count -gt 0 ]]; then + log "$warning_count warning(s)" + fi + exit 0 + else + err "Validation failed with $error_count error(s)" + if [[ $warning_count -gt 0 ]]; then + log "$warning_count warning(s)" + fi + exit 1 + fi +} + +main "$@" diff --git a/src/000-cloud/033-fabric-ontology/templates/kql/create-mapping.kql.tmpl b/src/000-cloud/033-fabric-ontology/templates/kql/create-mapping.kql.tmpl new file mode 100644 index 00000000..6821a9df --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/kql/create-mapping.kql.tmpl @@ -0,0 +1,18 @@ +// KQL CSV Ingestion Mapping Template +// Generates .create-or-alter table ingestion csv mapping command +// +// Required Variables (set via envsubst): +// TABLE_NAME - Name of the table +// MAPPING_NAME - Name for the mapping (typically "${TABLE_NAME}CsvMapping") +// MAPPING_JSON - JSON array of column mappings with Name, DataType, Ordinal +// +// Usage: +// export TABLE_NAME="FreezerTelemetry" +// export MAPPING_NAME="FreezerTelemetryCsvMapping" +// export MAPPING_JSON='[{"Name":"timestamp","DataType":"datetime","Ordinal":0},{"Name":"storeId","DataType":"string","Ordinal":1}]' +// envsubst < create-mapping.kql.tmpl +// +// Output Example: +// .create-or-alter table FreezerTelemetry ingestion csv mapping 'FreezerTelemetryCsvMapping' '[...]' + +.create-or-alter table ${TABLE_NAME} ingestion csv mapping '${MAPPING_NAME}' '${MAPPING_JSON}' diff --git a/src/000-cloud/033-fabric-ontology/templates/kql/create-table.kql.tmpl b/src/000-cloud/033-fabric-ontology/templates/kql/create-table.kql.tmpl new file mode 100644 index 00000000..c33955eb --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/kql/create-table.kql.tmpl @@ -0,0 +1,16 @@ +// KQL Table Creation Template +// Generates .create-merge table command from placeholder variables +// +// Required Variables (set via envsubst): +// TABLE_NAME - Name of the table to create +// COLUMN_SCHEMA - Comma-separated column definitions (e.g., "AssetId: string, Timestamp: datetime") +// +// Usage: +// export TABLE_NAME="FreezerTelemetry" +// export COLUMN_SCHEMA="timestamp: datetime, storeId: string, freezerId: string, temperatureC: real" +// envsubst < create-table.kql.tmpl +// +// Output Example: +// .create-merge table FreezerTelemetry (timestamp: datetime, storeId: string, freezerId: string, temperatureC: real) + +.create-merge table ${TABLE_NAME} (${COLUMN_SCHEMA}) diff --git a/src/000-cloud/033-fabric-ontology/templates/kql/retention-policy.kql.tmpl b/src/000-cloud/033-fabric-ontology/templates/kql/retention-policy.kql.tmpl new file mode 100644 index 00000000..b0ba693c --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/kql/retention-policy.kql.tmpl @@ -0,0 +1,27 @@ +// KQL Retention and Caching Policy Template +// Generates .alter table policy commands for retention and caching +// +// Required Variables (set via envsubst): +// TABLE_NAME - Name of the table +// +// Optional Variables: +// RETENTION_DAYS - Data retention period in days (default: 30) +// CACHING_DAYS - Hot cache period in days (default: 7) +// +// Usage: +// export TABLE_NAME="FreezerTelemetry" +// export RETENTION_DAYS="30" +// export CACHING_DAYS="7" +// envsubst < retention-policy.kql.tmpl +// +// Output Example: +// .alter table FreezerTelemetry policy retention @'{"SoftDeletePeriod":"30.00:00:00","Recoverability":"Enabled"}' +// .alter table FreezerTelemetry policy caching hot = 7d +// +// Notes: +// - SoftDeletePeriod format: "days.hours:minutes:seconds" +// - Caching hot period uses shorthand format: Nd (days) +// - Recoverability is enabled by default for data recovery + +.alter table ${TABLE_NAME} policy retention @'{"SoftDeletePeriod":"${RETENTION_DAYS}.00:00:00","Recoverability":"Enabled"}' +.alter table ${TABLE_NAME} policy caching hot = ${CACHING_DAYS}d diff --git a/src/000-cloud/033-fabric-ontology/templates/ontology/contextualization.json.tmpl b/src/000-cloud/033-fabric-ontology/templates/ontology/contextualization.json.tmpl new file mode 100644 index 00000000..19c51b6a --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/ontology/contextualization.json.tmpl @@ -0,0 +1,12 @@ +{ + "id": "{{ .ContextualizationId }}", + "dataBindingTable": { + "workspaceId": "{{ .WorkspaceId }}", + "itemId": "{{ .LakehouseId }}", + "sourceTableName": "{{ .TableName }}", + "sourceSchema": "dbo", + "sourceType": "LakehouseTable" + }, + "sourceKeyRefBindings": {{ .SourceKeyRefBindings }}, + "targetKeyRefBindings": {{ .TargetKeyRefBindings }} +} diff --git a/src/000-cloud/033-fabric-ontology/templates/ontology/data-binding-eventhouse.json.tmpl b/src/000-cloud/033-fabric-ontology/templates/ontology/data-binding-eventhouse.json.tmpl new file mode 100644 index 00000000..a4771b3c --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/ontology/data-binding-eventhouse.json.tmpl @@ -0,0 +1,16 @@ +{ + "id": "{{ .BindingId }}", + "dataBindingConfiguration": { + "dataBindingType": "TimeSeries", + "timestampColumnName": "{{ .TimestampColumn }}", + "propertyBindings": {{ .PropertyBindings }}, + "sourceTableProperties": { + "sourceType": "KustoTable", + "workspaceId": "{{ .WorkspaceId }}", + "itemId": "{{ .EventhouseId }}", + "clusterUri": "{{ .ClusterUri }}", + "databaseName": "{{ .DatabaseName }}", + "sourceTableName": "{{ .TableName }}" + } + } +} diff --git a/src/000-cloud/033-fabric-ontology/templates/ontology/data-binding-lakehouse.json.tmpl b/src/000-cloud/033-fabric-ontology/templates/ontology/data-binding-lakehouse.json.tmpl new file mode 100644 index 00000000..8257c737 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/ontology/data-binding-lakehouse.json.tmpl @@ -0,0 +1,14 @@ +{ + "id": "{{ .BindingId }}", + "dataBindingConfiguration": { + "dataBindingType": "NonTimeSeries", + "propertyBindings": {{ .PropertyBindings }}, + "sourceTableProperties": { + "sourceType": "LakehouseTable", + "workspaceId": "{{ .WorkspaceId }}", + "itemId": "{{ .LakehouseId }}", + "sourceTableName": "{{ .TableName }}", + "sourceSchema": "dbo" + } + } +} diff --git a/src/000-cloud/033-fabric-ontology/templates/ontology/definition.json.tmpl b/src/000-cloud/033-fabric-ontology/templates/ontology/definition.json.tmpl new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/ontology/definition.json.tmpl @@ -0,0 +1 @@ +{} diff --git a/src/000-cloud/033-fabric-ontology/templates/ontology/entity-type.json.tmpl b/src/000-cloud/033-fabric-ontology/templates/ontology/entity-type.json.tmpl new file mode 100644 index 00000000..23f4e35d --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/ontology/entity-type.json.tmpl @@ -0,0 +1,12 @@ +{ + "id": "{{ .EntityId }}", + "namespace": "usertypes", + "baseEntityTypeId": null, + "name": "{{ .EntityName }}", + "entityIdParts": {{ .EntityIdParts }}, + "displayNamePropertyId": "{{ .DisplayNamePropertyId }}", + "namespaceType": "Custom", + "visibility": "Visible", + "properties": {{ .Properties }}, + "timeseriesProperties": {{ .TimeseriesProperties }} +} diff --git a/src/000-cloud/033-fabric-ontology/templates/ontology/platform.json.tmpl b/src/000-cloud/033-fabric-ontology/templates/ontology/platform.json.tmpl new file mode 100644 index 00000000..bdad588f --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/ontology/platform.json.tmpl @@ -0,0 +1,6 @@ +{ + "metadata": { + "type": "Ontology", + "displayName": "{{ .OntologyName }}" + } +} diff --git a/src/000-cloud/033-fabric-ontology/templates/ontology/relationship-type.json.tmpl b/src/000-cloud/033-fabric-ontology/templates/ontology/relationship-type.json.tmpl new file mode 100644 index 00000000..3fabf763 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/ontology/relationship-type.json.tmpl @@ -0,0 +1,12 @@ +{ + "id": "{{ .RelationshipId }}", + "namespace": "usertypes", + "name": "{{ .RelationshipName }}", + "namespaceType": "Custom", + "source": { + "entityTypeId": "{{ .SourceEntityId }}" + }, + "target": { + "entityTypeId": "{{ .TargetEntityId }}" + } +} diff --git a/src/000-cloud/033-fabric-ontology/templates/semantic-model/database.tmdl.tmpl b/src/000-cloud/033-fabric-ontology/templates/semantic-model/database.tmdl.tmpl new file mode 100644 index 00000000..0ea4a4f5 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/semantic-model/database.tmdl.tmpl @@ -0,0 +1,2 @@ +database '${MODEL_NAME}' + compatibilityLevel: 1604 diff --git a/src/000-cloud/033-fabric-ontology/templates/semantic-model/definition.pbism.tmpl b/src/000-cloud/033-fabric-ontology/templates/semantic-model/definition.pbism.tmpl new file mode 100644 index 00000000..8e62e44b --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/semantic-model/definition.pbism.tmpl @@ -0,0 +1,4 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/semanticModel/definitionProperties/1.0.0/schema.json", + "version": "4.0" +} diff --git a/src/000-cloud/033-fabric-ontology/templates/semantic-model/expressions.tmdl.tmpl b/src/000-cloud/033-fabric-ontology/templates/semantic-model/expressions.tmdl.tmpl new file mode 100644 index 00000000..6078c4c0 --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/semantic-model/expressions.tmdl.tmpl @@ -0,0 +1,3 @@ +expression DatabaseQuery = + AzureStorage.DataLake("https://onelake.dfs.fabric.microsoft.com/${WORKSPACE_ID}/${LAKEHOUSE_ID}") + meta [IsParameterQuery=false] diff --git a/src/000-cloud/033-fabric-ontology/templates/semantic-model/model.tmdl.tmpl b/src/000-cloud/033-fabric-ontology/templates/semantic-model/model.tmdl.tmpl new file mode 100644 index 00000000..562ddb7d --- /dev/null +++ b/src/000-cloud/033-fabric-ontology/templates/semantic-model/model.tmdl.tmpl @@ -0,0 +1,5 @@ +model Model + culture: en-US + defaultPowerBIDataSourceVersion: powerBI_V3 + +${TABLE_REFS} diff --git a/src/000-cloud/055-vpn-gateway/bicep/README.md b/src/000-cloud/055-vpn-gateway/bicep/README.md index fdfc79a8..2c0f8cdb 100644 --- a/src/000-cloud/055-vpn-gateway/bicep/README.md +++ b/src/000-cloud/055-vpn-gateway/bicep/README.md @@ -3,7 +3,7 @@ # VPN Gateway Component -Creates a VPN Gateway with Point-to-Site and optional Site-to-Site connectivity. +Creates a VPN Gateway with Point-to-Site and optional Site-to-Site connectivity. Ths component currently only supports Azure AD (Entra ID) authentication for Point-to-Site VPN connections. ## Parameters diff --git a/src/100-edge/100-cncf-cluster/bicep/README.md b/src/100-edge/100-cncf-cluster/bicep/README.md index 68437673..0299aa80 100644 --- a/src/100-edge/100-cncf-cluster/bicep/README.md +++ b/src/100-edge/100-cncf-cluster/bicep/README.md @@ -8,35 +8,35 @@ The scripts handle primary and secondary node(s) setup, cluster administration, ## Parameters -| Name | Description | Type | Default | Required | -|:---------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------|:---------| -| common | The common component configuration. | `[_1.Common](#user-defined-types)` | n/a | yes | -| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | [format('arck-{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | -| arcOnboardingSpClientId | Service Principal Client ID with Kubernetes Cluster - Azure Arc Onboarding permissions. | `string` | n/a | no | -| arcOnboardingSpClientSecret | The Service Principal Client Secret for Arc onboarding. | `securestring` | n/a | no | -| arcOnboardingSpPrincipalId | Service Principal Object Id used when assigning roles for Arc onboarding. | `string` | n/a | no | -| arcOnboardingIdentityName | The resource name for the identity used for Arc onboarding. | `string` | n/a | no | -| customLocationsOid | The object id of the Custom Locations Entra ID application for your tenant.
Can be retrieved using:

  az ad sp show --id bc313c14-388c-4e7d-a58e-70017303ee3b --query id -o tsv

| `string` | n/a | yes | -| shouldAddCurrentUserClusterAdmin | Whether to add the current user as a cluster admin. | `bool` | `true` | no | -| shouldEnableArcAutoUpgrade | Whether to enable auto-upgrade for Azure Arc agents. | `bool` | [not(equals(parameters('common').environment, 'prod'))] | no | -| clusterAdminOid | The Object ID that will be given cluster-admin permissions. | `string` | n/a | no | -| clusterAdminUpn | The User Principal Name that will be given cluster-admin permissions. | `string` | n/a | no | -| clusterNodeVirtualMachineNames | The node virtual machines names. | `array` | n/a | no | -| clusterServerVirtualMachineName | The server virtual machines name. | `string` | n/a | no | -| clusterServerHostMachineUsername | Username used for the host machines that will be given kube-config settings on setup. (Otherwise, resource_prefix if it exists as a user) | `string` | [parameters('common').resourcePrefix] | no | -| clusterServerIp | The IP address for the server for the cluster. (Needed for mult-node cluster) | `string` | n/a | no | -| serverToken | The token that will be given to the server for the cluster or used by agent nodes. | `securestring` | n/a | no | -| shouldAssignRoles | Whether to assign roles for Arc Onboarding. | `bool` | `true` | no | -| shouldDeployScriptToVm | Whether to deploy the scripts to the VM. | `bool` | `true` | no | -| shouldSkipInstallingAzCli | Should skip downloading and installing Azure CLI on the server. | `bool` | `false` | no | -| shouldSkipAzCliLogin | Should skip login process with Azure CLI on the server. | `bool` | `false` | no | -| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | -| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | n/a | yes | -| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [resourceGroup().name] | no | -| k3sTokenSecretName | The name for the K3s token secret in Key Vault. | `string` | k3s-server-token | no | -| nodeScriptSecretName | The name for the node script secret in Key Vault. | `string` | cluster-node-ubuntu-k3s | no | -| serverScriptSecretName | The name for the server script secret in Key Vault. | `string` | cluster-server-ubuntu-k3s | no | -| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | +| Name | Description | Type | Default | Required | +|:---------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------|:---------| +| common | The common component configuration. | `[_1.Common](#user-defined-types)` | n/a | yes | +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | [format('arck-{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | +| arcOnboardingSpClientId | Service Principal Client ID with Kubernetes Cluster - Azure Arc Onboarding permissions. | `string` | n/a | no | +| arcOnboardingSpClientSecret | The Service Principal Client Secret for Arc onboarding. | `securestring` | n/a | no | +| arcOnboardingSpPrincipalId | Service Principal Object Id used when assigning roles for Arc onboarding. | `string` | n/a | no | +| arcOnboardingIdentityName | The resource name for the identity used for Arc onboarding. | `string` | n/a | no | +| customLocationsOid | The object id of the Custom Locations Entra ID application for your tenant.
Can be retrieved using:

  az ad sp show --id bc313c14-388c-4e7d-a58e-70017303ee3b --query id -o tsv
| `string` | n/a | yes | +| shouldAddCurrentUserClusterAdmin | Whether to add the current user as a cluster admin. | `bool` | `true` | no | +| shouldEnableArcAutoUpgrade | Whether to enable auto-upgrade for Azure Arc agents. | `bool` | [not(equals(parameters('common').environment, 'prod'))] | no | +| clusterAdminOid | The Object ID that will be given cluster-admin permissions. | `string` | n/a | no | +| clusterAdminUpn | The User Principal Name that will be given cluster-admin permissions. | `string` | n/a | no | +| clusterNodeVirtualMachineNames | The node virtual machines names. | `array` | n/a | no | +| clusterServerVirtualMachineName | The server virtual machines name. | `string` | n/a | no | +| clusterServerHostMachineUsername | Username used for the host machines that will be given kube-config settings on setup. (Otherwise, resource_prefix if it exists as a user) | `string` | [parameters('common').resourcePrefix] | no | +| clusterServerIp | The IP address for the server for the cluster. (Needed for mult-node cluster) | `string` | n/a | no | +| serverToken | The token that will be given to the server for the cluster or used by agent nodes. | `securestring` | n/a | no | +| shouldAssignRoles | Whether to assign roles for Arc Onboarding. | `bool` | `true` | no | +| shouldDeployScriptToVm | Whether to deploy the scripts to the VM. | `bool` | `true` | no | +| shouldSkipInstallingAzCli | Should skip downloading and installing Azure CLI on the server. | `bool` | `false` | no | +| shouldSkipAzCliLogin | Should skip login process with Azure CLI on the server. | `bool` | `false` | no | +| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | +| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | n/a | yes | +| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [resourceGroup().name] | no | +| k3sTokenSecretName | The name for the K3s token secret in Key Vault. | `string` | k3s-server-token | no | +| nodeScriptSecretName | The name for the node script secret in Key Vault. | `string` | cluster-node-ubuntu-k3s | no | +| serverScriptSecretName | The name for the server script secret in Key Vault. | `string` | cluster-server-ubuntu-k3s | no | +| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | ## Resources @@ -64,28 +64,28 @@ Configures K3s Kubernetes clusters on Ubuntu virtual machines and connects them #### Parameters for ubuntuK3s -| Name | Description | Type | Default | Required | -|:---------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------|:--------|:---------| -| common | The common component configuration. | `[_1.Common](#user-defined-types)` | n/a | yes | -| arcResourceName | The name of the Azure Arc resource. | `string` | n/a | yes | -| arcTenantId | The tenant ID for Azure Arc resource. | `string` | n/a | yes | -| customLocationsOid | The object id of the Custom Locations Entra ID application for your tenant.
If none is provided, the script will attempt to retrieve this requiring 'Application.Read.All' or 'Directory.Read.All' permissions.
Can be retrieved using:

  az ad sp show --id bc313c14-388c-4e7d-a58e-70017303ee3b --query id -o tsv

| `string` | n/a | yes | -| shouldEnableArcAutoUpgrade | Whether to enable auto-upgrades for Arc agents. | `bool` | n/a | yes | -| arcOnboardingSpClientId | The Service Principal Client ID for Arc onboarding. | `string` | n/a | no | -| arcOnboardingSpClientSecret | The Service Principal Client Secret for Arc onboarding. | `securestring` | n/a | no | -| clusterAdminOid | The Object ID that will be given cluster-admin permissions. | `string` | n/a | no | -| clusterAdminUpn | The User Principal Name that will be given cluster-admin permissions. | `string` | n/a | no | -| clusterServerHostMachineUsername | Username for the host machine with kube-config settings. | `string` | n/a | yes | -| clusterServerIp | The IP address for the server for the cluster. (Needed for mult-node cluster) | `string` | n/a | no | -| deployAdminOid | The Object ID that will be given deployment admin permissions. | `string` | n/a | no | -| serverToken | The token that will be given to the server for the cluster or used by agent nodes. | `securestring` | n/a | no | -| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | n/a | yes | -| keyVaultName | The name of the Key Vault to save the scripts to. | `string` | n/a | yes | -| k3sTokenSecretName | The name for the K3s token secret in Key Vault. | `string` | n/a | yes | -| nodeScriptSecretName | The name for the node script secret in Key Vault. | `string` | n/a | yes | -| serverScriptSecretName | The name for the server script secret in Key Vault. | `string` | n/a | yes | -| shouldSkipAzCliLogin | Should skip login process with Azure CLI on the server. | `bool` | n/a | yes | -| shouldSkipInstallingAzCli | Should skip downloading and installing Azure CLI on the server. | `bool` | n/a | yes | +| Name | Description | Type | Default | Required | +|:---------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------|:--------|:---------| +| common | The common component configuration. | `[_1.Common](#user-defined-types)` | n/a | yes | +| arcResourceName | The name of the Azure Arc resource. | `string` | n/a | yes | +| arcTenantId | The tenant ID for Azure Arc resource. | `string` | n/a | yes | +| customLocationsOid | The object id of the Custom Locations Entra ID application for your tenant.
If none is provided, the script will attempt to retrieve this requiring 'Application.Read.All' or 'Directory.Read.All' permissions.
Can be retrieved using:

  az ad sp show --id bc313c14-388c-4e7d-a58e-70017303ee3b --query id -o tsv
| `string` | n/a | yes | +| shouldEnableArcAutoUpgrade | Whether to enable auto-upgrades for Arc agents. | `bool` | n/a | yes | +| arcOnboardingSpClientId | The Service Principal Client ID for Arc onboarding. | `string` | n/a | no | +| arcOnboardingSpClientSecret | The Service Principal Client Secret for Arc onboarding. | `securestring` | n/a | no | +| clusterAdminOid | The Object ID that will be given cluster-admin permissions. | `string` | n/a | no | +| clusterAdminUpn | The User Principal Name that will be given cluster-admin permissions. | `string` | n/a | no | +| clusterServerHostMachineUsername | Username for the host machine with kube-config settings. | `string` | n/a | yes | +| clusterServerIp | The IP address for the server for the cluster. (Needed for mult-node cluster) | `string` | n/a | no | +| deployAdminOid | The Object ID that will be given deployment admin permissions. | `string` | n/a | no | +| serverToken | The token that will be given to the server for the cluster or used by agent nodes. | `securestring` | n/a | no | +| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | n/a | yes | +| keyVaultName | The name of the Key Vault to save the scripts to. | `string` | n/a | yes | +| k3sTokenSecretName | The name for the K3s token secret in Key Vault. | `string` | n/a | yes | +| nodeScriptSecretName | The name for the node script secret in Key Vault. | `string` | n/a | yes | +| serverScriptSecretName | The name for the server script secret in Key Vault. | `string` | n/a | yes | +| shouldSkipAzCliLogin | Should skip login process with Azure CLI on the server. | `bool` | n/a | yes | +| shouldSkipInstallingAzCli | Should skip downloading and installing Azure CLI on the server. | `bool` | n/a | yes | #### Resources for ubuntuK3s diff --git a/src/100-edge/109-arc-extensions/README.md b/src/100-edge/109-arc-extensions/README.md new file mode 100644 index 00000000..827f4422 --- /dev/null +++ b/src/100-edge/109-arc-extensions/README.md @@ -0,0 +1,269 @@ +--- +title: Arc Extensions +description: Component to deploy foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA) required by Azure IoT Operations and other Arc-enabled services +author: Edge AI Team +ms.date: 2025-12-30 +ms.topic: reference +keywords: + - arc extensions + - cert-manager + - azure container storage + - acsa + - edge volumes + - kubernetes extensions + - terraform + - bicep +estimated_reading_time: 5 +--- + +## Arc Extensions + +Component to deploy foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA). These extensions provide certificate management and persistent storage capabilities required by Azure IoT Operations and other Arc-enabled services. + +Learn more about the required configuration by reading the [./terraform/README.md](./terraform/README.md) + +## Extensions Deployed + +This component deploys two critical Arc extensions in the correct dependency order: + +### cert-manager (microsoft.certmanagement) + +Certificate management extension that provides automated certificate lifecycle management using cert-manager and trust-manager. This extension is a foundational dependency required by: + +- Azure Container Storage (ACSA) +- Azure IoT Operations Secret Store +- Any workload requiring automated TLS certificate management + +**Key Features:** + +- Automated certificate issuance and renewal +- Trust bundle management via trust-manager +- Integration with various certificate issuers +- Deployed to `cert-manager` namespace + +### Azure Container Storage (ACSA) (microsoft.arc.containerstorage) + +Provides persistent storage capabilities for Arc-enabled Kubernetes clusters with support for: + +- **Edge Volumes**: Local persistent storage on edge devices +- **Cloud Ingest**: Seamless data upload from edge to Azure Blob Storage +- **Fault-Tolerant Storage Pools**: High-availability storage configurations for multi-node clusters + +**Use Cases:** + +- Persistent data storage for IoT Operations workloads +- Media capture and buffering for media connector +- Local caching and data retention at the edge +- Automated cloud upload for analytics and archival + +## Dependencies + +**Requires:** + +- Arc-connected Kubernetes cluster (from 100-cncf-cluster component) +- Sufficient disk space on cluster nodes for storage volumes + +**Required by:** + +- 110-iot-ops component (requires cert-manager extension for Secret Store deployment) +- Media connector and other services using ACSA for persistent storage + +**Internal Dependencies:** + +- Azure Container Storage extension depends on cert-manager extension (enforced via `depends_on`) + +## Deployment Order + +This component must be deployed: + +1. **After**: 100-cncf-cluster (Arc-connected cluster must exist) +2. **Before**: 110-iot-ops (provides required cert-manager dependency) + +**Internal Extension Order:** + +1. cert-manager extension (foundational) +2. Azure Container Storage extension (depends on cert-manager) + +## Extension Configuration + +Both extensions support flexible configuration options: + +### Common Configuration + +- **Version and release train**: Configure specific extension versions and stability channels +- **Conditional deployment**: Enable/disable each extension independently via `enabled` flag +- **Extension-specific settings**: Customize behavior per extension requirements + +### cert-manager Configuration + +Configure certificate management behavior: + +- **Agent operation timeout**: Timeout for extension operations (default: 20 minutes) +- **Global telemetry**: Enable/disable telemetry collection (default: enabled) +- **Release train**: Stable, preview, or custom channels (default: stable) +- **Version**: Specific extension version (default: 0.7.0) + +### Container Storage Configuration + +Configure storage capabilities: + +- **Disk storage class**: Kubernetes storage class for persistent volumes (default: auto-detected) +- **Fault tolerance mode**: Enable fault-tolerant storage pools for multi-node clusters (default: disabled) +- **Disk mount point**: Host path for storage pool data (default: /mnt) +- **Release train**: Stable, preview, or custom channels (default: stable) +- **Version**: Specific extension version (default: 2.6.0) + +**Note**: Fault tolerance requires multi-node clusters and consumes additional disk space for replication. + +## Terraform + +Refer to [Terraform Components - Getting Started](../README.md#terraform-components---getting-started) for deployment instructions. + +Learn more about the required configuration by reading the [./terraform/README.md](./terraform/README.md) + +### Example Configuration + +Add the following to your `terraform.tfvars` file: + +```hcl +# Enable both extensions with default settings +arc_extensions = { + cert_manager_extension = { + enabled = true + version = "0.7.0" + train = "stable" + agent_operation_timeout_in_minutes = 20 + global_telemetry_enabled = true + } + + container_storage_extension = { + enabled = true + version = "2.6.0" + train = "stable" + disk_storage_class = "" # Auto-detect + fault_tolerance_enabled = false + disk_mount_point = "/mnt" + } +} +``` + +### Disable Specific Extensions + +To deploy only cert-manager without ACSA: + +```hcl +arc_extensions = { + cert_manager_extension = { + enabled = true + version = "0.7.0" + train = "stable" + agent_operation_timeout_in_minutes = 20 + global_telemetry_enabled = true + } + + container_storage_extension = { + enabled = false + version = "2.6.0" + train = "stable" + disk_storage_class = "" + fault_tolerance_enabled = false + disk_mount_point = "/mnt" + } +} +``` + +## Bicep + +Learn more about the required configuration by reading the [./bicep/README.md](./bicep/README.md) + +## Troubleshooting + +### Extension Installation Issues + +Check extension status using Azure CLI: + +```sh +# List all extensions on the cluster +az k8s-extension list \ + --cluster-name \ + --resource-group \ + --cluster-type connectedClusters + +# Check specific extension status +az k8s-extension show \ + --name certmanager \ + --cluster-name \ + --resource-group \ + --cluster-type connectedClusters +``` + +### cert-manager Issues + +Verify cert-manager pods are running: + +```sh +kubectl get pods -n cert-manager +``` + +Check cert-manager logs: + +```sh +kubectl logs -n cert-manager -l app=cert-manager +kubectl logs -n cert-manager -l app=webhook +kubectl logs -n cert-manager -l app=cainjector +``` + +### Azure Container Storage Issues + +Verify ACSA pods are running: + +```sh +kubectl get pods -n azure-arc-containerstorage +``` + +Check storage pool status: + +```sh +kubectl get storagepools -A +``` + +View ACSA logs: + +```sh +kubectl logs -n azure-arc-containerstorage -l app=azure-arc-containerstorage +``` + +### Common Issues + +**Extension installation timeout**: + +- Increase `agent_operation_timeout_in_minutes` for cert-manager +- Verify cluster has internet connectivity to download extension components + +**ACSA fails to create storage pool**: + +- Verify sufficient disk space at configured `disk_mount_point` +- Check node has required kernel modules loaded +- For fault tolerance, verify cluster has multiple nodes + +**cert-manager webhook errors**: + +- Verify cluster DNS is functioning correctly +- Check webhook service and endpoints are created + +## References + +- [Azure Arc Extensions Overview](https://learn.microsoft.com/azure/azure-arc/kubernetes/extensions) +- [cert-manager Documentation](https://cert-manager.io/docs/) +- [Azure Container Storage for Arc](https://learn.microsoft.com/azure/azure-arc/container-storage/) +- [Install Edge Volumes](https://learn.microsoft.com/azure/azure-arc/container-storage/howto-install-edge-volumes?tabs=single) +- [Configure Cloud Ingest](https://learn.microsoft.com/azure/azure-arc/container-storage/howto-configure-cloud-ingest-subvolumes?tabs=portal) +- [Multi-Node Storage Configuration](https://learn.microsoft.com/azure/iot-operations/deploy-iot-ops/howto-prepare-cluster?tabs=ubuntu#configure-multi-node-clusters-for-azure-container-storage-enabled-by-azure-arc) +- [Media Connector with ACSA](https://learn.microsoft.com/azure/iot-operations/discover-manage-assets/howto-use-media-connector?tabs=portal#deploy-the-media-connector) + +--- + + +*🤖 Crafted with precision by ✨Copilot following brilliant human instruction, +then carefully refined by our team of discerning human reviewers.* + diff --git a/src/100-edge/109-arc-extensions/bicep/README.md b/src/100-edge/109-arc-extensions/bicep/README.md new file mode 100644 index 00000000..4a450a24 --- /dev/null +++ b/src/100-edge/109-arc-extensions/bicep/README.md @@ -0,0 +1,64 @@ + + + +# Arc Extensions + +Deploys foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA). + +## Parameters + +| Name | Description | Type | Default | Required | +|:------------------------|:----------------------------------------------------------------------|:------------------------------------------------------|:----------------------------------------------------|:---------| +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | +| certManagerConfig | The settings for the cert-manager Extension. | `[_1.CertManagerExtension](#user-defined-types)` | [variables('_1.certManagerExtensionDefaults')] | no | +| containerStorageConfig | The settings for the Azure Container Storage for Azure Arc Extension. | `[_1.ContainerStorageExtension](#user-defined-types)` | [variables('_1.containerStorageExtensionDefaults')] | no | + +## Resources + +| Name | Type | API Version | +|:-----------------|:-----------------------------------------------|:------------| +| aioCertManager | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | +| containerStorage | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | + +## User Defined Types + +### `_1.CertManagerExtension` + +The settings for the cert-manager Extension. + +| Property | Type | Description | +|:---------|:------------------------------------|:----------------------------------------------| +| enabled | `bool` | Whether to deploy the cert-manager extension. | +| release | `[_1.Release](#user-defined-types)` | The common settings for the extension. | +| settings | `object` | | + +### `_1.ContainerStorageExtension` + +The settings for the Azure Container Storage for Azure Arc Extension. + +| Property | Type | Description | +|:---------|:------------------------------------|:---------------------------------------------------| +| enabled | `bool` | Whether to deploy the container storage extension. | +| release | `[_1.Release](#user-defined-types)` | The common settings for the extension. | +| settings | `object` | | + +### `_1.Release` + +The common settings for Azure Arc Extensions. + +| Property | Type | Description | +|:------------------------|:---------|:-----------------------------------------------------------------------------| +| version | `string` | The version of the extension. | +| train | `string` | The release train that has the version to deploy (ex., "preview", "stable"). | +| autoUpgradeMinorVersion | `bool` | Whether to automatically upgrade minor versions of the extension. | + +## Outputs + +| Name | Type | Description | +|:------------------------------|:---------|:----------------------------------------------------------| +| certManagerExtensionId | `string` | The resource ID of the cert-manager extension. | +| certManagerExtensionName | `string` | The name of the cert-manager extension. | +| containerStorageExtensionId | `string` | The resource ID of the Azure Container Storage extension. | +| containerStorageExtensionName | `string` | The name of the Azure Container Storage extension. | + + \ No newline at end of file diff --git a/src/100-edge/109-arc-extensions/bicep/main.bicep b/src/100-edge/109-arc-extensions/bicep/main.bicep new file mode 100644 index 00000000..ec751a77 --- /dev/null +++ b/src/100-edge/109-arc-extensions/bicep/main.bicep @@ -0,0 +1,112 @@ +metadata name = 'Arc Extensions' +metadata description = 'Deploys foundational Arc-enabled Kubernetes cluster extensions including cert-manager and Azure Container Storage (ACSA).' + +import * as types from './types.bicep' + +/* + Common Parameters +*/ + +@description('The resource name for the Arc connected cluster.') +param arcConnectedClusterName string + +/* + Extension Parameters +*/ + +@description('The settings for the cert-manager Extension.') +param certManagerConfig types.CertManagerExtension = types.certManagerExtensionDefaults + +@description('The settings for the Azure Container Storage for Azure Arc Extension.') +param containerStorageConfig types.ContainerStorageExtension = types.containerStorageExtensionDefaults + +/* + Existing Resources +*/ + +resource arcConnectedCluster 'Microsoft.Kubernetes/connectedClusters@2024-12-01-preview' existing = { + name: arcConnectedClusterName +} + +/* + cert-manager Extension +*/ + +resource aioCertManager 'Microsoft.KubernetesConfiguration/extensions@2024-11-01' = if (certManagerConfig.enabled) { + name: 'arc-cert-manager' + scope: arcConnectedCluster + identity: { + type: 'SystemAssigned' + } + properties: { + extensionType: 'microsoft.certmanagement' + version: certManagerConfig.release.version + releaseTrain: certManagerConfig.release.train + autoUpgradeMinorVersion: certManagerConfig.release.?autoUpgradeMinorVersion ?? false + scope: { + cluster: { + releaseNamespace: 'cert-manager' + } + } + configurationSettings: { + AgentOperationTimeoutInMinutes: certManagerConfig.settings.?agentOperationTimeoutInMinutes + 'global.telemetry.enabled': string(certManagerConfig.settings.?globalTelemetryEnabled ?? true) + } + } +} + +/* + Azure Container Storage Extension +*/ + +var defaultStorageClass = containerStorageConfig.settings.?faultToleranceEnabled + ? 'acstor-arccontainerstorage-storage-pool' + : 'default,local-path' +var kubernetesStorageClass = containerStorageConfig.settings.?diskStorageClass ?? defaultStorageClass +var diskMountPoint = containerStorageConfig.settings.?diskMountPoint ?? '/mnt' + +var containerStorageSettings = containerStorageConfig.settings.?faultToleranceEnabled + ? { + 'edgeStorageConfiguration.create': 'true' + 'feature.diskStorageClass': kubernetesStorageClass + 'acstorConfiguration.create': 'true' + 'acstorConfiguration.properties.diskMountPoint': diskMountPoint + } + : { + 'edgeStorageConfiguration.create': 'true' + 'feature.diskStorageClass': kubernetesStorageClass + } + +resource containerStorage 'Microsoft.KubernetesConfiguration/extensions@2024-11-01' = if (containerStorageConfig.enabled) { + name: 'azure-arc-containerstorage' + scope: arcConnectedCluster + identity: { + type: 'SystemAssigned' + } + properties: { + extensionType: 'microsoft.arc.containerstorage' + version: containerStorageConfig.release.version + releaseTrain: containerStorageConfig.release.train + autoUpgradeMinorVersion: false + configurationSettings: containerStorageSettings + } + dependsOn: [ + aioCertManager + ] +} + +/* + Outputs +*/ + +@description('The resource ID of the cert-manager extension.') +output certManagerExtensionId string = certManagerConfig.enabled ? aioCertManager.id : '' + +@description('The name of the cert-manager extension.') +output certManagerExtensionName string = certManagerConfig.enabled ? aioCertManager.name : '' + +@description('The resource ID of the Azure Container Storage extension.') +output containerStorageExtensionId string = containerStorageConfig.enabled ? containerStorage.id : '' + +@description('The name of the Azure Container Storage extension.') +output containerStorageExtensionName string = containerStorageConfig.enabled ? containerStorage.name : '' diff --git a/src/100-edge/109-arc-extensions/bicep/types.bicep b/src/100-edge/109-arc-extensions/bicep/types.bicep new file mode 100644 index 00000000..45ed4db3 --- /dev/null +++ b/src/100-edge/109-arc-extensions/bicep/types.bicep @@ -0,0 +1,83 @@ +/* + * IMPORTANT: The variable names in this file ('certManagerExtensionDefaults', + * 'containerStorageExtensionDefaults') are explicitly referenced + * by the aio-version-checker.py script. If you rename these variables or change their structure, + * you must also update the script and the aio-version-checker-template.yml pipeline. + */ + +@export() +@description('The common settings for Azure Arc Extensions.') +type Release = { + @description('The version of the extension.') + version: string + + @description('The release train that has the version to deploy (ex., "preview", "stable").') + train: string + + @description('Whether to automatically upgrade minor versions of the extension.') + autoUpgradeMinorVersion: bool? +} + +@export() +@description('The settings for the cert-manager Extension.') +type CertManagerExtension = { + @description('Whether to deploy the cert-manager extension.') + enabled: bool + + @description('The common settings for the extension.') + release: Release + + settings: { + @description('Agent operation timeout in minutes.') + agentOperationTimeoutInMinutes: string + @description('Enable or disable global telemetry.') + globalTelemetryEnabled: bool? + } +} + +@export() +var certManagerExtensionDefaults = { + enabled: true + release: { + version: '0.7.0' + train: 'stable' + autoUpgradeMinorVersion: false + } + settings: { + agentOperationTimeoutInMinutes: '20' + globalTelemetryEnabled: true + } +} + +@export() +@description('The settings for the Azure Container Storage for Azure Arc Extension.') +type ContainerStorageExtension = { + @description('Whether to deploy the container storage extension.') + enabled: bool + + @description('The common settings for the extension.') + release: Release + + settings: { + @description('Whether or not to enable fault tolerant storage in the cluster.') + faultToleranceEnabled: bool + + @description('The storage class for the cluster. (Otherwise, "acstor-arccontainerstorage-storage-pool" for fault tolerant storage else "default,local-path")') + diskStorageClass: string? + + @description('The disk mount point for the cluster when using fault tolerant storage. (Otherwise: "/mnt" when using fault tolerant storage)') + diskMountPoint: string? + } +} + +@export() +var containerStorageExtensionDefaults = { + enabled: true + release: { + version: '2.6.0' + train: 'stable' + } + settings: { + faultToleranceEnabled: false + } +} diff --git a/src/100-edge/109-arc-extensions/bicep/types.core.bicep b/src/100-edge/109-arc-extensions/bicep/types.core.bicep new file mode 100644 index 00000000..7f61cc1d --- /dev/null +++ b/src/100-edge/109-arc-extensions/bicep/types.core.bicep @@ -0,0 +1,15 @@ +@export() +@description('The common component configuration.') +type Common = { + @description('The environment for all resources. Example values: dev, test, prod.') + environment: string + + @description('The prefix for all resources.') + resourcePrefix: string + + @description('The location/region for all resources.') + location: string + + @description('The instance identifier for naming resources. Example values: 001, 002, etc.') + instance: string +} diff --git a/src/100-edge/109-arc-extensions/ci/bicep/main.bicep b/src/100-edge/109-arc-extensions/ci/bicep/main.bicep new file mode 100644 index 00000000..552b1662 --- /dev/null +++ b/src/100-edge/109-arc-extensions/ci/bicep/main.bicep @@ -0,0 +1,40 @@ +metadata name = 'Arc Extensions CI' +metadata description = 'CI deployment for Arc Extensions component.' + +import * as core from '../../bicep/types.core.bicep' + +/* + Common Parameters +*/ + +@description('The common component configuration.') +param common core.Common + +/* + Existing Resources +*/ + +var resourceGroupName = 'rg-${common.resourcePrefix}-${common.environment}-${common.instance}' +var arcConnectedClusterName = 'arck-${common.resourcePrefix}-${common.environment}-${common.instance}' + +resource resourceGroup 'Microsoft.Resources/resourceGroups@2024-03-01' existing = { + name: resourceGroupName + scope: subscription() +} + +resource arcConnectedCluster 'Microsoft.Kubernetes/connectedClusters@2024-12-01-preview' existing = { + name: arcConnectedClusterName + scope: resourceGroup +} + +/* + Module Deployment +*/ + +module ci '../../bicep/main.bicep' = { + name: 'arc-extensions-ci-deployment' + scope: resourceGroup + params: { + arcConnectedClusterName: arcConnectedCluster.name + } +} diff --git a/src/100-edge/109-arc-extensions/ci/terraform/main.tf b/src/100-edge/109-arc-extensions/ci/terraform/main.tf new file mode 100644 index 00000000..8fe29d85 --- /dev/null +++ b/src/100-edge/109-arc-extensions/ci/terraform/main.tf @@ -0,0 +1,24 @@ +resource "terraform_data" "defer" { + input = { + resource_group_name = "rg-${var.resource_prefix}-${var.environment}-${var.instance}" + arc_connected_cluster_name = "arck-${var.resource_prefix}-${var.environment}-${var.instance}" + } +} + +data "azurerm_resource_group" "arc_extensions" { + name = terraform_data.defer.input.resource_group_name +} + +data "azapi_resource" "arc_connected_cluster" { + type = "Microsoft.Kubernetes/connectedClusters@2024-01-01" + parent_id = data.azurerm_resource_group.arc_extensions.id + name = terraform_data.defer.input.arc_connected_cluster_name + + response_export_values = ["name", "id", "location"] +} + +module "ci" { + source = "../../terraform" + + arc_connected_cluster = data.azapi_resource.arc_connected_cluster.output +} diff --git a/src/100-edge/109-arc-extensions/ci/terraform/variables.tf b/src/100-edge/109-arc-extensions/ci/terraform/variables.tf new file mode 100644 index 00000000..0599151c --- /dev/null +++ b/src/100-edge/109-arc-extensions/ci/terraform/variables.tf @@ -0,0 +1,19 @@ +variable "environment" { + type = string + description = "Environment for all resources in this module: dev, test, or prod" +} + +variable "resource_prefix" { + type = string + validation { + condition = length(var.resource_prefix) > 0 && can(regex("^[a-zA-Z](?:-?[a-zA-Z0-9])*$", var.resource_prefix)) + error_message = "Resource prefix must not be empty, must only contain alphanumeric characters and dashes. Must start with an alphabetic character." + } + description = "Prefix for all resources in this module" +} + +variable "instance" { + type = string + description = "Instance identifier for naming resources: 001, 002, etc" + default = "001" +} diff --git a/src/100-edge/109-arc-extensions/ci/terraform/versions.tf b/src/100-edge/109-arc-extensions/ci/terraform/versions.tf new file mode 100644 index 00000000..3f132b50 --- /dev/null +++ b/src/100-edge/109-arc-extensions/ci/terraform/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 4.8.0" + } + azapi = { + source = "Azure/azapi" + version = ">= 2.3.0" + } + } + required_version = ">= 1.9.8, < 2.0" +} + +provider "azurerm" { + storage_use_azuread = true + features {} +} diff --git a/src/100-edge/109-arc-extensions/terraform/README.md b/src/100-edge/109-arc-extensions/terraform/README.md new file mode 100644 index 00000000..8349250d --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/README.md @@ -0,0 +1,38 @@ + +# Arc Extensions + +Deploys foundational Arc-enabled Kubernetes cluster extensions including +cert-manager and Azure Container Storage (ACSA). + +## Requirements + +| Name | Version | +|-----------|-----------------| +| terraform | >= 1.9.8, < 2.0 | +| azurerm | >= 4.8.0 | + +## Modules + +| Name | Source | Version | +|-------------------------------|-----------------------------|---------| +| cert\_manager\_extension | ./modules/cert-manager | n/a | +| container\_storage\_extension | ./modules/container-storage | n/a | + +## Inputs + +| Name | Description | Type | Default | Required | +|-------------------------|---------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:| +| arc\_connected\_cluster | Arc-connected Kubernetes cluster object containing id, name, and location | ```object({ id = string name = string location = string })``` | n/a | yes | +| arc\_extensions | Combined configuration object for Arc extensions (cert-manager and container storage) | ```object({ cert_manager_extension = optional(object({ enabled = optional(bool) version = optional(string) train = optional(string) auto_upgrade_minor_version = optional(bool) agent_operation_timeout_in_minutes = optional(number) global_telemetry_enabled = optional(bool) })) container_storage_extension = optional(object({ enabled = optional(bool) version = optional(string) train = optional(string) auto_upgrade_minor_version = optional(bool) disk_storage_class = optional(string) fault_tolerance_enabled = optional(bool) disk_mount_point = optional(string) })) })``` | ```{ "cert_manager_extension": { "agent_operation_timeout_in_minutes": 20, "auto_upgrade_minor_version": false, "enabled": true, "global_telemetry_enabled": true, "train": "stable", "version": "0.7.0" }, "container_storage_extension": { "auto_upgrade_minor_version": false, "disk_mount_point": "/mnt", "disk_storage_class": "", "enabled": true, "fault_tolerance_enabled": false, "train": "stable", "version": "2.6.0" } }``` | no | + +## Outputs + +| Name | Description | +|-------------------------------------|------------------------------------------------------------------------------------------------------| +| cert\_manager\_extension | Self-contained cert\_manager object (id, name, enabled, version, train) or null if not deployed | +| cert\_manager\_extension\_id | The resource ID of the cert-manager extension. | +| cert\_manager\_extension\_name | The name of the cert-manager extension. | +| container\_storage\_extension | Self-contained container\_storage object (id, name, enabled, version, train) or null if not deployed | +| container\_storage\_extension\_id | The resource ID of the Azure Container Storage extension. | +| container\_storage\_extension\_name | The name of the Azure Container Storage extension. | + diff --git a/src/100-edge/109-arc-extensions/terraform/main.tf b/src/100-edge/109-arc-extensions/terraform/main.tf new file mode 100644 index 00000000..79e5d4bf --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/main.tf @@ -0,0 +1,26 @@ +/** + * # Arc Extensions + * + * Deploys foundational Arc-enabled Kubernetes cluster extensions including + * cert-manager and Azure Container Storage (ACSA). + */ + +module "cert_manager_extension" { + count = var.arc_extensions.cert_manager_extension.enabled ? 1 : 0 + + source = "./modules/cert-manager" + + arc_connected_cluster_id = var.arc_connected_cluster.id + cert_manager_extension = var.arc_extensions.cert_manager_extension +} + +module "container_storage_extension" { + count = var.arc_extensions.container_storage_extension.enabled ? 1 : 0 + + source = "./modules/container-storage" + + depends_on = [module.cert_manager_extension] + + arc_connected_cluster_id = var.arc_connected_cluster.id + container_storage_extension = var.arc_extensions.container_storage_extension +} diff --git a/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/README.md b/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/README.md new file mode 100644 index 00000000..9a3d9fa0 --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/README.md @@ -0,0 +1,37 @@ + +# cert-manager Extension Module + +Deploys the cert-manager extension for Arc-enabled Kubernetes clusters. + +## Requirements + +| Name | Version | +|-----------|-----------------| +| terraform | >= 1.9.8, < 2.0 | + +## Providers + +| Name | Version | +|---------|---------| +| azurerm | n/a | + +## Resources + +| Name | Type | +|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| +| [azurerm_arc_kubernetes_cluster_extension.cert_manager](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/arc_kubernetes_cluster_extension) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|-----------------------------|---------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|:--------:| +| arc\_connected\_cluster\_id | The resource ID of the Arc-connected Kubernetes cluster | `string` | n/a | yes | +| cert\_manager\_extension | cert-manager extension configuration object | ```object({ enabled = optional(bool) version = optional(string) train = optional(string) auto_upgrade_minor_version = optional(bool) agent_operation_timeout_in_minutes = optional(number) global_telemetry_enabled = optional(bool) })``` | n/a | yes | + +## Outputs + +| Name | Description | +|---------------|-----------------------------------------------------| +| cert\_manager | Self-contained cert-manager object | +| extension | The cert-manager extension id and name as an object | + diff --git a/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/main.tf b/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/main.tf new file mode 100644 index 00000000..7eafa300 --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/main.tf @@ -0,0 +1,21 @@ +/** + * # cert-manager Extension Module + * + * Deploys the cert-manager extension for Arc-enabled Kubernetes clusters. + */ + +resource "azurerm_arc_kubernetes_cluster_extension" "cert_manager" { + name = "arc-cert-manager" + cluster_id = var.arc_connected_cluster_id + extension_type = "microsoft.certmanagement" + identity { + type = "SystemAssigned" + } + version = var.cert_manager_extension.version + release_train = var.cert_manager_extension.train + release_namespace = "cert-manager" + configuration_settings = { + "AgentOperationTimeoutInMinutes" = tostring(var.cert_manager_extension.agent_operation_timeout_in_minutes) + "global.telemetry.enabled" = var.cert_manager_extension.global_telemetry_enabled + } +} diff --git a/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/outputs.tf b/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/outputs.tf new file mode 100644 index 00000000..89a80c8d --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/outputs.tf @@ -0,0 +1,18 @@ +output "cert_manager" { + description = "Self-contained cert-manager object" + value = { + enabled = var.cert_manager_extension.enabled + id = azurerm_arc_kubernetes_cluster_extension.cert_manager.id + name = azurerm_arc_kubernetes_cluster_extension.cert_manager.name + version = var.cert_manager_extension.version + train = var.cert_manager_extension.train + } +} + +output "extension" { + description = "The cert-manager extension id and name as an object" + value = { + id = azurerm_arc_kubernetes_cluster_extension.cert_manager.id + name = azurerm_arc_kubernetes_cluster_extension.cert_manager.name + } +} diff --git a/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/variables.tf b/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/variables.tf new file mode 100644 index 00000000..868daa77 --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/variables.tf @@ -0,0 +1,15 @@ +variable "arc_connected_cluster_id" { + type = string + description = "The resource ID of the Arc-connected Kubernetes cluster" +} +variable "cert_manager_extension" { + type = object({ + enabled = optional(bool) + version = optional(string) + train = optional(string) + auto_upgrade_minor_version = optional(bool) + agent_operation_timeout_in_minutes = optional(number) + global_telemetry_enabled = optional(bool) + }) + description = "cert-manager extension configuration object" +} diff --git a/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/versions.tf b/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/versions.tf new file mode 100644 index 00000000..cb47f2e9 --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/modules/cert-manager/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_version = ">= 1.9.8, < 2.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + } + } +} diff --git a/src/100-edge/109-arc-extensions/terraform/modules/container-storage/README.md b/src/100-edge/109-arc-extensions/terraform/modules/container-storage/README.md new file mode 100644 index 00000000..030b415a --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/modules/container-storage/README.md @@ -0,0 +1,37 @@ + +# Azure Container Storage Extension Module + +Deploys the Azure Container Storage (ACSA) extension for Arc-enabled Kubernetes clusters. + +## Requirements + +| Name | Version | +|-----------|-----------------| +| terraform | >= 1.9.8, < 2.0 | + +## Providers + +| Name | Version | +|---------|---------| +| azurerm | n/a | + +## Resources + +| Name | Type | +|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| +| [azurerm_arc_kubernetes_cluster_extension.container_storage](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/arc_kubernetes_cluster_extension) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|-------------------------------|---------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|:--------:| +| arc\_connected\_cluster\_id | The resource ID of the Arc-connected Kubernetes cluster | `string` | n/a | yes | +| container\_storage\_extension | container-storage extension configuration object | ```object({ enabled = optional(bool) version = optional(string) train = optional(string) auto_upgrade_minor_version = optional(bool) disk_storage_class = optional(string) fault_tolerance_enabled = optional(bool) disk_mount_point = optional(string) })``` | n/a | yes | + +## Outputs + +| Name | Description | +|--------------------|----------------------------------------------------------| +| container\_storage | Self-contained container\_storage object | +| extension | The container storage extension id and name as an object | + diff --git a/src/100-edge/109-arc-extensions/terraform/modules/container-storage/main.tf b/src/100-edge/109-arc-extensions/terraform/modules/container-storage/main.tf new file mode 100644 index 00000000..3ed887c3 --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/modules/container-storage/main.tf @@ -0,0 +1,32 @@ +/** + * # Azure Container Storage Extension Module + * + * Deploys the Azure Container Storage (ACSA) extension for Arc-enabled Kubernetes clusters. + */ + +locals { + default_storage_class = var.container_storage_extension.fault_tolerance_enabled ? "acstor-arccontainerstorage-storage-pool" : "default,local-path" + kubernetes_storage_class = var.container_storage_extension.disk_storage_class != "" ? var.container_storage_extension.disk_storage_class : local.default_storage_class + + container_storage_settings = var.container_storage_extension.fault_tolerance_enabled ? { + "edgeStorageConfiguration.create" = "true" + "feature.diskStorageClass" = local.kubernetes_storage_class + "acstorConfiguration.create" = "true" + "acstorConfiguration.properties.diskMountPoint" = var.container_storage_extension.disk_mount_point + } : { + "edgeStorageConfiguration.create" = "true" + "feature.diskStorageClass" = local.kubernetes_storage_class + } +} + +resource "azurerm_arc_kubernetes_cluster_extension" "container_storage" { + name = "azure-arc-containerstorage" + cluster_id = var.arc_connected_cluster_id + extension_type = "microsoft.arc.containerstorage" + identity { + type = "SystemAssigned" + } + version = var.container_storage_extension.version + release_train = var.container_storage_extension.train + configuration_settings = local.container_storage_settings +} diff --git a/src/100-edge/109-arc-extensions/terraform/modules/container-storage/outputs.tf b/src/100-edge/109-arc-extensions/terraform/modules/container-storage/outputs.tf new file mode 100644 index 00000000..7f7c3cee --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/modules/container-storage/outputs.tf @@ -0,0 +1,18 @@ +output "container_storage" { + description = "Self-contained container_storage object" + value = { + enabled = var.container_storage_extension.enabled + id = azurerm_arc_kubernetes_cluster_extension.container_storage.id + name = azurerm_arc_kubernetes_cluster_extension.container_storage.name + version = var.container_storage_extension.version + train = var.container_storage_extension.train + } +} + +output "extension" { + description = "The container storage extension id and name as an object" + value = { + id = azurerm_arc_kubernetes_cluster_extension.container_storage.id + name = azurerm_arc_kubernetes_cluster_extension.container_storage.name + } +} diff --git a/src/100-edge/109-arc-extensions/terraform/modules/container-storage/variables.tf b/src/100-edge/109-arc-extensions/terraform/modules/container-storage/variables.tf new file mode 100644 index 00000000..a18680ab --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/modules/container-storage/variables.tf @@ -0,0 +1,16 @@ +variable "arc_connected_cluster_id" { + type = string + description = "The resource ID of the Arc-connected Kubernetes cluster" +} +variable "container_storage_extension" { + type = object({ + enabled = optional(bool) + version = optional(string) + train = optional(string) + auto_upgrade_minor_version = optional(bool) + disk_storage_class = optional(string) + fault_tolerance_enabled = optional(bool) + disk_mount_point = optional(string) + }) + description = "container-storage extension configuration object" +} diff --git a/src/100-edge/109-arc-extensions/terraform/modules/container-storage/versions.tf b/src/100-edge/109-arc-extensions/terraform/modules/container-storage/versions.tf new file mode 100644 index 00000000..cb47f2e9 --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/modules/container-storage/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_version = ">= 1.9.8, < 2.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + } + } +} diff --git a/src/100-edge/109-arc-extensions/terraform/outputs.tf b/src/100-edge/109-arc-extensions/terraform/outputs.tf new file mode 100644 index 00000000..10df3a72 --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/outputs.tf @@ -0,0 +1,29 @@ +output "cert_manager_extension_id" { + description = "The resource ID of the cert-manager extension." + value = try(module.cert_manager_extension[0].extension.id, null) +} + +output "cert_manager_extension_name" { + description = "The name of the cert-manager extension." + value = try(module.cert_manager_extension[0].extension.name, null) +} + +output "container_storage_extension_id" { + description = "The resource ID of the Azure Container Storage extension." + value = try(module.container_storage_extension[0].extension.id, null) +} + +output "container_storage_extension_name" { + description = "The name of the Azure Container Storage extension." + value = try(module.container_storage_extension[0].extension.name, null) +} + +output "cert_manager_extension" { + description = "Self-contained cert_manager object (id, name, enabled, version, train) or null if not deployed" + value = try(module.cert_manager_extension[0].cert_manager, null) +} + +output "container_storage_extension" { + description = "Self-contained container_storage object (id, name, enabled, version, train) or null if not deployed" + value = try(module.container_storage_extension[0].container_storage, null) +} diff --git a/src/100-edge/109-arc-extensions/terraform/variables.deps.tf b/src/100-edge/109-arc-extensions/terraform/variables.deps.tf new file mode 100644 index 00000000..9fecb437 --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/variables.deps.tf @@ -0,0 +1,8 @@ +variable "arc_connected_cluster" { + type = object({ + id = string + name = string + location = string + }) + description = "Arc-connected Kubernetes cluster object containing id, name, and location" +} diff --git a/src/100-edge/109-arc-extensions/terraform/variables.tf b/src/100-edge/109-arc-extensions/terraform/variables.tf new file mode 100644 index 00000000..86ff9e2d --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/variables.tf @@ -0,0 +1,41 @@ +variable "arc_extensions" { + type = object({ + cert_manager_extension = optional(object({ + enabled = optional(bool) + version = optional(string) + train = optional(string) + auto_upgrade_minor_version = optional(bool) + agent_operation_timeout_in_minutes = optional(number) + global_telemetry_enabled = optional(bool) + })) + container_storage_extension = optional(object({ + enabled = optional(bool) + version = optional(string) + train = optional(string) + auto_upgrade_minor_version = optional(bool) + disk_storage_class = optional(string) + fault_tolerance_enabled = optional(bool) + disk_mount_point = optional(string) + })) + }) + description = "Combined configuration object for Arc extensions (cert-manager and container storage)" + default = { + cert_manager_extension = { + enabled = true + version = "0.7.0" + train = "stable" + auto_upgrade_minor_version = false + agent_operation_timeout_in_minutes = 20 + global_telemetry_enabled = true + } + container_storage_extension = { + enabled = true + version = "2.6.0" + train = "stable" + auto_upgrade_minor_version = false + disk_storage_class = "" + fault_tolerance_enabled = false + disk_mount_point = "/mnt" + } + } +} diff --git a/src/100-edge/109-arc-extensions/terraform/versions.tf b/src/100-edge/109-arc-extensions/terraform/versions.tf new file mode 100644 index 00000000..cdcce40a --- /dev/null +++ b/src/100-edge/109-arc-extensions/terraform/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.9.8, < 2.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 4.8.0" + } + } +} diff --git a/src/100-edge/110-iot-ops/bicep/README.md b/src/100-edge/110-iot-ops/bicep/README.md index 0da7c07d..0862c5be 100644 --- a/src/100-edge/110-iot-ops/bicep/README.md +++ b/src/100-edge/110-iot-ops/bicep/README.md @@ -7,51 +7,49 @@ Deploys Azure IoT Operations extensions, instances, and configurations on Azure ## Parameters -| Name | Description | Type | Default | Required | -|:------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------|:---------| -| common | The common component configuration. | `[_2.Common](#user-defined-types)` | n/a | yes | -| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | -| containerStorageConfig | The settings for the Azure Container Store for Azure Arc Extension. | `[_1.ContainerStorageExtension](#user-defined-types)` | [variables('_1.containerStorageExtensionDefaults')] | no | -| aioCertManagerConfig | The settings for the Azure IoT Operations Platform Extension. | `[_1.AioCertManagerExtension](#user-defined-types)` | [variables('_1.aioCertManagerExtensionDefaults')] | no | -| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | [variables('_1.secretStoreExtensionDefaults')] | no | -| shouldInitAio | Whether to deploy the Azure IoT Operations initial connected cluster resources, Secret Sync, ACSA, OSM, AIO Platform. | `bool` | `true` | no | -| aioIdentityName | The name of the User Assigned Managed Identity for Azure IoT Operations. | `string` | n/a | yes | -| aioExtensionConfig | The settings for the Azure IoT Operations Extension. | `[_1.AioExtension](#user-defined-types)` | [variables('_1.aioExtensionDefaults')] | no | -| aioFeatures | AIO Instance features. | `[_1.AioFeatures](#user-defined-types)` | n/a | no | -| aioInstanceName | The name for the Azure IoT Operations Instance resource. | `string` | [format('{0}-ops-instance', parameters('arcConnectedClusterName'))] | no | -| aioDataFlowInstanceConfig | The settings for Azure IoT Operations Data Flow Instances. | `[_1.AioDataFlowInstance](#user-defined-types)` | [variables('_1.aioDataFlowInstanceDefaults')] | no | -| aioMqBrokerConfig | The settings for the Azure IoT Operations MQ Broker. | `[_1.AioMqBroker](#user-defined-types)` | [variables('_1.aioMqBrokerDefaults')] | no | -| brokerListenerAnonymousConfig | Configuration for the insecure anonymous AIO MQ Broker Listener. | `[_1.AioMqBrokerAnonymous](#user-defined-types)` | [variables('_1.aioMqBrokerAnonymousDefaults')] | no | -| configurationSettingsOverride | Optional configuration settings to override default IoT Operations extension configuration. Use the same key names as the az iot ops --ops-config parameter. | `object` | {} | no | -| schemaRegistryName | The resource name for the ADR Schema Registry for Azure IoT Operations. | `string` | n/a | yes | -| adrNamespaceName | The resource name for the ADR Namespace for Azure IoT Operations. | `string` | n/a | no | -| shouldDeployAio | Whether to deploy an Azure IoT Operations Instance and all of its required components into the connected cluster. | `bool` | `true` | no | -| shouldDeployResourceSyncRules | Whether or not to deploy the Custom Locations Resource Sync Rules for the Azure IoT Operations resources. | `bool` | `true` | no | -| shouldCreateAnonymousBrokerListener | Whether to enable an insecure anonymous AIO MQ Broker Listener. (Should only be used for dev or test environments) | `bool` | `false` | no | -| shouldEnableOtelCollector | Whether or not to enable the Open Telemetry Collector for Azure IoT Operations. | `bool` | `true` | no | -| shouldEnableOpcUaSimulator | Whether or not to enable the OPC UA Simulator for Azure IoT Operations. | `bool` | `true` | no | -| shouldEnableAkriRestConnector | Deploy Akri REST HTTP Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriMediaConnector | Deploy Akri Media Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriOnvifConnector | Deploy Akri ONVIF Connector template to the IoT Operations instance. | `bool` | `false` | no | -| shouldEnableAkriSseConnector | Deploy Akri SSE Connector template to the IoT Operations instance. | `bool` | `false` | no | -| customAkriConnectors | List of custom Akri connector templates with user-defined endpoint types and container images. | `array` | [] | no | -| akriMqttSharedConfig | Shared MQTT connection configuration for all Akri connectors. | `[_1.AkriMqttConfig](#user-defined-types)` | {'host': 'aio-broker:18883', 'audience': 'aio-internal', 'caConfigmap': 'azure-iot-operations-aio-ca-trust-bundle'} | no | -| customLocationName | The name for the Custom Locations resource. | `string` | [format('{0}-cl', parameters('arcConnectedClusterName'))] | no | -| additionalClusterExtensionIds | Additional cluster extension IDs to include in the custom location. (Appended to the default Secret Store and IoT Operations extension IDs) | `array` | [] | no | -| trustIssuerSettings | The trust issuer settings for Customer Managed Azure IoT Operations Settings. | `[_1.TrustIssuerConfig](#user-defined-types)` | {'trustSource': 'SelfSigned'} | no | -| sseKeyVaultName | The name of the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | n/a | yes | -| sseIdentityName | The name of the User Assigned Managed Identity for Secret Sync. | `string` | n/a | yes | -| sseKeyVaultResourceGroupName | The name of the Resource Group for the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | [resourceGroup().name] | no | -| shouldAssignSseKeyVaultRoles | Whether to assign roles for Key Vault to the provided Secret Sync Identity. | `bool` | `true` | no | -| shouldAssignDeployIdentityRoles | Whether to assign roles to the deploy identity. | `bool` | [not(empty(parameters('deployIdentityName')))] | no | -| deployIdentityName | The resource name for a managed identity that will be given deployment admin permissions. | `string` | n/a | no | -| shouldDeployAioDeploymentScripts | Whether to deploy DeploymentScripts for Azure IoT Operations. | `bool` | `false` | no | -| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | [parameters('sseKeyVaultName')] | no | -| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [parameters('sseKeyVaultResourceGroupName')] | no | -| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | -| deploymentScriptsSecretNamePrefix | The prefix used with constructing the secret name that will have the deployment script. | `string` | [format('{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | -| shouldAddDeployScriptsToKeyVault | Whether to add the deploy scripts for DeploymentScripts to Key Vault as secrets. (Required for DeploymentScripts) | `bool` | `false` | no | -| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | +| Name | Description | Type | Default | Required | +|:------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------|:---------| +| common | The common component configuration. | `[_2.Common](#user-defined-types)` | n/a | yes | +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | +| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | [variables('_1.secretStoreExtensionDefaults')] | no | +| shouldInitAio | Whether to deploy the Azure IoT Operations initial connected cluster resources, Secret Sync, ACSA, OSM, AIO Platform. | `bool` | `true` | no | +| aioIdentityName | The name of the User Assigned Managed Identity for Azure IoT Operations. | `string` | n/a | yes | +| aioExtensionConfig | The settings for the Azure IoT Operations Extension. | `[_1.AioExtension](#user-defined-types)` | [variables('_1.aioExtensionDefaults')] | no | +| aioFeatures | AIO Instance features. | `[_1.AioFeatures](#user-defined-types)` | n/a | no | +| aioInstanceName | The name for the Azure IoT Operations Instance resource. | `string` | [format('{0}-ops-instance', parameters('arcConnectedClusterName'))] | no | +| aioDataFlowInstanceConfig | The settings for Azure IoT Operations Data Flow Instances. | `[_1.AioDataFlowInstance](#user-defined-types)` | [variables('_1.aioDataFlowInstanceDefaults')] | no | +| aioMqBrokerConfig | The settings for the Azure IoT Operations MQ Broker. | `[_1.AioMqBroker](#user-defined-types)` | [variables('_1.aioMqBrokerDefaults')] | no | +| brokerListenerAnonymousConfig | Configuration for the insecure anonymous AIO MQ Broker Listener. | `[_1.AioMqBrokerAnonymous](#user-defined-types)` | [variables('_1.aioMqBrokerAnonymousDefaults')] | no | +| configurationSettingsOverride | Optional configuration settings to override default IoT Operations extension configuration. Use the same key names as the az iot ops --ops-config parameter. | `object` | {} | no | +| schemaRegistryName | The resource name for the ADR Schema Registry for Azure IoT Operations. | `string` | n/a | yes | +| adrNamespaceName | The resource name for the ADR Namespace for Azure IoT Operations. | `string` | n/a | no | +| shouldDeployAio | Whether to deploy an Azure IoT Operations Instance and all of its required components into the connected cluster. | `bool` | `true` | no | +| shouldDeployResourceSyncRules | Whether or not to deploy the Custom Locations Resource Sync Rules for the Azure IoT Operations resources. | `bool` | `true` | no | +| shouldCreateAnonymousBrokerListener | Whether to enable an insecure anonymous AIO MQ Broker Listener. (Should only be used for dev or test environments) | `bool` | `false` | no | +| shouldEnableOtelCollector | Whether or not to enable the Open Telemetry Collector for Azure IoT Operations. | `bool` | `true` | no | +| shouldEnableOpcUaSimulator | Whether or not to enable the OPC UA Simulator for Azure IoT Operations. | `bool` | `true` | no | +| shouldEnableAkriRestConnector | Deploy Akri REST HTTP Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriMediaConnector | Deploy Akri Media Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriOnvifConnector | Deploy Akri ONVIF Connector template to the IoT Operations instance. | `bool` | `false` | no | +| shouldEnableAkriSseConnector | Deploy Akri SSE Connector template to the IoT Operations instance. | `bool` | `false` | no | +| customAkriConnectors | List of custom Akri connector templates with user-defined endpoint types and container images. | `array` | [] | no | +| akriMqttSharedConfig | Shared MQTT connection configuration for all Akri connectors. | `[_1.AkriMqttConfig](#user-defined-types)` | {'host': 'aio-broker:18883', 'audience': 'aio-internal', 'caConfigmap': 'azure-iot-operations-aio-ca-trust-bundle'} | no | +| customLocationName | The name for the Custom Locations resource. | `string` | [format('{0}-cl', parameters('arcConnectedClusterName'))] | no | +| additionalClusterExtensionIds | Additional cluster extension IDs to include in the custom location. (Appended to the default Secret Store and IoT Operations extension IDs) | `array` | [] | no | +| trustIssuerSettings | The trust issuer settings for Customer Managed Azure IoT Operations Settings. | `[_1.TrustIssuerConfig](#user-defined-types)` | {'trustSource': 'SelfSigned'} | no | +| sseKeyVaultName | The name of the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | n/a | yes | +| sseIdentityName | The name of the User Assigned Managed Identity for Secret Sync. | `string` | n/a | yes | +| sseKeyVaultResourceGroupName | The name of the Resource Group for the Key Vault for Secret Sync. (Required when providing sseIdentityName) | `string` | [resourceGroup().name] | no | +| shouldAssignSseKeyVaultRoles | Whether to assign roles for Key Vault to the provided Secret Sync Identity. | `bool` | `true` | no | +| shouldAssignDeployIdentityRoles | Whether to assign roles to the deploy identity. | `bool` | [not(empty(parameters('deployIdentityName')))] | no | +| deployIdentityName | The resource name for a managed identity that will be given deployment admin permissions. | `string` | n/a | no | +| shouldDeployAioDeploymentScripts | Whether to deploy DeploymentScripts for Azure IoT Operations. | `bool` | `false` | no | +| deployKeyVaultName | The name of the Key Vault that will have scripts and secrets for deployment. | `string` | [parameters('sseKeyVaultName')] | no | +| deployKeyVaultResourceGroupName | The resource group name where the Key Vault is located. Defaults to the current resource group. | `string` | [parameters('sseKeyVaultResourceGroupName')] | no | +| deployUserTokenSecretName | The name for the deploy user token secret in Key Vault. | `string` | deploy-user-token | no | +| deploymentScriptsSecretNamePrefix | The prefix used with constructing the secret name that will have the deployment script. | `string` | [format('{0}-{1}-{2}', parameters('common').resourcePrefix, parameters('common').environment, parameters('common').instance)] | no | +| shouldAddDeployScriptsToKeyVault | Whether to add the deploy scripts for DeploymentScripts to Key Vault as secrets. (Required for DeploymentScripts) | `bool` | `false` | no | +| telemetry_opt_out | Whether to opt out of telemetry data collection. | `bool` | `false` | no | ## Resources @@ -75,7 +73,7 @@ Deploys Azure IoT Operations extensions, instances, and configurations on Azure | deployArcK8sRoleAssignments | Assigns required Azure Arc roles to the deployment identity for cluster access. | | deployKeyVaultRoleAssignments | Assigns required Key Vault roles to the deployment identity for script execution. | | sseKeyVaultRoleAssignments | Assigns roles for Secret Sync to access Key Vault. | -| iotOpsInit | Initializes and configures the required Arc extensions for Azure IoT Operations including Secret Store, Open Service Mesh, Container Storage, and IoT Operations Platform. | +| iotOpsInit | Initializes and configures the Secret Store extension for Azure IoT Operations. Depends on cert-manager deployed via 109-arc-extensions component. | | postInitScriptsSecrets | Creates secrets in Key Vault for deployment script setup and initialization for Azure IoT Operations. | | postInitScripts | Runs deployment scripts for IoT Operations using an Azure deploymentScript resource, including tool installation and script execution. | | iotOpsInstance | Deploys Azure IoT Operations instance, broker, authentication, listeners, and data flow components on an Azure Arc-enabled Kubernetes cluster. | @@ -160,36 +158,27 @@ Assigns roles for Secret Sync to access Key Vault. ### iotOpsInit -Initializes and configures the required Arc extensions for Azure IoT Operations including Secret Store, Open Service Mesh, Container Storage, and IoT Operations Platform. +Initializes and configures the Secret Store extension for Azure IoT Operations. Depends on cert-manager deployed via 109-arc-extensions component. #### Parameters for iotOpsInit -| Name | Description | Type | Default | Required | -|:------------------------|:------------------------------------------------------------------------------|:------------------------------------------------------|:--------|:---------| -| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | -| containerStorageConfig | The settings for the Azure Container Store for Azure Arc Extension. | `[_1.ContainerStorageExtension](#user-defined-types)` | n/a | yes | -| aioCertManagerConfig | The settings for the Azure IoT Operations Platform Extension. | `[_1.AioCertManagerExtension](#user-defined-types)` | n/a | yes | -| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | n/a | yes | -| trustIssuerSettings | The trust issuer settings for Customer Managed Azure IoT Operations Settings. | `[_1.TrustIssuerConfig](#user-defined-types)` | n/a | yes | +| Name | Description | Type | Default | Required | +|:------------------------|:-------------------------------------------------|:-------------------------------------------------|:--------|:---------| +| arcConnectedClusterName | The resource name for the Arc connected cluster. | `string` | n/a | yes | +| secretStoreConfig | The settings for the Secret Store Extension. | `[_1.SecretStoreExtension](#user-defined-types)` | n/a | yes | #### Resources for iotOpsInit -| Name | Type | API Version | -|:-----------------|:-----------------------------------------------|:------------| -| aioCertManager | `Microsoft.KubernetesConfiguration/extensions` | 2023-05-01 | -| containerStorage | `Microsoft.KubernetesConfiguration/extensions` | 2023-05-01 | -| secretStore | `Microsoft.KubernetesConfiguration/extensions` | 2023-05-01 | +| Name | Type | API Version | +|:------------|:-----------------------------------------------|:------------| +| secretStore | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | #### Outputs for iotOpsInit -| Name | Type | Description | -|:------------------------------|:---------|:-------------------------------------------------------------| -| containerStorageExtensionId | `string` | The ID of the Container Storage Extension. | -| containerStorageExtensionName | `string` | The name of the Container Storage Extension. | -| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | -| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | -| aioCertManagerExtensionId | `string` | The ID of the Azure IoT Operations Cert-Manager Extension. | -| aioCertManagerExtensionName | `string` | The name of the Azure IoT Operations Cert-Manager Extension. | +| Name | Type | Description | +|:-------------------------|:---------|:----------------------------------------| +| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | +| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | ### postInitScriptsSecrets @@ -298,7 +287,7 @@ Deploys Azure IoT Operations instance, broker, authentication, listeners, and da |:--------------------------------------|:--------------------------------------------------------------------------------|:-------------------| | sseIdentity::sseFedCred | `Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials` | 2023-01-31 | | aioIdentity::aioFedCred | `Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials` | 2023-01-31 | -| aioExtension | `Microsoft.KubernetesConfiguration/extensions` | 2023-05-01 | +| aioExtension | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | | aioExtensionSchemaRegistryContributor | `Microsoft.Authorization/roleAssignments` | 2022-04-01 | | customLocation | `Microsoft.ExtendedLocation/customLocations` | 2021-08-31-preview | | aioSyncRule | `Microsoft.ExtendedLocation/customLocations/resourceSyncRules` | 2021-08-31-preview | @@ -445,15 +434,6 @@ Configuration for Azure IoT Operations Certificate Authority. | caCertChainPem | `securestring` | The PEM-formatted CA certificate chain. | | caKeyPem | `securestring` | The PEM-formatted CA private key. | -### `_1.AioCertManagerExtension` - -The settings for the Azure IoT Operations Platform Extension. - -| Property | Type | Description | -|:---------|:------------------------------------|:---------------------------------------| -| release | `[_1.Release](#user-defined-types)` | The common settings for the extension. | -| settings | `object` | | - ### `_1.AioDataFlowInstance` The settings for Azure IoT Operations Data Flow Instances. @@ -584,15 +564,6 @@ Broker persistence configuration for disk-backed message storage. | subscriberQueue | `object` | Controls which subscriber queues should be persisted to disk. | | persistentVolumeClaimSpec | `object` | Persistent volume claim specification for storage. | -### `_1.ContainerStorageExtension` - -The settings for the Azure Container Store for Azure Arc Extension. - -| Property | Type | Description | -|:---------|:------------------------------------|:---------------------------------------| -| release | `[_1.Release](#user-defined-types)` | The common settings for the extension. | -| settings | `object` | | - ### `_1.CustomerManagedByoIssuerConfig` The configuration for Customer Managed Bring Your Own Issuer for Azure IoT Operations certificates. @@ -726,23 +697,21 @@ Common settings for the components. ## Outputs -| Name | Type | Description | -|:------------------------------|:---------|:-------------------------------------------------------------------| -| containerStorageExtensionId | `string` | The ID of the Container Storage Extension. | -| containerStorageExtensionName | `string` | The name of the Container Storage Extension. | -| aioCertManagerExtensionId | `string` | The ID of the Azure IoT Operations Cert-Manager Extension. | -| aioCertManagerExtensionName | `string` | The name of the Azure IoT Operations Cert-Manager Extension. | -| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | -| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | -| customLocationId | `string` | The ID of the deployed Custom Location. | -| customLocationName | `string` | The name of the deployed Custom Location. | -| aioInstanceId | `string` | The ID of the deployed Azure IoT Operations instance. | -| aioInstanceName | `string` | The name of the deployed Azure IoT Operations instance. | -| dataFlowProfileId | `string` | The ID of the deployed Azure IoT Operations Data Flow Profile. | -| dataFlowProfileName | `string` | The name of the deployed Azure IoT Operations Data Flow Profile. | -| dataFlowEndpointId | `string` | The ID of the deployed Azure IoT Operations Data Flow Endpoint. | -| dataFlowEndpointName | `string` | The name of the deployed Azure IoT Operations Data Flow Endpoint. | -| akriConnectorTemplates | `array` | Map of deployed Akri connector templates by name with id and type. | -| akriConnectorTypesDeployed | `array` | List of Akri connector types that were deployed. | +| Name | Type | Description | +|:---------------------------|:---------|:-------------------------------------------------------------------| +| aioPlatformExtensionId | `string` | The ID of the Azure IoT Operations Platform Extension. | +| aioPlatformExtensionName | `string` | The name of the Azure IoT Operations Platform Extension. | +| secretStoreExtensionId | `string` | The ID of the Secret Store Extension. | +| secretStoreExtensionName | `string` | The name of the Secret Store Extension. | +| customLocationId | `string` | The ID of the deployed Custom Location. | +| customLocationName | `string` | The name of the deployed Custom Location. | +| aioInstanceId | `string` | The ID of the deployed Azure IoT Operations instance. | +| aioInstanceName | `string` | The name of the deployed Azure IoT Operations instance. | +| dataFlowProfileId | `string` | The ID of the deployed Azure IoT Operations Data Flow Profile. | +| dataFlowProfileName | `string` | The name of the deployed Azure IoT Operations Data Flow Profile. | +| dataFlowEndpointId | `string` | The ID of the deployed Azure IoT Operations Data Flow Endpoint. | +| dataFlowEndpointName | `string` | The name of the deployed Azure IoT Operations Data Flow Endpoint. | +| akriConnectorTemplates | `array` | Map of deployed Akri connector templates by name with id and type. | +| akriConnectorTypesDeployed | `array` | List of Akri connector types that were deployed. | \ No newline at end of file diff --git a/src/100-edge/110-iot-ops/bicep/main.bicep b/src/100-edge/110-iot-ops/bicep/main.bicep index ac776c2b..07089eb0 100644 --- a/src/100-edge/110-iot-ops/bicep/main.bicep +++ b/src/100-edge/110-iot-ops/bicep/main.bicep @@ -18,12 +18,6 @@ param arcConnectedClusterName string Azure IoT Operations Init Parameters */ -@description('The settings for the Azure Container Store for Azure Arc Extension.') -param containerStorageConfig types.ContainerStorageExtension = types.containerStorageExtensionDefaults - -@description('The settings for the Azure IoT Operations Platform Extension.') -param aioCertManagerConfig types.AioCertManagerExtension = types.aioCertManagerExtensionDefaults - @description('The settings for the Secret Store Extension.') #disable-next-line secure-secrets-in-params param secretStoreConfig types.SecretStoreExtension = types.secretStoreExtensionDefaults @@ -288,11 +282,8 @@ module iotOpsInit 'modules/iot-ops-init.bicep' = if (shouldInitAio) { deployKeyVaultRoleAssignments ] params: { - aioCertManagerConfig: aioCertManagerConfig arcConnectedClusterName: arcConnectedClusterName - containerStorageConfig: containerStorageConfig secretStoreConfig: secretStoreConfig - trustIssuerSettings: trustIssuerSettings } } @@ -432,17 +423,11 @@ module postInstanceScripts 'modules/apply-scripts.bicep' = if (shouldDeployAioDe Outputs */ -@description('The ID of the Container Storage Extension.') -output containerStorageExtensionId string = (iotOpsInit.?outputs.?containerStorageExtensionId) ?? '' - -@description('The name of the Container Storage Extension.') -output containerStorageExtensionName string = (iotOpsInit.?outputs.?containerStorageExtensionName) ?? '' - -@description('The ID of the Azure IoT Operations Cert-Manager Extension.') -output aioCertManagerExtensionId string = (iotOpsInit.?outputs.?aioCertManagerExtensionId) ?? '' +@description('The ID of the Azure IoT Operations Platform Extension.') +output aioPlatformExtensionId string = shouldDeployAio ? (iotOpsInstance.?outputs.?aioExtensionId ?? '') : '' -@description('The name of the Azure IoT Operations Cert-Manager Extension.') -output aioCertManagerExtensionName string = (iotOpsInit.?outputs.?aioCertManagerExtensionName) ?? '' +@description('The name of the Azure IoT Operations Platform Extension.') +output aioPlatformExtensionName string = shouldDeployAio ? (iotOpsInstance.?outputs.?aioExtensionName ?? '') : '' @description('The ID of the Secret Store Extension.') output secretStoreExtensionId string = (iotOpsInit.?outputs.?secretStoreExtensionId) ?? '' diff --git a/src/100-edge/110-iot-ops/bicep/modules/iot-ops-init.bicep b/src/100-edge/110-iot-ops/bicep/modules/iot-ops-init.bicep index 1b5d1e56..fccd8a25 100644 --- a/src/100-edge/110-iot-ops/bicep/modules/iot-ops-init.bicep +++ b/src/100-edge/110-iot-ops/bicep/modules/iot-ops-init.bicep @@ -1,5 +1,5 @@ metadata name = 'IoT Operations Initialization Module' -metadata description = 'Initializes and configures the required Arc extensions for Azure IoT Operations including Secret Store, Open Service Mesh, Container Storage, and IoT Operations Platform.' +metadata description = 'Initializes and configures the Secret Store extension for Azure IoT Operations. Depends on cert-manager deployed via 109-arc-extensions component.' import * as types from '../types.bicep' @@ -10,39 +10,10 @@ import * as types from '../types.bicep' @description('The resource name for the Arc connected cluster.') param arcConnectedClusterName string -@description('The settings for the Azure Container Store for Azure Arc Extension.') -param containerStorageConfig types.ContainerStorageExtension - -@description('The settings for the Azure IoT Operations Platform Extension.') -param aioCertManagerConfig types.AioCertManagerExtension - @description('The settings for the Secret Store Extension.') #disable-next-line secure-secrets-in-params param secretStoreConfig types.SecretStoreExtension -@description('The trust issuer settings for Customer Managed Azure IoT Operations Settings.') -param trustIssuerSettings types.TrustIssuerConfig - -/* - Variables -*/ - -// Setup ACSA StorageClass based on either provided StorageClass, Fault Tolerance Enabled, or the -// default StorageClass provided by local-path. -var defaultStorageClass = containerStorageConfig.settings.faultToleranceEnabled - ? 'acstor-arccontainerstorage-storage-pool' - : 'default,local-path' - -var kubernetesStorageClass = containerStorageConfig.settings.?diskStorageClass ?? defaultStorageClass -var diskMountPoint = containerStorageConfig.settings.?diskMountPoint ?? '/mnt' - -var faultToleranceConfig = containerStorageConfig.settings.faultToleranceEnabled - ? { - 'acstorConfiguration.create': 'true' - 'acstorConfiguration.properties.diskMountPoint': diskMountPoint - } - : {} - /* Resources */ @@ -51,53 +22,7 @@ resource arcConnectedCluster 'Microsoft.Kubernetes/connectedClusters@2021-03-01' name: arcConnectedClusterName } -resource aioCertManager 'Microsoft.KubernetesConfiguration/extensions@2023-05-01' = if (trustIssuerSettings.trustSource != 'CustomerManagedByoIssuer') { - scope: arcConnectedCluster - name: 'cert-manager' - identity: { - type: 'SystemAssigned' - } - properties: { - extensionType: 'microsoft.certmanagement' - version: aioCertManagerConfig.release.version - releaseTrain: aioCertManagerConfig.release.train - autoUpgradeMinorVersion: false - scope: { - cluster: { - releaseNamespace: 'cert-manager' - } - } - configurationSettings: { - AgentOperationTimeoutInMinutes: aioCertManagerConfig.settings.agentOperationTimeoutInMinutes - 'global.telemetry.enabled': '${aioCertManagerConfig.settings.?globalTelemetryEnabled} ?? true' - } - } -} - -resource containerStorage 'Microsoft.KubernetesConfiguration/extensions@2023-05-01' = { - scope: arcConnectedCluster - // 'azure-arc-containerstorage' is the required extension name for ACSA. - name: 'azure-arc-containerstorage' - identity: { - type: 'SystemAssigned' - } - properties: { - extensionType: 'microsoft.arc.containerstorage' - autoUpgradeMinorVersion: false - version: containerStorageConfig.release.version - releaseTrain: containerStorageConfig.release.train - configurationSettings: { - 'edgeStorageConfiguration.create': 'true' - 'feature.diskStorageClass': kubernetesStorageClass - ...faultToleranceConfig - } - } - dependsOn: [ - aioCertManager - ] -} - -resource secretStore 'Microsoft.KubernetesConfiguration/extensions@2023-05-01' = { +resource secretStore 'Microsoft.KubernetesConfiguration/extensions@2024-11-01' = { scope: arcConnectedCluster // 'azure-secret-store' is the required extension name for SSE. name: 'azure-secret-store' @@ -114,29 +39,14 @@ resource secretStore 'Microsoft.KubernetesConfiguration/extensions@2023-05-01' = 'validatingAdmissionPolicies.applyPolicies': 'false' } } - dependsOn: [ - aioCertManager - ] } /* Outputs */ -@description('The ID of the Container Storage Extension.') -output containerStorageExtensionId string = containerStorage.id - -@description('The name of the Container Storage Extension.') -output containerStorageExtensionName string = containerStorage.name - @description('The ID of the Secret Store Extension.') output secretStoreExtensionId string = secretStore.id @description('The name of the Secret Store Extension.') output secretStoreExtensionName string = secretStore.name - -@description('The ID of the Azure IoT Operations Cert-Manager Extension.') -output aioCertManagerExtensionId string = aioCertManager.id - -@description('The name of the Azure IoT Operations Cert-Manager Extension.') -output aioCertManagerExtensionName string = aioCertManager.name diff --git a/src/100-edge/110-iot-ops/bicep/modules/iot-ops-instance.bicep b/src/100-edge/110-iot-ops/bicep/modules/iot-ops-instance.bicep index ca26e9ec..91a40978 100644 --- a/src/100-edge/110-iot-ops/bicep/modules/iot-ops-instance.bicep +++ b/src/100-edge/110-iot-ops/bicep/modules/iot-ops-instance.bicep @@ -157,7 +157,7 @@ resource arcConnectedCluster 'Microsoft.Kubernetes/connectedClusters@2024-12-01- name: arcConnectedClusterName } -resource aioExtension 'Microsoft.KubernetesConfiguration/extensions@2023-05-01' = { +resource aioExtension 'Microsoft.KubernetesConfiguration/extensions@2024-11-01' = { scope: arcConnectedCluster name: 'azure-iot-operations-${take(uniqueString(arcConnectedCluster.id), 5)}' identity: { diff --git a/src/100-edge/110-iot-ops/bicep/types.bicep b/src/100-edge/110-iot-ops/bicep/types.bicep index f3256769..8a49929f 100644 --- a/src/100-edge/110-iot-ops/bicep/types.bicep +++ b/src/100-edge/110-iot-ops/bicep/types.bicep @@ -1,9 +1,10 @@ /* - * IMPORTANT: The variable names in this file ('aioCertManagerExtensionDefaults', - * 'secretStoreExtensionDefaults', 'containerStorageExtensionDefaults', + * IMPORTANT: The variable names in this file ('secretStoreExtensionDefaults', * 'aioExtensionDefaults') are explicitly referenced * by the aio-version-checker.py script. If you rename these variables or change their structure, * you must also update the script and the aio-version-checker-template.yml pipeline. + * NOTE: 'aioCertManagerExtensionDefaults' and 'containerStorageExtensionDefaults' have been + * moved to the 109-arc-extensions component. */ @export() @@ -31,61 +32,6 @@ var secretStoreExtensionDefaults = { } } -@export() -@description('The settings for the Azure Container Store for Azure Arc Extension.') -type ContainerStorageExtension = { - @description('The common settings for the extension.') - release: Release - - settings: { - @description('Whether or not to enable fault tolerant storage in the cluster.') - faultToleranceEnabled: bool - - @description('The storage class for the cluster. (Otherwise, "acstor-arccontainerstorage-storage-pool" for fault tolerant storage else "default,local-path")') - diskStorageClass: string? - - @description('The disk mount point for the cluster when using fault tolerant storage. (Otherwise: "/mnt" when using fault tolerant storage)') - diskMountPoint: string? - } -} - -@export() -var containerStorageExtensionDefaults = { - release: { - version: '2.6.0' - train: 'stable' - } - settings: { - faultToleranceEnabled: false - } -} - -@export() -@description('The settings for the Azure IoT Operations Platform Extension.') -type AioCertManagerExtension = { - @description('The common settings for the extension.') - release: Release - - settings: { - @description('Agent operation timeout in minutes.') - agentOperationTimeoutInMinutes: string - @description('Enable or disable global telemetry.') - globalTelemetryEnabled: bool? - } -} - -@export() -var aioCertManagerExtensionDefaults = { - release: { - version: '0.7.0' - train: 'stable' - } - settings: { - agentOperationTimeoutInMinutes: '20' - globalTelemetryEnabled: true - } -} - @export() @description('The settings for the Azure IoT Operations Extension.') type AioExtension = { diff --git a/src/100-edge/110-iot-ops/scripts/init-scripts.sh b/src/100-edge/110-iot-ops/scripts/init-scripts.sh index 64f8d9f3..fa61c37c 100755 --- a/src/100-edge/110-iot-ops/scripts/init-scripts.sh +++ b/src/100-edge/110-iot-ops/scripts/init-scripts.sh @@ -1,14 +1,74 @@ #!/usr/bin/env bash +# Azure Arc-enabled Kubernetes Proxy Initialization Script +# +# This script establishes connectivity to an Azure Arc-enabled Kubernetes cluster +# by managing the az connectedk8s proxy lifecycle and ensuring the AIO namespace exists. +# +# Key Features: +# - Detects existing kubectl connectivity before starting a new proxy +# - Manages az connectedk8s proxy with proper cleanup on exit/interrupt +# - Implements race condition fix for kubeconfig file creation +# - Creates AIO namespace if not present +# +# Race Condition Fix: +# The az connectedk8s proxy writes kubeconfig asynchronously in the background. +# To prevent partial/incomplete reads, the proxy writes to a temporary file first. +# A wrapper process monitors the temp file, waits for it to be complete, then +# atomically moves it to the final location. This ensures the kubeconfig file +# only appears when fully written and ready for use. +# +# Required Environment Variables: +# - TF_CONNECTED_CLUSTER_NAME: Name of the Arc-enabled Kubernetes cluster +# - TF_RESOURCE_GROUP_NAME: Azure resource group containing the cluster +# - TF_AIO_NAMESPACE: Namespace to create/ensure exists +# - TF_MODULE_PATH: Path to module containing YAML resources +# +# Optional Environment Variables: +# - DEPLOY_USER_TOKEN_SECRET: Key Vault secret name for deploy token +# - DEPLOY_KEY_VAULT_NAME: Key Vault name for deploy token retrieval +# (both must be set together if using token-based authentication) + # Set error handling to continue on errors set +e +# Validate required environment variables +required_vars=( + "TF_CONNECTED_CLUSTER_NAME" + "TF_RESOURCE_GROUP_NAME" + "TF_AIO_NAMESPACE" + "TF_MODULE_PATH" +) + +missing_vars=() +for var in "${required_vars[@]}"; do + if [[ -z "${!var}" ]]; then + missing_vars+=("$var") + fi +done + +if [ ${#missing_vars[@]} -gt 0 ]; then + echo "ERROR: Required environment variables not set:" >&2 + printf " - %s\n" "${missing_vars[@]}" >&2 + exit 1 +fi + +# Validate optional token variables are both set or both unset +if [[ -n "${DEPLOY_USER_TOKEN_SECRET}" && -z "${DEPLOY_KEY_VAULT_NAME}" ]]; then + echo "ERROR: DEPLOY_USER_TOKEN_SECRET is set but DEPLOY_KEY_VAULT_NAME is not" >&2 + exit 1 +elif [[ -z "${DEPLOY_USER_TOKEN_SECRET}" && -n "${DEPLOY_KEY_VAULT_NAME}" ]]; then + echo "ERROR: DEPLOY_KEY_VAULT_NAME is set but DEPLOY_USER_TOKEN_SECRET is not" >&2 + exit 1 +fi + # Function to clean up resources cleanup() { local exit_code=$? echo "Cleaning up..." [ -f "$kube_config_file" ] && rm "$kube_config_file" && echo "Deleted kubeconfig file" + [ -f "${kube_config_temp:-}" ] && rm "$kube_config_temp" && echo "Deleted temporary kubeconfig file" # Kill the proxy process group if [[ ${proxy_pid:-} ]]; then @@ -50,7 +110,13 @@ cleanup() { } check_connected_to_cluster() { - if connected_to_cluster=$(kubectl get cm azure-clusterconfig -n azure-arc -o jsonpath="{.data.AZURE_RESOURCE_NAME}" --kubeconfig "$kube_config_file" 2>/dev/null); then + # Check if kubeconfig file exists and has already been populated by az connectedk8s proxy running in background + if [[ ! -s "$kube_config_file" ]]; then + return 1 + fi + + # Verify connectivity and cluster identity + if connected_to_cluster=$(kubectl get cm azure-clusterconfig -n azure-arc -o jsonpath="{.data.AZURE_RESOURCE_NAME}" --kubeconfig "$kube_config_file" --request-timeout=10s 2>/dev/null); then if [ "$connected_to_cluster" == "$TF_CONNECTED_CLUSTER_NAME" ]; then return 0 fi @@ -60,14 +126,24 @@ check_connected_to_cluster() { start_proxy() { # Use a custom kubeconfig file to ensure the current user's context is not affected - kube_config_file=$(mktemp -t "${TF_CONNECTED_CLUSTER_NAME}.XXX") - # Start proxy in its own process group with -m - set -m + if ! kube_config_file=$(mktemp -t "${TF_CONNECTED_CLUSTER_NAME}.XXX"); then + echo "ERROR: Failed to create temporary kubeconfig file" >&2 + exit 1 + fi + + # Race condition fix: az connectedk8s proxy writes to temp file first, then atomically moved to final location + # This ensures kubeconfig file only has non-empty content when fully written, avoiding partial/incomplete reads + if ! kube_config_temp=$(mktemp -t "${TF_CONNECTED_CLUSTER_NAME}.temp.XXX"); then + echo "ERROR: Failed to create secondary temporary kubeconfig file" >&2 + exit 1 + fi + + # Build proxy arguments local -a proxy_args=( "-n" "$TF_CONNECTED_CLUSTER_NAME" "-g" "$TF_RESOURCE_GROUP_NAME" "--port" "9800" - "--file" "$kube_config_file" + "--file" "$kube_config_temp" ) local deploy_user_token="" if [[ $DEPLOY_USER_TOKEN_SECRET ]]; then @@ -83,7 +159,51 @@ start_proxy() { echo "Got Deploy User Token..." proxy_args+=("--token" "$deploy_user_token") fi - az connectedk8s proxy "${proxy_args[@]}" >/dev/null & + + # Start proxy wrapper in its own process group + set -m + { + # Start az connectedk8s proxy writing to temp file + az connectedk8s proxy "${proxy_args[@]}" >/dev/null & + az_pid=$! + + # Wait for temp file to have content + local wait_count=0 + while [[ ! -s "$kube_config_temp" ]]; do + if ! kill -0 "$az_pid" 2>/dev/null; then + echo "ERROR: az connectedk8s proxy exited unexpectedly" >&2 + kill "$az_pid" 2>/dev/null + # Signal parent to trigger cleanup and exit + kill -INT $$ 2>/dev/null + exit 1 + fi + sleep 0.5 + ((wait_count += 1)) + if [ "$wait_count" -gt 60 ]; then + echo "ERROR: timeout waiting for kubeconfig file creation" >&2 + kill "$az_pid" 2>/dev/null + # Signal parent to trigger cleanup and exit + kill -INT $$ 2>/dev/null + exit 1 + fi + done + + # Give az connectedk8s proxy time to finish writing + sleep 2 + + # Atomically move temp file to final location + if ! mv "$kube_config_temp" "$kube_config_file"; then + echo "ERROR: Failed to move kubeconfig file from temp location" >&2 + kill "$az_pid" 2>/dev/null + # Signal parent to trigger cleanup and exit + kill -INT $$ 2>/dev/null + exit 1 + fi + + # Keep az proxy running in foreground of this subshell + wait "$az_pid" || exit 1 + } & + export proxy_pid=$! proxy_pgid=$(ps -o pgid= -p "$proxy_pid" 2>/dev/null | tr -d ' ') if [[ ! $proxy_pgid ]]; then @@ -96,6 +216,10 @@ start_proxy() { timeout=0 until check_connected_to_cluster; do + if ! kill -0 "$proxy_pid" 2>/dev/null; then + echo "ERROR: az connectedk8s proxy wrapper exited unexpectedly" >&2 + return 1 + fi sleep 1 ((timeout += 1)) if [ "$timeout" -gt 30 ]; then @@ -121,14 +245,21 @@ else echo "Starting 'az connectedk8s proxy'" - start_proxy + start_proxy || exit 1 fi # Ensure aio namespace is created and exists -if ! kubectl get namespace "$TF_AIO_NAMESPACE" --kubeconfig "$kube_config_file"; then +if ! kubectl get namespace "$TF_AIO_NAMESPACE" --kubeconfig "$kube_config_file" &>/dev/null; then + echo "Namespace $TF_AIO_NAMESPACE not found, attempting to create..." + timeout=0 until envsubst <"$TF_MODULE_PATH/yaml/aio-namespace.yaml" | kubectl apply -f - --kubeconfig "$kube_config_file"; do - echo "Error applying aio-namespace.yaml, retrying in 5 seconds" + echo "Error applying aio-namespace.yaml, retrying in 5 seconds..." sleep 5 + ((timeout += 5)) + if [ "$timeout" -gt 60 ]; then + echo "ERROR: timed out creating namespace $TF_AIO_NAMESPACE" >&2 + exit 1 + fi done fi diff --git a/src/100-edge/110-iot-ops/terraform/README.md b/src/100-edge/110-iot-ops/terraform/README.md index a878654d..93d1c777 100644 --- a/src/100-edge/110-iot-ops/terraform/README.md +++ b/src/100-edge/110-iot-ops/terraform/README.md @@ -40,15 +40,12 @@ Instance can be created, and after. | secret\_sync\_key\_vault | Azure Key Vault ID to use with Secret Sync Extension. | ```object({ name = string id = string })``` | n/a | yes | | additional\_cluster\_extension\_ids | Additional cluster extension IDs to include in the custom location. Appended to the default Secret Store and IoT Operations extension IDs | `list(string)` | `[]` | no | | aio\_ca | CA certificate for the MQTT broker, can be either Root CA or Root CA with any number of Intermediate CAs. If not provided, a self-signed Root CA with a intermediate will be generated. Only valid when Trust Source is set to CustomerManaged | ```object({ root_ca_cert_pem = string ca_cert_chain_pem = string ca_key_pem = string })``` | `null` | no | -| aio\_cert\_manager\_config | Install cert-manager | ```object({ agent_operation_timeout_in_minutes = string global_telemetry_enabled = bool })``` | ```{ "agent_operation_timeout_in_minutes": "20", "global_telemetry_enabled": true }``` | no | | aio\_features | AIO Instance features with mode ('Stable', 'Preview', 'Disabled') and settings ('Enabled', 'Disabled'). | ```map(object({ mode = optional(string) settings = optional(map(string)) }))``` | `null` | no | | broker\_listener\_anonymous\_config | Configuration for the insecure anonymous AIO MQ Broker Listener. For additional information, refer to: | ```object({ serviceName = string port = number nodePort = number })``` | ```{ "nodePort": 31884, "port": 18884, "serviceName": "aio-broker-anon" }``` | no | | byo\_issuer\_trust\_settings | Settings for CustomerManagedByoIssuer (Bring Your Own Issuer) trust configuration | ```object({ issuer_name = string issuer_kind = string configmap_name = string configmap_key = string })``` | `null` | no | -| cert\_manager | n/a | ```object({ version = string train = string })``` | ```{ "train": "stable", "version": "0.7.0" }``` | no | | configuration\_settings\_override | Optional configuration settings to override default IoT Operations extension configuration. Use the same key names as the az iot ops --ops-config parameter. | `map(string)` | `{}` | no | | custom\_akri\_connectors | List of custom Akri connector templates with user-defined endpoint types and container images. Supports built-in types (rest, media, onvif, sse) or custom types with custom\_endpoint\_type and custom\_image\_name. Built-in connectors default to mcr.microsoft.com/azureiotoperations/akri-connectors/connector\_type:0.5.1. Examples: # ONVIF Camera Connector (Built-in) custom\_akri\_connectors = [ { name = "warehouse-camera-connector" type = "onvif" replicas = 2 log\_level = "info" } ] # SSE Event Connector (Built-in) custom\_akri\_connectors = [ { name = "analytics-camera-connector" type = "sse" replicas = 1 log\_level = "info" } ] # REST API Connector (Built-in) custom\_akri\_connectors = [ { name = "sensor-api-connector" type = "rest" replicas = 1 log\_level = "info" } ] # Custom Modbus Connector custom\_akri\_connectors = [ { name = "modbus-telemetry-connector" type = "custom" custom\_endpoint\_type = "Contoso.Modbus" custom\_image\_name = "my\_acr.azurecr.io/modbus-telemetry-connector" custom\_endpoint\_version = "2.0" registry = "my\_acr.azurecr.io" image\_tag = "v1.2.3" replicas = 2 log\_level = "debug" } ] # Multiple Connectors with MQTT Override custom\_akri\_connectors = [ { name = "warehouse-ptz-cameras" type = "onvif" replicas = 3 log\_level = "info" mqtt\_config = { host = "aio-broker.azure-iot-operations" audience = "aio-broker" ca\_configmap = "aio-ca-trust-bundle" keep\_alive\_seconds = 60 max\_inflight\_messages = 100 session\_expiry\_seconds = 600 } }, { name = "analytics-event-stream" type = "sse" replicas = 2 log\_level = "debug" } ] | ```list(object({ name = string type = string // "rest", "media", "onvif", "sse", "custom" // Custom Connector Fields (required when type = "custom") custom_endpoint_type = optional(string) // e.g., "Contoso.Modbus", "Acme.CustomProtocol" custom_image_name = optional(string) // e.g., "my_acr.azurecr.io/custom-connector" custom_endpoint_version = optional(string, "1.0") // Runtime Configuration (defaults applied based on connector type) registry = optional(string) // Defaults: mcr.microsoft.com for built-in types image_tag = optional(string) // Defaults: 0.5.1 for built-in types, latest for custom replicas = optional(number, 1) image_pull_policy = optional(string) // Default: IfNotPresent // Diagnostics log_level = optional(string) // Default: info (lowercase: trace, debug, info, warning, error, critical) // MQTT Override (uses shared config if not provided) mqtt_config = optional(object({ host = string audience = string ca_configmap = string keep_alive_seconds = optional(number, 60) max_inflight_messages = optional(number, 100) session_expiry_seconds = optional(number, 600) })) // Optional Advanced Fields aio_min_version = optional(string, "1.2.37") aio_max_version = optional(string) allocation = optional(object({ policy = string // "Bucketized" bucket_size = number // 1-100 })) additional_configuration = optional(map(string)) secrets = optional(list(object({ secret_alias = string secret_key = string secret_ref = string }))) trust_settings = optional(object({ trust_list_secret_ref = string })) }))``` | `[]` | no | | dataflow\_instance\_count | Number of dataflow instances. Defaults to 1. | `number` | `1` | no | -| edge\_storage\_accelerator | n/a | ```object({ version = string train = string diskStorageClass = string faultToleranceEnabled = bool diskMountPoint = string })``` | ```{ "diskMountPoint": "/mnt", "diskStorageClass": "", "faultToleranceEnabled": false, "train": "stable", "version": "2.6.0" }``` | no | | enable\_instance\_secret\_sync | Whether to enable secret sync on the Azure IoT Operations instance | `bool` | `true` | no | | enable\_opc\_ua\_simulator | Deploy OPC UA Simulator to the cluster | `bool` | `true` | no | | mqtt\_broker\_config | n/a | ```object({ brokerListenerServiceName = string brokerListenerPort = number serviceAccountAudience = string frontendReplicas = number frontendWorkers = number backendRedundancyFactor = number backendWorkers = number backendPartitions = number memoryProfile = string serviceType = string logsLevel = optional(string, "info") })``` | ```{ "backendPartitions": 2, "backendRedundancyFactor": 2, "backendWorkers": 2, "brokerListenerPort": 18883, "brokerListenerServiceName": "aio-broker", "frontendReplicas": 2, "frontendWorkers": 2, "logsLevel": "info", "memoryProfile": "Medium", "serviceAccountAudience": "aio-internal", "serviceType": "ClusterIp" }``` | no | diff --git a/src/100-edge/110-iot-ops/terraform/main.tf b/src/100-edge/110-iot-ops/terraform/main.tf index 5622a579..8255b86a 100644 --- a/src/100-edge/110-iot-ops/terraform/main.tf +++ b/src/100-edge/110-iot-ops/terraform/main.tf @@ -65,10 +65,6 @@ module "iot_ops_init" { depends_on = [module.role_assignments] arc_connected_cluster_id = var.arc_connected_cluster.id - trust_config_source = var.trust_config_source - aio_cert_manager_config = var.aio_cert_manager_config - cert_manager = var.cert_manager - edge_storage_accelerator = var.edge_storage_accelerator secret_sync_controller = var.secret_sync_controller resource_group = var.resource_group connected_cluster_name = var.arc_connected_cluster.name diff --git a/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/README.md b/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/README.md index af0314d4..ab0ffc71 100644 --- a/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/README.md +++ b/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/README.md @@ -20,8 +20,6 @@ Deploys resources necessary to enable Azure IoT Operations (AIO) and creates an | Name | Type | |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------| -| [azurerm_arc_kubernetes_cluster_extension.cert_manager](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/arc_kubernetes_cluster_extension) | resource | -| [azurerm_arc_kubernetes_cluster_extension.container_storage](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/arc_kubernetes_cluster_extension) | resource | | [azurerm_arc_kubernetes_cluster_extension.secret_store](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/arc_kubernetes_cluster_extension) | resource | | [azurerm_federated_identity_credential.federated_identity_cred_aio_instance](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/federated_identity_credential) | resource | | [azurerm_federated_identity_credential.federated_identity_cred_sse_aio](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/federated_identity_credential) | resource | @@ -29,20 +27,16 @@ Deploys resources necessary to enable Azure IoT Operations (AIO) and creates an ## Inputs -| Name | Description | Type | Default | Required | -|----------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------|---------|:--------:| -| aio\_cert\_manager\_config | Install cert-manager and trust-manager extensions | ```object({ agent_operation_timeout_in_minutes = string global_telemetry_enabled = bool })``` | n/a | yes | -| aio\_namespace | Azure IoT Operations namespace | `string` | n/a | yes | -| aio\_user\_managed\_identity\_id | ID of the User Assigned Managed Identity for the Azure IoT Operations instance | `string` | n/a | yes | -| arc\_connected\_cluster\_id | The resource ID of the connected cluster to deploy Azure IoT Operations Platform to | `string` | n/a | yes | -| cert\_manager | n/a | ```object({ version = string train = string })``` | n/a | yes | -| connected\_cluster\_name | The name of the connected cluster to deploy Azure IoT Operations to | `string` | n/a | yes | -| edge\_storage\_accelerator | n/a | ```object({ version = string train = string diskStorageClass = string faultToleranceEnabled = bool diskMountPoint = string })``` | n/a | yes | -| enable\_instance\_secret\_sync | Whether to enable secret sync on the Azure IoT Operations instance | `bool` | n/a | yes | -| resource\_group | Resource group object containing name and id where resources will be deployed | ```object({ id = string name = string })``` | n/a | yes | -| secret\_sync\_controller | n/a | ```object({ version = string train = string })``` | n/a | yes | -| secret\_sync\_identity | Secret Sync Extension user managed identity id and client id | ```object({ id = string client_id = string })``` | n/a | yes | -| trust\_config\_source | TrustConfig source must be one of 'SelfSigned', 'CustomerManagedByoIssuer' or 'CustomerManagedGenerateIssuer'. Defaults to SelfSigned. When choosing CustomerManagedGenerateIssuer, ensure connectedk8s proxy is enabled on the cluster for current user. When choosing CustomerManagedByoIssuer, ensure an Issuer and ConfigMap resources exist in the cluster. | `string` | n/a | yes | +| Name | Description | Type | Default | Required | +|----------------------------------|-------------------------------------------------------------------------------------|---------------------------------------------------|---------|:--------:| +| aio\_namespace | Azure IoT Operations namespace | `string` | n/a | yes | +| aio\_user\_managed\_identity\_id | ID of the User Assigned Managed Identity for the Azure IoT Operations instance | `string` | n/a | yes | +| arc\_connected\_cluster\_id | The resource ID of the connected cluster to deploy Azure IoT Operations Platform to | `string` | n/a | yes | +| connected\_cluster\_name | The name of the connected cluster to deploy Azure IoT Operations to | `string` | n/a | yes | +| enable\_instance\_secret\_sync | Whether to enable secret sync on the Azure IoT Operations instance | `bool` | n/a | yes | +| resource\_group | Resource group object containing name and id where resources will be deployed | ```object({ id = string name = string })``` | n/a | yes | +| secret\_sync\_controller | n/a | ```object({ version = string train = string })``` | n/a | yes | +| secret\_sync\_identity | Secret Sync Extension user managed identity id and client id | ```object({ id = string client_id = string })``` | n/a | yes | ## Outputs diff --git a/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/main.tf b/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/main.tf index 9c876937..23421fcd 100644 --- a/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/main.tf +++ b/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/main.tf @@ -5,22 +5,6 @@ * */ -locals { - default_storage_class = var.edge_storage_accelerator.faultToleranceEnabled ? "acstor-arccontainerstorage-storage-pool" : "default,local-path" - kubernetes_storage_class = var.edge_storage_accelerator.diskStorageClass != "" ? var.edge_storage_accelerator.diskStorageClass : local.default_storage_class - diskMountPoint = coalesce(var.edge_storage_accelerator.diskMountPoint, "/mnt") - - container_storage_settings = var.edge_storage_accelerator.faultToleranceEnabled ? { - "edgeStorageConfiguration.create" = "true" - "feature.diskStorageClass" = local.kubernetes_storage_class - "acstorConfiguration.create" = "true" - "acstorConfiguration.properties.diskMountPoint" = local.diskMountPoint - } : { - "edgeStorageConfiguration.create" = "true" - "feature.diskStorageClass" = local.kubernetes_storage_class - } -} - data "azapi_resource" "cluster_oidc_issuer" { name = var.connected_cluster_name parent_id = var.resource_group.id @@ -42,37 +26,6 @@ resource "azurerm_arc_kubernetes_cluster_extension" "secret_store" { "rotationPollIntervalInSeconds" = "120" "validatingAdmissionPolicies.applyPolicies" = "false" } - depends_on = [azurerm_arc_kubernetes_cluster_extension.cert_manager] -} - -resource "azurerm_arc_kubernetes_cluster_extension" "container_storage" { - name = "azure-arc-containerstorage" - cluster_id = var.arc_connected_cluster_id - extension_type = "microsoft.arc.containerstorage" - identity { - type = "SystemAssigned" - } - version = var.edge_storage_accelerator.version - release_train = var.edge_storage_accelerator.train - configuration_settings = local.container_storage_settings - depends_on = [azurerm_arc_kubernetes_cluster_extension.cert_manager] -} - -resource "azurerm_arc_kubernetes_cluster_extension" "cert_manager" { - count = var.trust_config_source != "CustomerManagedByoIssuer" ? 1 : 0 - name = "cert-manager" - cluster_id = var.arc_connected_cluster_id - extension_type = "microsoft.certmanagement" - identity { - type = "SystemAssigned" - } - version = var.cert_manager.version - release_train = var.cert_manager.train - release_namespace = "cert-manager" - configuration_settings = { - "AgentOperationTimeoutInMinutes" = var.aio_cert_manager_config.agent_operation_timeout_in_minutes - "global.telemetry.enabled" = var.aio_cert_manager_config.global_telemetry_enabled - } } resource "azurerm_federated_identity_credential" "federated_identity_cred_sse_aio" { diff --git a/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/variables.tf b/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/variables.tf index 7ef6acaa..e5c64c97 100644 --- a/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/variables.tf +++ b/src/100-edge/110-iot-ops/terraform/modules/iot-ops-init/variables.tf @@ -3,26 +3,6 @@ variable "arc_connected_cluster_id" { description = "The resource ID of the connected cluster to deploy Azure IoT Operations Platform to" } -variable "trust_config_source" { - type = string - description = "TrustConfig source must be one of 'SelfSigned', 'CustomerManagedByoIssuer' or 'CustomerManagedGenerateIssuer'. Defaults to SelfSigned. When choosing CustomerManagedGenerateIssuer, ensure connectedk8s proxy is enabled on the cluster for current user. When choosing CustomerManagedByoIssuer, ensure an Issuer and ConfigMap resources exist in the cluster." -} - -variable "aio_cert_manager_config" { - type = object({ - agent_operation_timeout_in_minutes = string - global_telemetry_enabled = bool - }) - description = "Install cert-manager and trust-manager extensions" -} - -variable "cert_manager" { - type = object({ - version = string - train = string - }) -} - variable "secret_sync_controller" { type = object({ version = string @@ -30,16 +10,6 @@ variable "secret_sync_controller" { }) } -variable "edge_storage_accelerator" { - type = object({ - version = string - train = string - diskStorageClass = string - faultToleranceEnabled = bool - diskMountPoint = string - }) -} - variable "resource_group" { type = object({ id = string diff --git a/src/100-edge/110-iot-ops/terraform/variables.init.tf b/src/100-edge/110-iot-ops/terraform/variables.init.tf index 7094dd69..c1ea5cc2 100644 --- a/src/100-edge/110-iot-ops/terraform/variables.init.tf +++ b/src/100-edge/110-iot-ops/terraform/variables.init.tf @@ -1,40 +1,12 @@ /* * Optional Variables * - * IMPORTANT: The variable names in this file ('cert_manager', 'secret_sync_controller', - * 'edge_storage_accelerator') are explicitly referenced by the - * aio-version-checker.py script. If you rename these variables or change their structure, + * IMPORTANT: The variable names in this file ('secret_sync_controller') + * are explicitly referenced by the aio-version-checker.py script. + * If you rename these variables or change their structure, * you must also update the script and the aio-version-checker-template.yml pipeline. */ -variable "cert_manager" { - type = object({ - version = string - train = string - }) - default = { - version = "0.7.0" - train = "stable" - } -} - -variable "edge_storage_accelerator" { - type = object({ - version = string - train = string - diskStorageClass = string - faultToleranceEnabled = bool - diskMountPoint = string - }) - default = { - version = "2.6.0" - train = "stable" - diskStorageClass = "" - faultToleranceEnabled = false - diskMountPoint = "/mnt" - } -} - variable "secret_sync_controller" { type = object({ version = string diff --git a/src/100-edge/110-iot-ops/terraform/variables.tf b/src/100-edge/110-iot-ops/terraform/variables.tf index 2bc538b7..2ee7297f 100644 --- a/src/100-edge/110-iot-ops/terraform/variables.tf +++ b/src/100-edge/110-iot-ops/terraform/variables.tf @@ -53,18 +53,6 @@ variable "aio_ca" { description = "CA certificate for the MQTT broker, can be either Root CA or Root CA with any number of Intermediate CAs. If not provided, a self-signed Root CA with a intermediate will be generated. Only valid when Trust Source is set to CustomerManaged" } -variable "aio_cert_manager_config" { - type = object({ - agent_operation_timeout_in_minutes = string - global_telemetry_enabled = bool - }) - default = { - agent_operation_timeout_in_minutes = "20" - global_telemetry_enabled = true - } - description = "Install cert-manager" -} - variable "enable_opc_ua_simulator" { type = bool default = true diff --git a/src/100-edge/120-observability/bicep/README.md b/src/100-edge/120-observability/bicep/README.md index 37e79b73..c209aec9 100644 --- a/src/100-edge/120-observability/bicep/README.md +++ b/src/100-edge/120-observability/bicep/README.md @@ -53,8 +53,8 @@ Creates the cluster extensions required to expose cluster and container metrics. | Name | Type | API Version | |:------------------------|:-----------------------------------------------|:------------| -| azuremonitor-metrics | `Microsoft.KubernetesConfiguration/extensions` | 2023-05-01 | -| azuremonitor-containers | `Microsoft.KubernetesConfiguration/extensions` | 2023-05-01 | +| azuremonitor-metrics | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | +| azuremonitor-containers | `Microsoft.KubernetesConfiguration/extensions` | 2024-11-01 | #### Outputs for clusterExtensionsObs diff --git a/src/100-edge/120-observability/bicep/modules/cluster-extensions-obs.bicep b/src/100-edge/120-observability/bicep/modules/cluster-extensions-obs.bicep index e72d73a1..51ca5d40 100644 --- a/src/100-edge/120-observability/bicep/modules/cluster-extensions-obs.bicep +++ b/src/100-edge/120-observability/bicep/modules/cluster-extensions-obs.bicep @@ -48,7 +48,7 @@ resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09 scope: resourceGroup(logAnalyticsWorkspaceResourceGroupName) } -resource containerMetricsExtension 'Microsoft.KubernetesConfiguration/extensions@2023-05-01' = { +resource containerMetricsExtension 'Microsoft.KubernetesConfiguration/extensions@2024-11-01' = { scope: arcConnectedCluster name: 'azuremonitor-metrics' properties: { @@ -66,7 +66,7 @@ resource containerMetricsExtension 'Microsoft.KubernetesConfiguration/extensions } } -resource containerLogsExtension 'Microsoft.KubernetesConfiguration/extensions@2023-05-01' = { +resource containerLogsExtension 'Microsoft.KubernetesConfiguration/extensions@2024-11-01' = { scope: arcConnectedCluster name: 'azuremonitor-containers' properties: {