I created Application Gateway & then AKS cluster using C# (Pulumi Azure Native). Resources are created successfully (code is given below). While creating AKS, I enabled AGIC add-on with AddonProfiles = { //... }

AddonProfiles = { 
  "ingressApplicationGateway", new ManagedClusterAddonProfileArgs { //... } 
}

I am trying to achieve Fanout Ingress with AGIC add-on (apps will run in AKS and will be served by Application Gateway). The problem is ingress is not working unless I manually set Contributor role to AGIC add-on managed identity at scope 'ApplicationGatewayResourceId'.

So far, I did the followings:

1. Not working: created an ingress (see Kubernetes manifest demo-api.yaml below). But ingress is not working

2. Working but requires manual intervention:

   • a managed identity "ingressapplicationgateway-xxx" is created automatically (in node resource group "MC_xxx") and used by AGIC add-on
   • I assigned Contributor role to ingressapplicationgateway-xxx managed identity at scope 'ApplicationGatewayResourceId' (using Azure portal)
   • created Kubernetes ingress again and this time it works
   • just to be confirmed, I removed Contributor role and re-created ingress -> ingress did not work
   • So, ingressapplicationgateway-xxx requires Contributor role which I can easily assign using portal/cli/PowerShell, but that's not what I want because I am implementing Infrastructure-as-Code using Pulumi C#

3. Bring your own identity is not applicable for AGIC add-on: I thought I will create Managed Identity, assign role and then use it for AGIC add-on, but according to Microsoft Documentation, "Bring your own identity" is only supported for API server and Kubelet identity (not possible for AGIC add-on)

4. I tried to get AGIC add-on Managed Identity ingressapplicationgateway-xxx (that belongs to node resource group MC_xxx) but failed to do so and asked question here (no answer yet).

Kubernetes manifest for ingress: demo-api.yaml

apiVersion: v1
kind: Service
metadata:
  name: demo-api 
  namespace: default
spec:
  type: ClusterIP
  ports:
 3. port: 80
  selector:
    app: demo-api
---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: demo-api
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: demo-api
  template:
    metadata:
      labels:
        app: demo-api
    spec:
      containers:
      - name: demo-api
        image: hovermind.azurecr.io/demoapi:latest
        ports:
        - containerPort: 80
---

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: demo-ingress
  namespace: default
  annotations:
    kubernetes.io/ingress.class: azure/application-gateway
    appgw.ingress.kubernetes.io/backend-path-prefix: "/api/" # API has url prefix -> [Route("api")]
spec:
  rules:
  - host: agic-appgw-public-ip.japaneast.cloudapp.azure.com
    http:
      paths:
      - path: /api*
        pathType: Prefix
        backend:
          service:
            name: demo-api
            port:
              number: 80

C# Code for Application Gateway

// ... ... ...

var hubAppGwSubnet = new Subnet($"{HubVirtualNetwork}.{ApplicationGatewaySubnet}", new AzureNative.Network.SubnetArgs {
    // ... ... ...
}, new CustomResourceOptions { DependsOn = { hubVnet } });


//
// Create Public IP for App Gateway
//
var agicAppGwPublicIp = new PublicIPAddress(AgicApplicationGatewayPublicIp, new AzureNative.Network.PublicIPAddressArgs {
    // ... ... ...
});


//
// Application Gateway configs from stack settings file
//
var agicAppGwArgs = config.RequireObject<JsonElement>(AgicApplicationGatewayArgs);
var agicAppGwName = agicAppGwArgs.GetName();
var agicAppGwSku = agicAppGwArgs.GetSku();
var agicAppGwMinCapacity = agicAppGwArgs.GetInt(MinCapacity);
var agicAppGwMaxCapacity = agicAppGwArgs.GetInt(MaxCapacity);

// Gateway IP Config (subnet to which Application Gateway would be deployed)
var appGwIpConfig = new ApplicationGatewayIPConfigurationArgs {
    Name = AppGatewayIpConfigName,
    Subnet = new SubResourceArgs {
        Id = hubAppGwSubnet.Id,
    }
};

//
// Create App Gateway
//
var agicAppGw = new ApplicationGateway(AgicApplicationGateway, new ApplicationGatewayArgs {
    ApplicationGatewayName = agicAppGwName,
    ResourceGroupName = mainResourceGroup.Name,
    Sku = new ApplicationGatewaySkuArgs {
        Name = agicAppGwSku,
        Tier = agicAppGwSku
    },
    AutoscaleConfiguration = new ApplicationGatewayAutoscaleConfigurationArgs {
        MinCapacity = agicAppGwMinCapacity,
        MaxCapacity = agicAppGwMaxCapacity
    },
    GatewayIPConfigurations = { appGwIpConfig },
    FrontendIPConfigurations = {
            new ApplicationGatewayFrontendIPConfigurationArgs {
                Name = AppGatewayFrontendIpConfigName,
                PublicIPAddress = new SubResourceArgs {
                    Id = agicAppGwPublicIp.Id,
                }
            },
            new ApplicationGatewayFrontendIPConfigurationArgs {
                Name = $"{AppGatewayFrontendIpConfigName}_private",
                PrivateIPAllocationMethod = IPAllocationMethod.Static,
                PrivateIPAddress = "10.10.1.5", //hubApplicationGatewaySubnet.GetFirstUsableIp(), 
                Subnet = new SubResourceArgs {
                    Id = hubAppGwSubnet.Id
                }
            }
        },
    FrontendPorts = {
            new ApplicationGatewayFrontendPortArgs {
                Name = AppGatewayFrontendPort80Name,
                Port = Port80
            },
        },
    HttpListeners = {
            new ApplicationGatewayHttpListenerArgs {
                Name = AppGatewayHttpListenerName,
                Protocol = Http,
                FrontendIPConfiguration = new SubResourceArgs {
                    Id = $"/subscriptions/{subscriptionId}/resourceGroups/{mainResourceGroup.Name}/providers/Microsoft.Network/applicationGateways/{agicAppGwName}/frontendIPConfigurations/{AppGatewayFrontendIpConfigName}"
                },
                FrontendPort = new SubResourceArgs {
                    Id = $"/subscriptions/{subscriptionId}/resourceGroups/{mainResourceGroup.Name}/providers/Microsoft.Network/applicationGateways/{agicAppGwName}/frontendPorts/{AppGatewayFrontendPort80Name}",
                },
            }
        },
    BackendAddressPools = {
            new ApplicationGatewayBackendAddressPoolArgs {
                Name = AppGatewayBackendPoolName,
                //BackendAddresses = { }
            }
        },
    BackendHttpSettingsCollection = {
            new ApplicationGatewayBackendHttpSettingsArgs {
                Name = AppGatewayHttpSettingName,
                Port = 80,
                Protocol = Http,
                RequestTimeout = 20,
                CookieBasedAffinity = ApplicationGatewayCookieBasedAffinity.Enabled
            }
        },
    RequestRoutingRules = {
            new ApplicationGatewayRequestRoutingRuleArgs {
                Name = AppGatewayRoutingName,
                RuleType = ApplicationGatewayRequestRoutingRuleType.Basic,
                Priority = 10,
                HttpListener = new SubResourceArgs {
                    Id = $"/subscriptions/{subscriptionId}/resourceGroups/{mainResourceGroup.Name}/providers/Microsoft.Network/applicationGateways/{agicAppGwName}/httpListeners/{AppGatewayHttpListenerName}",
                },
                BackendAddressPool = new SubResourceArgs {
                    Id = $"/subscriptions/{subscriptionId}/resourceGroups/{mainResourceGroup.Name}/providers/Microsoft.Network/applicationGateways/{agicAppGwName}/backendAddressPools/{AppGatewayBackendPoolName}",
                },
                BackendHttpSettings = new SubResourceArgs {
                    Id = $"/subscriptions/{subscriptionId}/resourceGroups/{mainResourceGroup.Name}/providers/Microsoft.Network/applicationGateways/{agicAppGwName}/backendHttpSettingsCollection/{AppGatewayHttpSettingName}",
                },
            }
        },
    WebApplicationFirewallConfiguration = new ApplicationGatewayWebApplicationFirewallConfigurationArgs { },
}, new CustomResourceOptions { DependsOn = { hubAppGwSubnet } });

C# code for AKS Cluster

// ... ... ...

var systemNodePool = new ManagedClusterAgentPoolProfileArgs { // ... ... ...};
var userNodePool = new ManagedClusterAgentPoolProfileArgs { // ... ... ... };
var aadProfile = new ManagedClusterAADProfileArgs { // ... ... ... };
var networkProfile = new ContainerServiceNetworkProfileArgs { // ... ... ... };

//
// Create AKS Cluster
//
var aksAppClusterName = "xxx-aks-appcluster-dev-japaneast";
var aksAppClusterNodeRgName = $"{AksEntities.NodePoolResourceGroupPrefix}_{aksAppClusterName}";
var aksAppClusterDnsPrefix = "xxx-yyy";

var aksAppCluster = new ManagedCluster(AksAppplicationCluster, new ManagedClusterArgs {
    ResourceName = aksAppClusterName,
    ResourceGroupName = mainResourceGroup.Name,
    AgentPoolProfiles = { systemNodePool, userNodePool },
    DnsPrefix = aksAppClusterDnsPrefix,
    EnableRBAC = true,
    AadProfile = aadProfile,
    NetworkProfile = networkProfile,
    //ServicePrincipalProfile = spProfile,
    NodeResourceGroup = aksAppClusterNodeRgName,
    DisableLocalAccounts = true,
    Identity = new ManagedClusterIdentityArgs { Type = AzureNative.ContainerService.ResourceIdentityType.SystemAssigned },
    AddonProfiles = {
        {
            AksEntities.AddonProfileKeys.Agic, new ManagedClusterAddonProfileArgs {
                Enabled = true,
                Config = {
                    {
                        AksEntities.AddonProfileKeys.ApplicationGatewayId, agicAppGw.Id
                    },
                }
            }
        },
    },
}, new CustomResourceOptions { DependsOn = { spokeAksAppplicationSubnet, agicAppGw } });