apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
  name: environment-provisioner
  namespace: argo-workflows
spec:
  serviceAccountName: argo-workflow
  # Clean up pods after workflow completion
  podGC:
    strategy: OnWorkflowSuccess    # Delete pods when workflow succeeds
    deleteDelayDuration: 60s       # Wait 60s before deleting (allows log collection)
  # Auto-delete workflow after completion
  ttlStrategy:
    secondsAfterCompletion: 3600   # Delete workflow 1 hour after completion
    secondsAfterSuccess: 1800      # Delete workflow 30 min after success
    secondsAfterFailure: 86400     # Keep failed workflows for 24 hours for debugging
  arguments:
    parameters:
    - name: namespace
      value: default
    - name: services
      value: ""
    - name: ttl
      value: "8"
    - name: owner
      value: unknown
    - name: componentMetadata
      value: '{}'
    - name: dependencyPreferences
      value: '{}'
    - name: valuesFileOverrides
      value: '{}'
    - name: manualDependencyUrls
      value: '{}'
    - name: valuesContentEdits
      value: '{}'
    - name: helmExternalParams
      value: '{}'
  entrypoint: provision-environment
  templates:
  - inputs:
      parameters:
      - name: namespace
      - name: services
      - name: ttl
      - name: owner
      - name: componentMetadata
      - name: dependencyPreferences
      - name: valuesFileOverrides
      - name: manualDependencyUrls
      - name: valuesContentEdits
      - name: helmExternalParams
    name: provision-environment
    steps:
    - - arguments:
          parameters:
          - name: namespace
            value: '{{inputs.parameters.namespace}}'
          - name: services
            value: '{{inputs.parameters.services}}'
          - name: ttl
            value: '{{inputs.parameters.ttl}}'
          - name: owner
            value: '{{inputs.parameters.owner}}'
          - name: componentMetadata
            value: '{{inputs.parameters.componentMetadata}}'
          - name: dependencyPreferences
            value: '{{inputs.parameters.dependencyPreferences}}'
          - name: valuesFileOverrides
            value: '{{inputs.parameters.valuesFileOverrides}}'
          - name: manualDependencyUrls
            value: '{{inputs.parameters.manualDependencyUrls}}'
          - name: valuesContentEdits
            value: '{{inputs.parameters.valuesContentEdits}}'
          - name: helmExternalParams
            value: '{{inputs.parameters.helmExternalParams}}'
        name: create-git-repo
        template: create-repo
    - - arguments:
          parameters:
          - name: namespace
            value: '{{inputs.parameters.namespace}}'
          - name: owner
            value: '{{inputs.parameters.owner}}'
        name: create-argocd-app
        template: create-argo-app
    - - arguments:
          parameters:
          - name: namespace
            value: '{{inputs.parameters.namespace}}'
          - name: services
            value: '{{inputs.parameters.services}}'
        name: register-hades-ips
        template: register-hades-ips
  - inputs:
      parameters:
      - name: namespace
      - name: services
      - name: ttl
      - name: owner
      - name: componentMetadata
      - name: dependencyPreferences
      - name: valuesFileOverrides
      - name: manualDependencyUrls
      - name: valuesContentEdits
      - name: helmExternalParams
    name: create-repo
    script:
      command:
      - sh
      env:
      # Credentials from External Secrets (GCP Secret Manager)
      - name: GITEA_URL
        valueFrom:
          secretKeyRef:
            name: gitea-credentials
            key: url
      - name: GITEA_USER
        valueFrom:
          secretKeyRef:
            name: gitea-credentials
            key: username
      - name: GITEA_PASSWORD
        valueFrom:
          secretKeyRef:
            name: gitea-credentials
            key: password
      image: alpine/git:latest
      source: |
        #!/bin/sh
        set -e

        NAMESPACE="{{inputs.parameters.namespace}}"
        SERVICES="{{inputs.parameters.services}}"
        TTL="{{inputs.parameters.ttl}}"
        OWNER="{{inputs.parameters.owner}}"
        COMPONENT_METADATA=$(cat <<'EOFPARAM'
        {{inputs.parameters.componentMetadata}}
        EOFPARAM
        )
        DEPENDENCY_PREFS=$(cat <<'EOFPARAM'
        {{inputs.parameters.dependencyPreferences}}
        EOFPARAM
        )
        VALUES_FILE_OVERRIDES=$(cat <<'EOFPARAM'
        {{inputs.parameters.valuesFileOverrides}}
        EOFPARAM
        )
        MANUAL_DEPENDENCY_URLS=$(cat <<'EOFPARAM'
        {{inputs.parameters.manualDependencyUrls}}
        EOFPARAM
        )
        VALUES_CONTENT_EDITS=$(cat <<'EOFPARAM'
        {{inputs.parameters.valuesContentEdits}}
        EOFPARAM
        )
        HELM_EXTERNAL_PARAMS=$(cat <<'EOFPARAM'
        {{inputs.parameters.helmExternalParams}}
        EOFPARAM
        )

        # Sanitize namespace name (replace spaces and special chars with hyphens)
        NAMESPACE=$(echo "$NAMESPACE" | tr '[:upper:]' '[:lower:]' | tr ' _' '-' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')

        # Sanitize owner name for labels (Kubernetes labels have strict requirements)
        OWNER_LABEL=$(echo "$OWNER" | tr '[:upper:]' '[:lower:]' | tr ' _' '-' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')

        echo "Creating Git repository for namespace: $NAMESPACE"

        # Install curl and jq
        apk add --no-cache curl jq

        # Create repository in Gitea
        REPO_NAME="environment-$NAMESPACE"

        # Extract hostname from GITEA_URL for git credentials
        GITEA_HOST=$(echo "$GITEA_URL" | sed 's|https://||' | sed 's|http://||' | cut -d'/' -f1)

        echo "Creating Gitea repository: $REPO_NAME"
        curl -X POST "$GITEA_URL/api/v1/user/repos" \
          -u "$GITEA_USER:$GITEA_PASSWORD" \
          -H "Content-Type: application/json" \
          -d "{
            \"name\": \"$REPO_NAME\",
            \"description\": \"Environment for $NAMESPACE (owner: $OWNER, ttl: ${TTL}h)\",
            \"private\": false,
            \"auto_init\": true
          }" || echo "Repository may already exist"

        # Clone the repository
        git config --global http.sslVerify true
        git config --global user.email "backstage@osmos.io"
        git config --global user.name "Backstage"

        # URL encode the password for git credentials
        ENCODED_PASSWORD=$(printf %s "$GITEA_PASSWORD" | jq -sRr @uri)

        # Configure credentials
        git config --global credential.helper store
        printf "https://%s:%s@%s\n" "$GITEA_USER" "$ENCODED_PASSWORD" "$GITEA_HOST" > $HOME/.git-credentials

        REPO_URL="$GITEA_URL/$GITEA_USER/$REPO_NAME.git"
        git clone "$REPO_URL" /tmp/repo 2>&1 || (cd /tmp/repo && git pull 2>&1)
        cd /tmp/repo

        # Create manifests directory
        mkdir -p manifests

        # Create namespace manifest
        cat > manifests/namespace.yaml <<EOF
        apiVersion: v1
        kind: Namespace
        metadata:
          name: $NAMESPACE
          labels:
            name: $NAMESPACE
            environment: test
            owner: $OWNER_LABEL
            ttl: ${TTL}h
            provisioned-by: backstage
            managed-by: argocd
          annotations:
            ttl-hours: "$TTL"
            created-by: backstage-environment-provisioner
            description: "Test environment for $OWNER"
            original-owner: "$OWNER"
        EOF

        # Create NetworkPolicy for isolation (allows ingress from same namespace, ingress-nginx, and argo-workflows)
        cat > manifests/networkpolicy.yaml <<EOF
        apiVersion: networking.k8s.io/v1
        kind: NetworkPolicy
        metadata:
          name: default-deny-ingress
          namespace: $NAMESPACE
        spec:
          podSelector: {}
          policyTypes:
          - Ingress
          ingress:
          - from:
            - namespaceSelector:
                matchLabels:
                  name: $NAMESPACE
          - from:
            - namespaceSelector:
                matchLabels:
                  kubernetes.io/metadata.name: ingress-nginx
          - from:
            - namespaceSelector:
                matchLabels:
                  kubernetes.io/metadata.name: argo-workflows
        EOF

        # Create service manifests
        # Convert comma-separated services to space-separated for iteration
        SERVICES_LIST=$(echo "$SERVICES" | tr ',' ' ')
        for service in $SERVICES_LIST; do
          service=$(echo "$service" | xargs) # trim whitespace
          if [ -n "$service" ]; then
            # Skip resource references (already filtered in frontend, but double-check)
            if echo "$service" | grep -q "^resource:"; then
              echo "Skipping resource reference: $service"
              continue
            fi

            echo "Processing service: $service"

            # Sanitize service name for filename (remove special chars)
            SANITIZED_SERVICE=$(echo "$service" | tr '[:upper:]' '[:lower:]' | tr ' /:' '-' | sed 's/[^a-z0-9-]//g')

            # Create truncated app name (Helm release names must be <= 53 chars)
            APP_NAME=$(echo "$NAMESPACE-$SANITIZED_SERVICE" | cut -c1-53 | sed 's/-$//')

            # Check if component has Helm chart annotations
            HELM_REPO=$(echo "$COMPONENT_METADATA" | jq -r ".[\"$service\"][\"helm.cnoe.io/git-repo\"] // empty")
            HELM_PATH=$(echo "$COMPONENT_METADATA" | jq -r ".[\"$service\"][\"helm.cnoe.io/chart-path\"] // empty")
            HELM_VALUES_DEFAULT=$(echo "$COMPONENT_METADATA" | jq -r ".[\"$service\"][\"helm.cnoe.io/values-file\"] // \"values.yaml\"")
            HELM_VALUES=$(echo "$VALUES_FILE_OVERRIDES" | jq -r ".[\"$service\"] // \"$HELM_VALUES_DEFAULT\"")
            HELM_GIT_REF=$(echo "$COMPONENT_METADATA" | jq -r ".[\"$service\"][\"helm.cnoe.io/git-ref\"] // \"HEAD\"")

            # DEBUG: Log what we're checking
            echo "  Checking for Helm annotations..."
            echo "  HELM_REPO: $HELM_REPO"
            echo "  HELM_PATH: $HELM_PATH"
            echo "  HELM_VALUES_DEFAULT: $HELM_VALUES_DEFAULT"
            echo "  HELM_VALUES (after override): $HELM_VALUES"
            echo "  HELM_GIT_REF: $HELM_GIT_REF"

            if [ -n "$HELM_REPO" ]; then
              # Component has Helm chart - create ArgoCD Application manifest
              echo "Creating ArgoCD Application for Helm chart: $service"

              # Check dependency preferences for MySQL
              MYSQL_ENABLED=$(echo "$DEPENDENCY_PREFS" | jq -r '.mysql // "true"')
              echo "  MySQL enabled: $MYSQL_ENABLED"

              # Process dependency URL mappings
              DEPENDENCY_URL_PARAMS=""
              DEPENDENCY_URLS=$(echo "$COMPONENT_METADATA" | jq -r ".[\"$service\"][\"helm.cnoe.io/dependency-urls\"] // empty")

              if [ -n "$DEPENDENCY_URLS" ]; then
                echo "  Processing dependency URL mappings..."

                for dep_key in $(echo "$DEPENDENCY_URLS" | jq -r 'keys[]'); do
                  echo "    Checking dependency: $dep_key"

                  # Check if this dependency is in our resolved services list
                  DEP_IN_SERVICES=$(echo "$SERVICES" | tr ',' '\n' | grep -c "^${dep_key}$" || echo "0")

                  # Get dependency configuration
                  SERVICE_DNS=$(echo "$DEPENDENCY_URLS" | jq -r ".[\"$dep_key\"].serviceDns // \"$dep_key\"")
                  SERVICE_PORT=$(echo "$DEPENDENCY_URLS" | jq -r ".[\"$dep_key\"].port // 8080")
                  HELM_PARAM=$(echo "$DEPENDENCY_URLS" | jq -r ".[\"$dep_key\"].helmParam")

                  # Sanitize service DNS name
                  SERVICE_DNS=$(echo "$SERVICE_DNS" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]//g')

                  CONSTRUCTED_URL=""

                  if [ "$DEP_IN_SERVICES" -gt 0 ]; then
                    # Dependency is being deployed to same namespace - use short DNS
                    echo "    Dependency $dep_key is being deployed to same namespace"
                    CONSTRUCTED_URL="http://${SERVICE_DNS}:${SERVICE_PORT}"
                  else
                    # Check for manual URL override
                    MANUAL_URL=$(echo "$MANUAL_DEPENDENCY_URLS" | jq -r ".[\"$service\"][\"$HELM_PARAM\"] // empty")
                    if [ -n "$MANUAL_URL" ]; then
                      echo "    Using manual URL for $dep_key"
                      CONSTRUCTED_URL="$MANUAL_URL"
                    else
                      echo "    Dependency $dep_key NOT in deployed services and no manual URL - using defaults"
                    fi
                  fi

                  if [ -n "$CONSTRUCTED_URL" ] && [ -n "$HELM_PARAM" ]; then
                    echo "    Setting $HELM_PARAM = $CONSTRUCTED_URL"
                    DEPENDENCY_URL_PARAMS="$DEPENDENCY_URL_PARAMS
                - name: $HELM_PARAM
                  value: \"$CONSTRUCTED_URL\""
                  fi
                done
              fi

              # Check for edited values content - write to override file if exists
              INLINE_VALUES=""
              EDITED_CONTENT=$(echo "$VALUES_CONTENT_EDITS" | jq -r ".\"$service\" // empty")
              if [ -n "$EDITED_CONTENT" ] && [ "$EDITED_CONTENT" != "null" ]; then
                echo "  Writing edited values content to override file..."
                # Write the edited content to a values override file
                VALUES_OVERRIDE_FILE="values-override-${SANITIZED_SERVICE}.yaml"
                echo "$EDITED_CONTENT" > "manifests/$VALUES_OVERRIDE_FILE"
                echo "  Created: manifests/$VALUES_OVERRIDE_FILE"
                # Set inline values for ArgoCD - indent the content properly
                INDENTED_CONTENT=$(echo "$EDITED_CONTENT" | sed 's/^/                /')
                INLINE_VALUES="
              values: |
        $INDENTED_CONTENT"
              fi

              # Process external params for disabled Helm dependencies (e.g., external database config)
              CUSTOM_HELM_PARAMS=""
              SERVICE_EXTERNAL_PARAMS=$(echo "$HELM_EXTERNAL_PARAMS" | jq -r ".\"$service\" // {}")
              if [ "$SERVICE_EXTERNAL_PARAMS" != "{}" ] && [ -n "$SERVICE_EXTERNAL_PARAMS" ]; then
                echo "  Processing external params for disabled dependencies..."
                for param_key in $(echo "$SERVICE_EXTERNAL_PARAMS" | jq -r 'keys[]'); do
                  PARAM_VALUE=$(echo "$SERVICE_EXTERNAL_PARAMS" | jq -r ".\"$param_key\"")
                  if [ -n "$PARAM_VALUE" ]; then
                    echo "    Setting $param_key = $PARAM_VALUE"
                    # Use param_key directly - dots are Helm nested path separators, not literal chars
                    CUSTOM_HELM_PARAMS="${CUSTOM_HELM_PARAMS}
                - name: ${param_key}
                  value: \"${PARAM_VALUE}\""
                  fi
                done
              fi

              # Extract ingress hostname from GITEA_URL domain
              INGRESS_DOMAIN=$(echo "$GITEA_URL" | sed 's|https://gitea\.||' | sed 's|http://gitea\.||')

              cat > "manifests/$APP_NAME-application.yaml" <<EOF
        apiVersion: argoproj.io/v1alpha1
        kind: Application
        metadata:
          name: $APP_NAME
          namespace: argocd
          labels:
            environment: $NAMESPACE
            provisioned-by: backstage
        spec:
          project: default
          source:
            repoURL: $HELM_REPO
            targetRevision: $HELM_GIT_REF
            path: $HELM_PATH
            helm:
              valueFiles:
                - $HELM_VALUES$INLINE_VALUES
              parameters:
                - name: mysql.enabled
                  value: "$MYSQL_ENABLED"
                # Override fullname to keep K8s resource names short (max 63 chars)
                - name: fullnameOverride
                  value: "$SANITIZED_SERVICE"
                # Add commonLabels for hades resources
                - name: commonLabels.backstage\.io/kubernetes-id
                  value: "$service"
                # Add podLabels for MySQL resources
                - name: mysql.podLabels.backstage\.io/kubernetes-id
                  value: "$service"
                # Enable ingress via Helm chart
                - name: ingress.enabled
                  value: "true"
                - name: ingress.className
                  value: "nginx"
                - name: ingress.hosts[0].host
                  value: "${SANITIZED_SERVICE}-${NAMESPACE}.${INGRESS_DOMAIN}"
                - name: ingress.hosts[0].paths[0].path
                  value: "/"
                - name: ingress.hosts[0].paths[0].pathType
                  value: "Prefix"
                # TLS Configuration for HTTPS
                - name: ingress.tls[0].hosts[0]
                  value: "${SANITIZED_SERVICE}-${NAMESPACE}.${INGRESS_DOMAIN}"
                - name: ingress.tls[0].secretName
                  value: "${SANITIZED_SERVICE}-${NAMESPACE}-tls"
                # Annotations for cert-manager and HTTPS
                - name: ingress.annotations.nginx\.ingress\.kubernetes\.io/force-ssl-redirect
                  value: '"true"'
                - name: ingress.annotations.nginx\.ingress\.kubernetes\.io/ssl-redirect
                  value: '"true"'
                - name: ingress.annotations.cert-manager\.io/cluster-issuer
                  value: letsencrypt-prod$DEPENDENCY_URL_PARAMS$CUSTOM_HELM_PARAMS
          destination:
            server: https://kubernetes.default.svc
            namespace: $NAMESPACE
          syncPolicy:
            automated:
              prune: true
              selfHeal: true
            syncOptions:
              - CreateNamespace=false
              - ServerSideApply=true
        EOF
            echo "  Ingress URL: https://${SANITIZED_SERVICE}-${NAMESPACE}.${INGRESS_DOMAIN}"
            else
              # No Helm chart - create simple deployment manifest
              echo "Creating deployment manifest for service: $service"

              cat > "manifests/$SANITIZED_SERVICE-deployment.yaml" <<EOF
        apiVersion: apps/v1
        kind: Deployment
        metadata:
          name: $service
          namespace: $NAMESPACE
          labels:
            app: $service
            provisioned-by: backstage
            managed-by: argocd
        spec:
          replicas: 1
          selector:
            matchLabels:
              app: $service
          template:
            metadata:
              labels:
                app: $service
            spec:
              containers:
              - name: $service
                image: nginx:alpine
                ports:
                - containerPort: 80
                  name: http
                resources:
                  limits:
                    cpu: 500m
                    memory: 512Mi
                  requests:
                    cpu: 100m
                    memory: 128Mi
                livenessProbe:
                  httpGet:
                    path: /
                    port: http
                  initialDelaySeconds: 10
                  periodSeconds: 10
                readinessProbe:
                  httpGet:
                    path: /
                    port: http
                  initialDelaySeconds: 5
                  periodSeconds: 5
        ---
        apiVersion: v1
        kind: Service
        metadata:
          name: $service
          namespace: $NAMESPACE
          labels:
            app: $service
            provisioned-by: backstage
            managed-by: argocd
        spec:
          type: ClusterIP
          selector:
            app: $service
          ports:
          - port: 80
            targetPort: http
            protocol: TCP
            name: http
        EOF
            fi
          fi
        done

        # Create README
        cat > README.md <<EOF
        # Environment: $NAMESPACE

        **Owner:** $OWNER
        **TTL:** $TTL hours
        **Services:** $SERVICES

        ## Description
        This is an automatically provisioned test environment managed by ArgoCD.

        ## Resources
        - Namespace with resource quotas and limits
        - Network policies for isolation
        - Deployed services: $(echo $SERVICES | tr ',' ' ')

        ## Management
        This environment is managed by ArgoCD. Any changes pushed to this repository will be automatically synced to the cluster.

        **Note:** This environment will be cleaned up after $TTL hours.
        EOF

        # Commit and push
        git add .
        git commit -m "Initial environment setup for $NAMESPACE by $OWNER"
        git push origin main || git push origin master

        echo "Git repository created and pushed successfully"
        echo "Repository URL: $GITEA_URL/$GITEA_USER/$REPO_NAME"
  - inputs:
      parameters:
      - name: namespace
      - name: owner
    name: create-argo-app
    script:
      command:
      - sh
      env:
      # Credentials from External Secrets (GCP Secret Manager)
      - name: ARGOCD_SERVER
        valueFrom:
          secretKeyRef:
            name: argocd-credentials
            key: server
      - name: ARGOCD_USERNAME
        valueFrom:
          secretKeyRef:
            name: argocd-credentials
            key: username
      - name: ARGOCD_PASSWORD
        valueFrom:
          secretKeyRef:
            name: argocd-credentials
            key: password
      - name: GITEA_URL
        valueFrom:
          secretKeyRef:
            name: gitea-credentials
            key: url
      - name: GITEA_USER
        valueFrom:
          secretKeyRef:
            name: gitea-credentials
            key: username
      image: argoproj/argocd:latest
      source: |
        #!/bin/sh
        set -e

        NAMESPACE="{{inputs.parameters.namespace}}"
        OWNER="{{inputs.parameters.owner}}"

        # Sanitize namespace name (replace spaces and special chars with hyphens)
        NAMESPACE=$(echo "$NAMESPACE" | tr '[:upper:]' '[:lower:]' | tr ' _' '-' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')

        # Sanitize owner name for label (Kubernetes labels have strict requirements)
        OWNER_LABEL=$(echo "$OWNER" | tr '[:upper:]' '[:lower:]' | tr ' _' '-' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')

        REPO_NAME="environment-$NAMESPACE"

        echo "Creating ArgoCD application for namespace: $NAMESPACE"

        # Login to ArgoCD (TLS enabled for production)
        argocd login $ARGOCD_SERVER \
          --username $ARGOCD_USERNAME \
          --password $ARGOCD_PASSWORD \
          --grpc-web

        # Create ArgoCD application
        argocd app create "$NAMESPACE-environment" \
          --repo "$GITEA_URL/$GITEA_USER/$REPO_NAME.git" \
          --path manifests \
          --dest-server https://kubernetes.default.svc \
          --dest-namespace $NAMESPACE \
          --sync-policy automated \
          --auto-prune \
          --self-heal \
          --sync-option CreateNamespace=true \
          --upsert \
          --label "owner=$OWNER_LABEL" \
          --label "provisioned-by=backstage" \
          --label "managed-by=argocd"

        echo "ArgoCD application created successfully"
        echo "Auto-sync is enabled - ArgoCD will automatically sync the application"
        echo "View in ArgoCD UI: https://$ARGOCD_SERVER/applications/$NAMESPACE-environment"
  - inputs:
      parameters:
      - name: namespace
      - name: services
    name: register-hades-ips
    script:
      command:
      - bash
      image: bitnami/kubectl:latest
      source: |
        #!/bin/bash
        set -e

        NAMESPACE="{{inputs.parameters.namespace}}"
        SERVICES="{{inputs.parameters.services}}"

        # Sanitize namespace
        NAMESPACE=$(echo "$NAMESPACE" | tr '[:upper:]' '[:lower:]' | tr ' _' '-' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')

        # MySQL connection details for Hades
        MYSQL_HOST="hades-mysql-db-service.${NAMESPACE}.svc.cluster.local"
        MYSQL_PORT="3306"
        MYSQL_USER="root"
        MYSQL_PWD="local-root-pass"
        MYSQL_DB="hades_db"

        echo "=== Hades IP Registration ==="
        echo "Namespace: $NAMESPACE"
        echo "MySQL Host: $MYSQL_HOST"
        echo ""

        # kubectl is already available in this image

        # Wait for pods to be created by ArgoCD (may take time to sync)
        echo "Waiting for Hades MySQL pod to be created..."
        MYSQL_POD=""
        POD_WAIT=0
        POD_MAX_WAIT=180

        while [ -z "$MYSQL_POD" ] && [ $POD_WAIT -lt $POD_MAX_WAIT ]; do
          # Try label used by WSO2 MySQL chart (hades uses this)
          MYSQL_POD=$(kubectl get pods -n "$NAMESPACE" -l "app=hades-mysql-db-service" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")

          if [ -z "$MYSQL_POD" ]; then
            # Try Bitnami MySQL chart label
            MYSQL_POD=$(kubectl get pods -n "$NAMESPACE" -l "app.kubernetes.io/name=mysql" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
          fi

          if [ -z "$MYSQL_POD" ]; then
            # Fallback: grep for hades mysql in pod names
            MYSQL_POD=$(kubectl get pods -n "$NAMESPACE" -o name 2>/dev/null | grep -i "hades.*mysql" | head -1 | sed 's/pod\///')
          fi

          if [ -z "$MYSQL_POD" ]; then
            echo "  Waiting for MySQL pod... (${POD_WAIT}s)"
            sleep 10
            POD_WAIT=$((POD_WAIT + 10))
          fi
        done

        if [ -z "$MYSQL_POD" ]; then
          echo "Warning: Could not find Hades MySQL pod after ${POD_MAX_WAIT}s"
          echo "Skipping IP registration - Hades may not be deployed"
          exit 0
        fi

        echo "Found MySQL pod: $MYSQL_POD"

        # Wait for Hades MySQL to be ready (should already be running from previous steps)
        echo "Waiting for Hades MySQL to be ready..."
        MAX_WAIT=120
        WAIT_TIME=0
        while [ $WAIT_TIME -lt $MAX_WAIT ]; do
          if kubectl exec -n "$NAMESPACE" "$MYSQL_POD" -- mysql -u "$MYSQL_USER" -p"$MYSQL_PWD" -e "SELECT 1" "$MYSQL_DB" > /dev/null 2>&1; then
            echo "Hades MySQL is ready"
            break
          fi
          echo "  Waiting for MySQL... (${WAIT_TIME}s)"
          sleep 5
          WAIT_TIME=$((WAIT_TIME + 5))
        done

        if [ $WAIT_TIME -ge $MAX_WAIT ]; then
          echo "Warning: Could not connect to Hades MySQL after ${MAX_WAIT}s"
          echo "Skipping IP registration - Hades may not be deployed"
          exit 0
        fi

        # Wait for Hades dbinit job to complete (creates the database schema)
        echo "Waiting for Hades dbinit job to complete..."
        DBINIT_MAX_WAIT=120
        DBINIT_WAIT_TIME=0
        while [ $DBINIT_WAIT_TIME -lt $DBINIT_MAX_WAIT ]; do
          # Check if dbinit job exists and is complete
          DBINIT_JOB=$(kubectl get jobs -n "$NAMESPACE" -o name 2>/dev/null | grep -i "hades.*dbinit" | head -1) || DBINIT_JOB=""

          if [ -n "$DBINIT_JOB" ]; then
            DBINIT_COMPLETE=$(kubectl get "$DBINIT_JOB" -n "$NAMESPACE" -o jsonpath='{.status.succeeded}' 2>/dev/null) || DBINIT_COMPLETE="0"
            if [ "$DBINIT_COMPLETE" = "1" ]; then
              echo "Hades dbinit job completed"
              break
            fi
          fi

          echo "  Waiting for dbinit job... (${DBINIT_WAIT_TIME}s)"
          sleep 5
          DBINIT_WAIT_TIME=$((DBINIT_WAIT_TIME + 5))
        done

        if [ $DBINIT_WAIT_TIME -ge $DBINIT_MAX_WAIT ]; then
          echo "Warning: Hades dbinit job not completed after ${DBINIT_MAX_WAIT}s"
          echo "Tables may not exist - attempting registration anyway"
        fi

        # Helper function to run MySQL commands via kubectl exec
        # Using -c to specify container explicitly to avoid kubectl container selection messages
        MYSQL_CONTAINER=$(kubectl get pod -n "$NAMESPACE" "$MYSQL_POD" -o jsonpath='{.spec.containers[0].name}' 2>/dev/null)
        echo "MySQL container name: $MYSQL_CONTAINER"

        run_mysql() {
          local result
          local exit_code
          # Run kubectl exec with explicit container, redirect kubectl stderr to /dev/null
          # but capture MySQL output (including MySQL warnings on stderr)
          result=$(kubectl exec -n "$NAMESPACE" "$MYSQL_POD" -c "$MYSQL_CONTAINER" -- \
            mysql -u "$MYSQL_USER" -p"$MYSQL_PWD" "$MYSQL_DB" -sN -e "$1" 2>&1)
          exit_code=$?

          # Filter out MySQL password warning
          result=$(echo "$result" | grep -v "Using a password on the command line interface can be insecure")

          if [ $exit_code -ne 0 ]; then
            echo "  MySQL error: $result" >&2
            return 1
          fi
          echo "$result"
        }

        # Function to register IP for an app
        register_ip() {
          local app_name="$1"
          local ip="$2"

          if [ -z "$app_name" ] || [ -z "$ip" ]; then
            return 0
          fi

          # Get or create app_id (using INSERT ... ON DUPLICATE KEY UPDATE for idempotency)
          echo "  Ensuring app entry exists for: $app_name"
          if ! run_mysql "INSERT INTO apps (name, token, creation_date, last_update, is_deleted) VALUES ('$app_name', UUID(), NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE last_update=NOW()"; then
            echo "  Warning: INSERT command returned non-zero, but continuing..."
          fi

          APP_ID=$(run_mysql "SELECT id FROM apps WHERE name='$app_name'" || echo "")

          if [ -z "$APP_ID" ]; then
            echo "  Failed to get app_id for: $app_name"
            return 1
          fi
          echo "  Using app_id: $APP_ID"

          # Register IP (using INSERT IGNORE to handle duplicates gracefully)
          echo "  Registering IP $ip for app $app_name (app_id: $APP_ID)"
          if run_mysql "INSERT IGNORE INTO app_ips_accesses (app_id, ipaddress, creation_date, last_update, is_deleted) VALUES ($APP_ID, '$ip', NOW(), NOW(), 0)"; then
            echo "  Successfully registered IP $ip"
          else
            echo "  Failed to register IP $ip"
          fi
          return 0
        }

        # Verify required tables exist
        echo "Checking database tables..."
        TABLES=$(run_mysql "SHOW TABLES")
        echo "Available tables: $TABLES"

        if ! echo "$TABLES" | grep -q "apps"; then
          echo "ERROR: 'apps' table not found in database"
          echo "Database may not be initialized. Exiting."
          exit 1
        fi

        if ! echo "$TABLES" | grep -q "app_ips_accesses"; then
          echo "ERROR: 'app_ips_accesses' table not found in database"
          echo "Database may not be initialized. Exiting."
          exit 1
        fi

        echo "Required tables found. Proceeding with IP registration..."
        echo ""

        # 1. Collect common bridge/gateway IPs (handles SNAT across different clusters)
        COMMON_BRIDGE_IPS="10.244.0.1 10.96.0.1 172.17.0.1"
        echo "Common bridge IPs: $COMMON_BRIDGE_IPS"

        # 2. Get node internal IPs
        NODE_IPS=$(kubectl get nodes -o jsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null || echo "")
        echo "Node IPs: $NODE_IPS"

        # 3. Get pod network CIDR gateways from ALL nodes
        echo "Collecting pod CIDR gateways from all nodes..."
        POD_CIDRS=$(kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' 2>/dev/null || echo "")
        for cidr in $POD_CIDRS; do
          if [ -n "$cidr" ]; then
            GATEWAY_IP=$(echo "$cidr" | sed 's/\.[0-9]*\/.*/.1/')
            COMMON_BRIDGE_IPS="$COMMON_BRIDGE_IPS $GATEWAY_IP"
            echo "  Pod CIDR gateway: $GATEWAY_IP (from $cidr)"
          fi
        done

        # Combine all IPs to register
        ALL_IPS="$COMMON_BRIDGE_IPS $NODE_IPS"
        echo ""

        # Process each service
        SERVICES_LIST=$(echo "$SERVICES" | tr ',' ' ')
        for service in $SERVICES_LIST; do
          service=$(echo "$service" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
          [ -z "$service" ] && continue

          # Skip resource references
          if echo "$service" | grep -q "^resource:"; then
            continue
          fi

          echo "Processing service: $service"

          # Get actual pod IPs for this service
          # Try multiple label selectors as different charts use different labels
          POD_IPS=$(kubectl get pods -n "$NAMESPACE" \
            -l "app.kubernetes.io/instance=${NAMESPACE}-${service}" \
            -o jsonpath='{.items[*].status.podIP}' 2>/dev/null) || POD_IPS=""

          if [ -z "$POD_IPS" ]; then
            # Try backstage.io/kubernetes-id label
            POD_IPS=$(kubectl get pods -n "$NAMESPACE" \
              -l "backstage.io/kubernetes-id=${service}" \
              -o jsonpath='{.items[*].status.podIP}' 2>/dev/null) || POD_IPS=""
          fi

          echo "  Found pod IPs: ${POD_IPS:-none}"

          # Register pod IPs
          for ip in $POD_IPS; do
            [ -n "$ip" ] && register_ip "$service" "$ip"
          done

          # Register bridge/gateway/node IPs
          for ip in $ALL_IPS; do
            [ -n "$ip" ] && register_ip "$service" "$ip"
          done

          echo ""
        done

        # Register IPs for 'nagios' app (used by health probes)
        echo "Registering IPs for nagios (health probes)..."
        for ip in $ALL_IPS; do
          [ -n "$ip" ] && register_ip "nagios" "$ip"
        done

        echo ""
        echo "=== IP Registration Summary ==="
        kubectl exec -n "$NAMESPACE" "$MYSQL_POD" -- mysql -u "$MYSQL_USER" -p"$MYSQL_PWD" "$MYSQL_DB" -e \
          "SELECT a.name as app_name, i.ipaddress FROM apps a JOIN app_ips_accesses i ON a.id = i.app_id WHERE i.is_deleted = 0 ORDER BY a.name, i.ipaddress" 2>/dev/null || echo "Could not retrieve summary"

        # Update allhosts ConfigMap with registered services
        echo ""
        echo "=== Updating allhosts.tsv ConfigMap ==="

        # Find the allhosts ConfigMap (name depends on Helm release/fullnameOverride)
        CONFIGMAP_NAME=$(kubectl get configmap -n "$NAMESPACE" -o name 2>/dev/null | grep -E "allhosts$" | head -1 | sed 's|configmap/||')

        if [ -n "$CONFIGMAP_NAME" ]; then
          echo "Found allhosts ConfigMap: $CONFIGMAP_NAME"

          # Get current content
          CURRENT_CONTENT=$(kubectl get configmap "$CONFIGMAP_NAME" -n "$NAMESPACE" -o jsonpath='{.data.allhosts\.tsv}')

          # Build new entries for each service
          NEW_ENTRIES=""
          for service in $SERVICES_LIST; do
            service=$(echo "$service" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
            [ -z "$service" ] && continue

            # Skip resource references
            if echo "$service" | grep -q "^resource:"; then
              continue
            fi

            # Get pod IP for this service
            POD_IP=$(kubectl get pods -n "$NAMESPACE" \
              -l "app.kubernetes.io/instance=${NAMESPACE}-${service}" \
              -o jsonpath='{.items[0].status.podIP}' 2>/dev/null) || POD_IP=""

            if [ -z "$POD_IP" ]; then
              POD_IP=$(kubectl get pods -n "$NAMESPACE" \
                -l "backstage.io/kubernetes-id=${service}" \
                -o jsonpath='{.items[0].status.podIP}' 2>/dev/null) || POD_IP=""
            fi

            if [ -n "$POD_IP" ]; then
              HOSTNAME="${service}.${NAMESPACE}.onlinesales.ai"
              # Check if entry already exists
              if ! echo "$CURRENT_CONTENT" | grep -q "^${HOSTNAME}[[:space:]]"; then
                NEW_ENTRY="${HOSTNAME}\t${POD_IP}\t${service}\tplatform\tk8s.pod"
                NEW_ENTRIES="${NEW_ENTRIES}${NEW_ENTRY}\n"
                echo "  Adding: $HOSTNAME -> $POD_IP"
              else
                echo "  Skipping (exists): $HOSTNAME"
              fi
            fi
          done

          if [ -n "$NEW_ENTRIES" ]; then
            # Append new entries to content
            UPDATED_CONTENT="${CURRENT_CONTENT}
        $(echo -e "$NEW_ENTRIES")"

            # Patch the ConfigMap using kubectl patch (avoids last-applied-configuration warning)
            kubectl patch configmap "$CONFIGMAP_NAME" -n "$NAMESPACE" \
              --type merge -p "{\"data\":{\"allhosts.tsv\":$(echo "$UPDATED_CONTENT" | jq -Rs .)}}"

            echo "ConfigMap updated successfully"
          else
            echo "No new entries to add"
          fi
        else
          echo "Warning: No allhosts ConfigMap found in namespace $NAMESPACE"
          echo "Looking for ConfigMaps ending with '-allhosts'..."
          kubectl get configmap -n "$NAMESPACE" -o name 2>/dev/null || echo "  No ConfigMaps found"
          echo "Skipping ConfigMap update"
        fi

        echo ""
        echo "Hades IP registration completed"