From dd17a6b5dca74d2291343826c01c1842dc5f566c Mon Sep 17 00:00:00 2001
From: Vaibhav Pathak <vaibhav.pathak@Vaibhavs-MacBook-Air.local>
Date: Fri, 16 Jan 2026 16:22:51 +0530
Subject: [PATCH] intial commit

---
 README.md                                     | 174 ++++
 argocd-apps/environment-provisioner.yaml      |  26 +
 environment-provisioner/backstage-rbac.yaml   |  49 ++
 environment-provisioner/external-secrets.yaml |  71 ++
 environment-provisioner/workflow-rbac.yaml    |  77 ++
 .../workflow-template.yaml                    | 770 ++++++++++++++++++
 6 files changed, 1167 insertions(+)
 create mode 100644 README.md
 create mode 100644 argocd-apps/environment-provisioner.yaml
 create mode 100644 environment-provisioner/backstage-rbac.yaml
 create mode 100644 environment-provisioner/external-secrets.yaml
 create mode 100644 environment-provisioner/workflow-rbac.yaml
 create mode 100644 environment-provisioner/workflow-template.yaml

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..21d9f64
--- /dev/null
+++ b/README.md
@@ -0,0 +1,174 @@
+# OSMOS Infrastructure
+
+This repository contains infrastructure configurations for the OSMOS platform, managed via ArgoCD GitOps.
+
+## Repository Structure
+
+```
+osmos-infra/
+├── README.md
+├── argocd-apps/
+│   └── environment-provisioner.yaml    # ArgoCD App for provisioner
+└── environment-provisioner/
+    ├── workflow-template.yaml          # Argo Workflows WorkflowTemplate
+    ├── workflow-rbac.yaml              # ClusterRole for workflow execution
+    ├── backstage-rbac.yaml             # ClusterRole for Backstage
+    └── external-secrets.yaml           # ExternalSecret for GCP Secret Manager
+```
+
+## Environment Provisioner
+
+The environment provisioner is an Argo Workflows WorkflowTemplate that creates isolated test environments on-demand via Backstage.
+
+### Features
+
+- **Self-service environments**: Users can create test environments via Backstage UI
+- **GitOps-based**: Each environment gets its own Git repository with manifests
+- **ArgoCD managed**: Environments are synced and managed by ArgoCD
+- **Isolated namespaces**: Each environment is isolated with NetworkPolicies
+- **Automatic cleanup**: TTL-based cleanup (configurable)
+
+### Prerequisites
+
+1. **External Secrets Operator** installed with GCP Secret Manager provider
+2. **ClusterSecretStore** `gcpsm-secret-store` configured
+3. **ArgoCD** installed in `argocd` namespace
+4. **Argo Workflows** installed in `argo` namespace
+5. **Gitea** accessible for environment repositories
+
+### GCP Secret Manager Secrets Required
+
+Create these secrets in GCP Secret Manager before deployment:
+
+| Secret Name | Description |
+|-------------|-------------|
+| `gitea-url` | Gitea server URL (e.g., `https://gitea.os-tf-qa.onlinesales.ai`) |
+| `gitea-username` | Gitea admin username |
+| `gitea-password` | Gitea admin password |
+| `argocd-server` | ArgoCD server address (e.g., `argocd.os-tf-qa.onlinesales.ai:443`) |
+| `argocd-username` | ArgoCD admin username |
+| `argocd-password` | ArgoCD admin password |
+| `argo-workflows-token` | Long-lived token for Argo Workflows API |
+
+### Deployment
+
+1. **Create GCP Secret Manager secrets** (if not already created):
+   ```bash
+   # Using gcloud CLI
+   gcloud secrets create gitea-url --data-file=- <<< "https://gitea.os-tf-qa.onlinesales.ai"
+   gcloud secrets create gitea-username --data-file=- <<< "giteaAdmin"
+   gcloud secrets create gitea-password --data-file=- <<< "<password>"
+   gcloud secrets create argocd-server --data-file=- <<< "argocd.os-tf-qa.onlinesales.ai:443"
+   gcloud secrets create argocd-username --data-file=- <<< "admin"
+   gcloud secrets create argocd-password --data-file=- <<< "<password>"
+
+   # Generate Argo Workflows token and store it
+   ARGO_TOKEN=$(kubectl create token argo-server -n argo --duration=999999h)
+   gcloud secrets create argo-workflows-token --data-file=- <<< "$ARGO_TOKEN"
+   ```
+
+2. **Create repository** in Gitea:
+   ```bash
+   # Create osmos-infra repository in Gitea via UI or API
+   ```
+
+3. **Push this repository** to Gitea:
+   ```bash
+   git remote add origin https://gitea.os-tf-qa.onlinesales.ai/giteaAdmin/osmos-infra.git
+   git push -u origin main
+   ```
+
+4. **Verify ClusterSecretStore exists**:
+   ```bash
+   kubectl get clustersecretstore gcpsm-secret-store
+   ```
+
+5. **Apply ArgoCD Application**:
+   ```bash
+   kubectl apply -f argocd-apps/environment-provisioner.yaml
+   ```
+
+6. **Verify deployment**:
+   ```bash
+   # Check ExternalSecrets synced successfully
+   kubectl get externalsecret -n argo
+   kubectl get externalsecret -n backstage
+
+   # Check secrets were created
+   kubectl get secret gitea-credentials argocd-credentials -n argo
+   kubectl get secret argo-workflows-token -n backstage
+
+   # Check WorkflowTemplate
+   kubectl get workflowtemplate -n argo
+
+   # Check RBAC
+   kubectl get clusterrole workflow-executor backstage-environment-manager
+   ```
+
+### Usage
+
+Once deployed, environments can be created via:
+1. **Backstage UI**: Navigate to the Environments page and click "Create Environment"
+2. **Direct API**: Submit a workflow with the required parameters
+
+### Architecture
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                    ArgoCD (GitOps)                               │
+├─────────────────────────────────────────────────────────────────┤
+│  App: environment-provisioner-infra                              │
+│  ├── WorkflowTemplate (environment-provisioner)                  │
+│  ├── ClusterRole (workflow-executor)                             │
+│  ├── ClusterRoleBinding (workflow-executor-binding)              │
+│  ├── ClusterRole (backstage-environment-manager)                 │
+│  ├── ClusterRoleBinding (backstage-environment-manager-binding)  │
+│  └── ExternalSecrets (gitea-credentials, argocd-credentials)     │
+└─────────────────────────────────────────────────────────────────┘
+                              │
+                              ▼
+┌─────────────────────────────────────────────────────────────────┐
+│                    Argo Workflows                                │
+├─────────────────────────────────────────────────────────────────┤
+│  WorkflowTemplate: environment-provisioner                       │
+│  ├── Step 1: create-repo (creates Gitea repo with manifests)    │
+│  ├── Step 2: create-argo-app (creates ArgoCD Application)       │
+│  └── Step 3: register-hades-ips (optional health registration)  │
+└─────────────────────────────────────────────────────────────────┘
+                              │
+                              ▼
+┌─────────────────────────────────────────────────────────────────┐
+│                    Per-Environment                               │
+├─────────────────────────────────────────────────────────────────┤
+│  Gitea Repo: environment-{namespace}                             │
+│  └── manifests/                                                  │
+│      ├── namespace.yaml                                          │
+│      ├── networkpolicy.yaml                                      │
+│      └── {service}-application.yaml (ArgoCD App per service)     │
+└─────────────────────────────────────────────────────────────────┘
+```
+
+### Security
+
+- **Secrets**: All credentials are stored in GCP Secret Manager and synced via External Secrets Operator
+- **RBAC**: Minimal permissions granted to workflow and backstage service accounts
+- **TLS**: All connections use TLS in production
+- **Network Isolation**: Each environment is isolated via NetworkPolicies
+
+### Troubleshooting
+
+**ExternalSecret not syncing:**
+```bash
+kubectl describe externalsecret gitea-credentials -n argo
+```
+
+**WorkflowTemplate not found:**
+```bash
+kubectl get workflowtemplate -n argo
+kubectl describe workflowtemplate environment-provisioner -n argo
+```
+
+**Permission denied errors:**
+```bash
+kubectl auth can-i create namespaces --as=system:serviceaccount:argo:argo
+```
diff --git a/argocd-apps/environment-provisioner.yaml b/argocd-apps/environment-provisioner.yaml
new file mode 100644
index 0000000..a90a085
--- /dev/null
+++ b/argocd-apps/environment-provisioner.yaml
@@ -0,0 +1,26 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: environment-provisioner-infra
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: environment-provisioner
+    app.kubernetes.io/component: infrastructure
+    app.kubernetes.io/managed-by: argocd
+spec:
+  project: default
+  source:
+    # Update this URL to point to your osmos-infra repository in Gitea
+    repoURL: https://gitea.os-tf-qa.onlinesales.ai/giteaAdmin/osmos-infra.git
+    targetRevision: HEAD
+    path: environment-provisioner
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argo
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
diff --git a/environment-provisioner/backstage-rbac.yaml b/environment-provisioner/backstage-rbac.yaml
new file mode 100644
index 0000000..68b19bd
--- /dev/null
+++ b/environment-provisioner/backstage-rbac.yaml
@@ -0,0 +1,49 @@
+---
+# ClusterRole for Backstage service account - allows managing environments
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: backstage-environment-manager
+rules:
+  # Namespace management for environment lifecycle
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "list", "watch", "delete"]
+
+  # Read access to environment resources for display
+  - apiGroups: [""]
+    resources: ["resourcequotas", "limitranges", "pods", "services"]
+    verbs: ["get", "list", "watch"]
+
+  - apiGroups: ["apps"]
+    resources: ["deployments", "replicasets"]
+    verbs: ["get", "list", "watch"]
+
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["networkpolicies"]
+    verbs: ["get", "list", "watch"]
+
+  # ArgoCD Applications - for cleanup when deleting environments
+  - apiGroups: ["argoproj.io"]
+    resources: ["applications"]
+    verbs: ["get", "list", "watch", "delete"]
+
+  # Argo Workflows - for monitoring environment provisioning status
+  - apiGroups: ["argoproj.io"]
+    resources: ["workflows"]
+    verbs: ["get", "list", "watch"]
+
+---
+# Bind the role to backstage service account
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: backstage-environment-manager-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: backstage-environment-manager
+subjects:
+  - kind: ServiceAccount
+    name: backstage
+    namespace: backstage
diff --git a/environment-provisioner/external-secrets.yaml b/environment-provisioner/external-secrets.yaml
new file mode 100644
index 0000000..864bd4e
--- /dev/null
+++ b/environment-provisioner/external-secrets.yaml
@@ -0,0 +1,71 @@
+---
+# ExternalSecret for Gitea credentials
+# Syncs secrets from GCP Secret Manager to Kubernetes
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: gitea-credentials
+  namespace: argo
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: gcpsm-secret-store
+  target:
+    name: gitea-credentials
+    creationPolicy: Owner
+  data:
+    - secretKey: url
+      remoteRef:
+        key: gitea-url
+    - secretKey: username
+      remoteRef:
+        key: gitea-username
+    - secretKey: password
+      remoteRef:
+        key: gitea-password
+---
+# ExternalSecret for ArgoCD credentials
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: argocd-credentials
+  namespace: argo
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: gcpsm-secret-store
+  target:
+    name: argocd-credentials
+    creationPolicy: Owner
+  data:
+    - secretKey: server
+      remoteRef:
+        key: argocd-server
+    - secretKey: username
+      remoteRef:
+        key: argocd-username
+    - secretKey: password
+      remoteRef:
+        key: argocd-password
+---
+# ExternalSecret for Argo Workflows token (for Backstage)
+# This secret is created in the backstage namespace
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: argo-workflows-token
+  namespace: backstage
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: gcpsm-secret-store
+  target:
+    name: argo-workflows-token
+    creationPolicy: Owner
+  data:
+    - secretKey: token
+      remoteRef:
+        key: argo-workflows-token
diff --git a/environment-provisioner/workflow-rbac.yaml b/environment-provisioner/workflow-rbac.yaml
new file mode 100644
index 0000000..10f5137
--- /dev/null
+++ b/environment-provisioner/workflow-rbac.yaml
@@ -0,0 +1,77 @@
+---
+# ClusterRole for workflow execution - allows creating namespaces and deploying resources
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: workflow-executor
+rules:
+  # Namespace management
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["create", "get", "list", "patch", "update", "delete"]
+
+  # Apps - Deployments, DaemonSets, StatefulSets, ReplicaSets
+  - apiGroups: ["apps"]
+    resources: ["deployments", "daemonsets", "statefulsets", "replicasets"]
+    verbs: ["create", "get", "list", "watch", "patch", "update", "delete"]
+
+  # Core resources - Services, ConfigMaps, Secrets, ReplicationControllers
+  - apiGroups: [""]
+    resources: ["services", "configmaps", "secrets", "replicationcontrollers"]
+    verbs: ["create", "get", "list", "watch", "patch", "update", "delete"]
+
+  # Batch - Jobs and CronJobs
+  - apiGroups: ["batch"]
+    resources: ["jobs", "cronjobs"]
+    verbs: ["create", "get", "list", "watch", "patch", "update", "delete"]
+
+  # Autoscaling - HorizontalPodAutoscalers
+  - apiGroups: ["autoscaling"]
+    resources: ["horizontalpodautoscalers"]
+    verbs: ["get", "list", "watch"]
+
+  # Resource Quotas and Limit Ranges
+  - apiGroups: [""]
+    resources: ["resourcequotas", "limitranges"]
+    verbs: ["create", "get", "list", "patch", "update", "delete"]
+
+  # Network Policies
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["networkpolicies"]
+    verbs: ["create", "get", "list", "patch", "update", "delete"]
+
+  # Pods for verification and workflow execution
+  - apiGroups: [""]
+    resources: ["pods", "pods/log"]
+    verbs: ["get", "list", "watch", "patch", "update"]
+
+  # ArgoCD Applications (CRDs)
+  - apiGroups: ["argoproj.io"]
+    resources: ["applications"]
+    verbs: ["create", "get", "list", "patch", "update", "delete"]
+
+  # Argo Workflows - for labeling workflows
+  - apiGroups: ["argoproj.io"]
+    resources: ["workflows"]
+    verbs: ["get", "list", "patch", "update"]
+
+---
+# Bind the workflow-executor role to the default service account in argo namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: workflow-executor-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: workflow-executor
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: argo
+  - kind: ServiceAccount
+    name: argo
+    namespace: argo
+  - kind: ServiceAccount
+    name: argo-server
+    namespace: argo
diff --git a/environment-provisioner/workflow-template.yaml b/environment-provisioner/workflow-template.yaml
new file mode 100644
index 0000000..37c678a
--- /dev/null
+++ b/environment-provisioner/workflow-template.yaml
@@ -0,0 +1,770 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: environment-provisioner
+  namespace: argo
+spec:
+  serviceAccountName: argo
+  arguments:
+    parameters:
+    - name: namespace
+      value: default
+    - name: services
+      value: ""
+    - name: ttl
+      value: "8"
+    - name: owner
+      value: unknown
+    - name: componentMetadata
+      value: '{}'
+    - name: dependencyPreferences
+      value: '{}'
+    - name: valuesFileOverrides
+      value: '{}'
+    - name: manualDependencyUrls
+      value: '{}'
+    - name: helmParameterOverrides
+      value: '{}'
+    - name: helmExternalParams
+      value: '{}'
+  entrypoint: provision-environment
+  templates:
+  - inputs:
+      parameters:
+      - name: namespace
+      - name: services
+      - name: ttl
+      - name: owner
+      - name: componentMetadata
+      - name: dependencyPreferences
+      - name: valuesFileOverrides
+      - name: manualDependencyUrls
+      - name: helmParameterOverrides
+      - name: helmExternalParams
+    name: provision-environment
+    steps:
+    - - arguments:
+          parameters:
+          - name: namespace
+            value: '{{inputs.parameters.namespace}}'
+          - name: services
+            value: '{{inputs.parameters.services}}'
+          - name: ttl
+            value: '{{inputs.parameters.ttl}}'
+          - name: owner
+            value: '{{inputs.parameters.owner}}'
+          - name: componentMetadata
+            value: '{{inputs.parameters.componentMetadata}}'
+          - name: dependencyPreferences
+            value: '{{inputs.parameters.dependencyPreferences}}'
+          - name: valuesFileOverrides
+            value: '{{inputs.parameters.valuesFileOverrides}}'
+          - name: manualDependencyUrls
+            value: '{{inputs.parameters.manualDependencyUrls}}'
+          - name: helmParameterOverrides
+            value: '{{inputs.parameters.helmParameterOverrides}}'
+          - name: helmExternalParams
+            value: '{{inputs.parameters.helmExternalParams}}'
+        name: create-git-repo
+        template: create-repo
+    - - arguments:
+          parameters:
+          - name: namespace
+            value: '{{inputs.parameters.namespace}}'
+          - name: owner
+            value: '{{inputs.parameters.owner}}'
+        name: create-argocd-app
+        template: create-argo-app
+    - - arguments:
+          parameters:
+          - name: namespace
+            value: '{{inputs.parameters.namespace}}'
+          - name: services
+            value: '{{inputs.parameters.services}}'
+        name: register-hades-ips
+        template: register-hades-ips
+  - inputs:
+      parameters:
+      - name: namespace
+      - name: services
+      - name: ttl
+      - name: owner
+      - name: componentMetadata
+      - name: dependencyPreferences
+      - name: valuesFileOverrides
+      - name: manualDependencyUrls
+      - name: helmParameterOverrides
+      - name: helmExternalParams
+    name: create-repo
+    script:
+      command:
+      - sh
+      env:
+      # Credentials from External Secrets (GCP Secret Manager)
+      - name: GITEA_URL
+        valueFrom:
+          secretKeyRef:
+            name: gitea-credentials
+            key: url
+      - name: GITEA_USER
+        valueFrom:
+          secretKeyRef:
+            name: gitea-credentials
+            key: username
+      - name: GITEA_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: gitea-credentials
+            key: password
+      image: alpine/git:latest
+      source: |
+        #!/bin/sh
+        set -e
+
+        NAMESPACE="{{inputs.parameters.namespace}}"
+        SERVICES="{{inputs.parameters.services}}"
+        TTL="{{inputs.parameters.ttl}}"
+        OWNER="{{inputs.parameters.owner}}"
+        COMPONENT_METADATA='{{inputs.parameters.componentMetadata}}'
+        DEPENDENCY_PREFS='{{inputs.parameters.dependencyPreferences}}'
+        VALUES_FILE_OVERRIDES='{{inputs.parameters.valuesFileOverrides}}'
+        MANUAL_DEPENDENCY_URLS='{{inputs.parameters.manualDependencyUrls}}'
+        HELM_PARAMETER_OVERRIDES='{{inputs.parameters.helmParameterOverrides}}'
+        HELM_EXTERNAL_PARAMS='{{inputs.parameters.helmExternalParams}}'
+
+        # Sanitize namespace name (replace spaces and special chars with hyphens)
+        NAMESPACE=$(echo "$NAMESPACE" | tr '[:upper:]' '[:lower:]' | tr ' _' '-' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')
+
+        # Sanitize owner name for labels (Kubernetes labels have strict requirements)
+        OWNER_LABEL=$(echo "$OWNER" | tr '[:upper:]' '[:lower:]' | tr ' _' '-' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')
+
+        echo "Creating Git repository for namespace: $NAMESPACE"
+
+        # Install curl and jq
+        apk add --no-cache curl jq
+
+        # Create repository in Gitea
+        REPO_NAME="environment-$NAMESPACE"
+
+        # Extract hostname from GITEA_URL for git credentials
+        GITEA_HOST=$(echo "$GITEA_URL" | sed 's|https://||' | sed 's|http://||' | cut -d'/' -f1)
+
+        echo "Creating Gitea repository: $REPO_NAME"
+        curl -X POST "$GITEA_URL/api/v1/user/repos" \
+          -u "$GITEA_USER:$GITEA_PASSWORD" \
+          -H "Content-Type: application/json" \
+          -d "{
+            \"name\": \"$REPO_NAME\",
+            \"description\": \"Environment for $NAMESPACE (owner: $OWNER, ttl: ${TTL}h)\",
+            \"private\": false,
+            \"auto_init\": true
+          }" || echo "Repository may already exist"
+
+        # Clone the repository
+        git config --global http.sslVerify true
+        git config --global user.email "backstage@osmos.io"
+        git config --global user.name "Backstage"
+
+        # URL encode the password for git credentials
+        ENCODED_PASSWORD=$(printf %s "$GITEA_PASSWORD" | jq -sRr @uri)
+
+        # Configure credentials
+        git config --global credential.helper store
+        printf "https://%s:%s@%s\n" "$GITEA_USER" "$ENCODED_PASSWORD" "$GITEA_HOST" > $HOME/.git-credentials
+
+        REPO_URL="$GITEA_URL/giteaAdmin/$REPO_NAME.git"
+        git clone "$REPO_URL" /tmp/repo 2>&1 || (cd /tmp/repo && git pull 2>&1)
+        cd /tmp/repo
+
+        # Create manifests directory
+        mkdir -p manifests
+
+        # Create namespace manifest
+        cat > manifests/namespace.yaml <<EOF
+        apiVersion: v1
+        kind: Namespace
+        metadata:
+          name: $NAMESPACE
+          labels:
+            name: $NAMESPACE
+            environment: test
+            owner: $OWNER_LABEL
+            ttl: ${TTL}h
+            provisioned-by: backstage
+            managed-by: argocd
+          annotations:
+            ttl-hours: "$TTL"
+            created-by: backstage-environment-provisioner
+            description: "Test environment for $OWNER"
+            original-owner: "$OWNER"
+        EOF
+
+        # Create NetworkPolicy for isolation (allows ingress from same namespace and ingress-nginx)
+        cat > manifests/networkpolicy.yaml <<EOF
+        apiVersion: networking.k8s.io/v1
+        kind: NetworkPolicy
+        metadata:
+          name: default-deny-ingress
+          namespace: $NAMESPACE
+        spec:
+          podSelector: {}
+          policyTypes:
+          - Ingress
+          ingress:
+          - from:
+            - namespaceSelector:
+                matchLabels:
+                  name: $NAMESPACE
+          - from:
+            - namespaceSelector:
+                matchLabels:
+                  kubernetes.io/metadata.name: ingress-nginx
+        EOF
+
+        # Create service manifests
+        # Convert comma-separated services to space-separated for iteration
+        SERVICES_LIST=$(echo "$SERVICES" | tr ',' ' ')
+        for service in $SERVICES_LIST; do
+          service=$(echo "$service" | xargs) # trim whitespace
+          if [ -n "$service" ]; then
+            # Skip resource references (already filtered in frontend, but double-check)
+            if echo "$service" | grep -q "^resource:"; then
+              echo "Skipping resource reference: $service"
+              continue
+            fi
+
+            echo "Processing service: $service"
+
+            # Sanitize service name for filename (remove special chars)
+            SANITIZED_SERVICE=$(echo "$service" | tr '[:upper:]' '[:lower:]' | tr ' /:' '-' | sed 's/[^a-z0-9-]//g')
+
+            # Create truncated app name (Helm release names must be <= 53 chars)
+            APP_NAME=$(echo "$NAMESPACE-$SANITIZED_SERVICE" | cut -c1-53 | sed 's/-$//')
+
+            # Check if component has Helm chart annotations
+            HELM_REPO=$(echo "$COMPONENT_METADATA" | jq -r ".[\"$service\"][\"helm.cnoe.io/git-repo\"] // empty")
+            HELM_PATH=$(echo "$COMPONENT_METADATA" | jq -r ".[\"$service\"][\"helm.cnoe.io/chart-path\"] // empty")
+            HELM_VALUES_DEFAULT=$(echo "$COMPONENT_METADATA" | jq -r ".[\"$service\"][\"helm.cnoe.io/values-file\"] // \"values.yaml\"")
+            HELM_VALUES=$(echo "$VALUES_FILE_OVERRIDES" | jq -r ".[\"$service\"] // \"$HELM_VALUES_DEFAULT\"")
+
+            # DEBUG: Log what we're checking
+            echo "  Checking for Helm annotations..."
+            echo "  HELM_REPO: $HELM_REPO"
+            echo "  HELM_PATH: $HELM_PATH"
+            echo "  HELM_VALUES_DEFAULT: $HELM_VALUES_DEFAULT"
+            echo "  HELM_VALUES (after override): $HELM_VALUES"
+
+            if [ -n "$HELM_REPO" ]; then
+              # Component has Helm chart - create ArgoCD Application manifest
+              echo "Creating ArgoCD Application for Helm chart: $service"
+
+              # Check dependency preferences for MySQL
+              MYSQL_ENABLED=$(echo "$DEPENDENCY_PREFS" | jq -r '.mysql // "true"')
+              echo "  MySQL enabled: $MYSQL_ENABLED"
+
+              # Process dependency URL mappings
+              DEPENDENCY_URL_PARAMS=""
+              DEPENDENCY_URLS=$(echo "$COMPONENT_METADATA" | jq -r ".[\"$service\"][\"helm.cnoe.io/dependency-urls\"] // empty")
+
+              if [ -n "$DEPENDENCY_URLS" ]; then
+                echo "  Processing dependency URL mappings..."
+
+                for dep_key in $(echo "$DEPENDENCY_URLS" | jq -r 'keys[]'); do
+                  echo "    Checking dependency: $dep_key"
+
+                  # Check if this dependency is in our resolved services list
+                  DEP_IN_SERVICES=$(echo "$SERVICES" | tr ',' '\n' | grep -c "^${dep_key}$" || echo "0")
+
+                  # Get dependency configuration
+                  SERVICE_DNS=$(echo "$DEPENDENCY_URLS" | jq -r ".[\"$dep_key\"].serviceDns // \"$dep_key\"")
+                  SERVICE_PORT=$(echo "$DEPENDENCY_URLS" | jq -r ".[\"$dep_key\"].port // 8080")
+                  HELM_PARAM=$(echo "$DEPENDENCY_URLS" | jq -r ".[\"$dep_key\"].helmParam")
+
+                  # Sanitize service DNS name
+                  SERVICE_DNS=$(echo "$SERVICE_DNS" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]//g')
+
+                  CONSTRUCTED_URL=""
+
+                  if [ "$DEP_IN_SERVICES" -gt 0 ]; then
+                    # Dependency is being deployed to same namespace - use short DNS
+                    echo "    Dependency $dep_key is being deployed to same namespace"
+                    CONSTRUCTED_URL="http://${SERVICE_DNS}:${SERVICE_PORT}"
+                  else
+                    # Check for manual URL override
+                    MANUAL_URL=$(echo "$MANUAL_DEPENDENCY_URLS" | jq -r ".[\"$service\"][\"$HELM_PARAM\"] // empty")
+                    if [ -n "$MANUAL_URL" ]; then
+                      echo "    Using manual URL for $dep_key"
+                      CONSTRUCTED_URL="$MANUAL_URL"
+                    else
+                      echo "    Dependency $dep_key NOT in deployed services and no manual URL - using defaults"
+                    fi
+                  fi
+
+                  if [ -n "$CONSTRUCTED_URL" ] && [ -n "$HELM_PARAM" ]; then
+                    echo "    Setting $HELM_PARAM = $CONSTRUCTED_URL"
+                    DEPENDENCY_URL_PARAMS="$DEPENDENCY_URL_PARAMS
+                - name: $HELM_PARAM
+                  value: \"$CONSTRUCTED_URL\""
+                  fi
+                done
+              fi
+
+              # Process custom Helm parameter overrides
+              CUSTOM_HELM_PARAMS=""
+              SERVICE_OVERRIDES=$(echo "$HELM_PARAMETER_OVERRIDES" | jq -r ".[\"$service\"] // {}")
+              if [ "$SERVICE_OVERRIDES" != "{}" ] && [ -n "$SERVICE_OVERRIDES" ]; then
+                echo "  Processing custom Helm parameter overrides..."
+                for param_key in $(echo "$SERVICE_OVERRIDES" | jq -r 'keys[]'); do
+                  PARAM_VALUE=$(echo "$SERVICE_OVERRIDES" | jq -r ".[\"$param_key\"]")
+                  echo "    Setting $param_key = $PARAM_VALUE"
+                  # Escape dots in parameter names for Helm
+                  ESCAPED_KEY=$(echo "$param_key" | sed 's/\./\\./g')
+                  CUSTOM_HELM_PARAMS="$CUSTOM_HELM_PARAMS
+                - name: $ESCAPED_KEY
+                  value: \"$PARAM_VALUE\""
+                done
+              fi
+
+              # Process external params for disabled Helm dependencies (e.g., external database config)
+              SERVICE_EXTERNAL_PARAMS=$(echo "$HELM_EXTERNAL_PARAMS" | jq -r ".[\"$service\"] // {}")
+              if [ "$SERVICE_EXTERNAL_PARAMS" != "{}" ] && [ -n "$SERVICE_EXTERNAL_PARAMS" ]; then
+                echo "  Processing external params for disabled dependencies..."
+                for param_key in $(echo "$SERVICE_EXTERNAL_PARAMS" | jq -r 'keys[]'); do
+                  PARAM_VALUE=$(echo "$SERVICE_EXTERNAL_PARAMS" | jq -r ".[\"$param_key\"]")
+                  if [ -n "$PARAM_VALUE" ]; then
+                    echo "    Setting $param_key = $PARAM_VALUE"
+                    # Escape dots in parameter names for Helm
+                    ESCAPED_KEY=$(echo "$param_key" | sed 's/\./\\./g')
+                    CUSTOM_HELM_PARAMS="$CUSTOM_HELM_PARAMS
+                - name: $ESCAPED_KEY
+                  value: \"$PARAM_VALUE\""
+                  fi
+                done
+              fi
+
+              # Extract ingress hostname from GITEA_URL domain
+              INGRESS_DOMAIN=$(echo "$GITEA_URL" | sed 's|https://gitea\.||' | sed 's|http://gitea\.||')
+
+              cat > "manifests/$APP_NAME-application.yaml" <<EOF
+        apiVersion: argoproj.io/v1alpha1
+        kind: Application
+        metadata:
+          name: $APP_NAME
+          namespace: argocd
+          labels:
+            environment: $NAMESPACE
+            provisioned-by: backstage
+        spec:
+          project: default
+          source:
+            repoURL: $HELM_REPO
+            targetRevision: HEAD
+            path: $HELM_PATH
+            helm:
+              valueFiles:
+                - $HELM_VALUES
+              parameters:
+                - name: mysql.enabled
+                  value: "$MYSQL_ENABLED"
+                # Override fullname to keep K8s resource names short (max 63 chars)
+                - name: fullnameOverride
+                  value: "$SANITIZED_SERVICE"
+                # Add commonLabels for hades resources
+                - name: commonLabels.backstage\.io/kubernetes-id
+                  value: "$service"
+                # Add podLabels for MySQL resources
+                - name: mysql.podLabels.backstage\.io/kubernetes-id
+                  value: "$service"
+                # Enable ingress via Helm chart
+                - name: ingress.enabled
+                  value: "true"
+                - name: ingress.className
+                  value: "nginx"
+                - name: ingress.hosts[0].host
+                  value: "${SANITIZED_SERVICE}-${NAMESPACE}.${INGRESS_DOMAIN}"
+                - name: ingress.hosts[0].paths[0].path
+                  value: "/"
+                - name: ingress.hosts[0].paths[0].pathType
+                  value: "Prefix"$DEPENDENCY_URL_PARAMS$CUSTOM_HELM_PARAMS
+          destination:
+            server: https://kubernetes.default.svc
+            namespace: $NAMESPACE
+          syncPolicy:
+            automated:
+              prune: true
+              selfHeal: true
+            syncOptions:
+              - CreateNamespace=false
+              - ServerSideApply=true
+        EOF
+            echo "  Ingress URL: http://${SANITIZED_SERVICE}-${NAMESPACE}.${INGRESS_DOMAIN}"
+            else
+              # No Helm chart - create simple deployment manifest
+              echo "Creating deployment manifest for service: $service"
+
+              cat > "manifests/$SANITIZED_SERVICE-deployment.yaml" <<EOF
+        apiVersion: apps/v1
+        kind: Deployment
+        metadata:
+          name: $service
+          namespace: $NAMESPACE
+          labels:
+            app: $service
+            provisioned-by: backstage
+            managed-by: argocd
+        spec:
+          replicas: 1
+          selector:
+            matchLabels:
+              app: $service
+          template:
+            metadata:
+              labels:
+                app: $service
+            spec:
+              containers:
+              - name: $service
+                image: nginx:alpine
+                ports:
+                - containerPort: 80
+                  name: http
+                resources:
+                  limits:
+                    cpu: 500m
+                    memory: 512Mi
+                  requests:
+                    cpu: 100m
+                    memory: 128Mi
+                livenessProbe:
+                  httpGet:
+                    path: /
+                    port: http
+                  initialDelaySeconds: 10
+                  periodSeconds: 10
+                readinessProbe:
+                  httpGet:
+                    path: /
+                    port: http
+                  initialDelaySeconds: 5
+                  periodSeconds: 5
+        ---
+        apiVersion: v1
+        kind: Service
+        metadata:
+          name: $service
+          namespace: $NAMESPACE
+          labels:
+            app: $service
+            provisioned-by: backstage
+            managed-by: argocd
+        spec:
+          type: ClusterIP
+          selector:
+            app: $service
+          ports:
+          - port: 80
+            targetPort: http
+            protocol: TCP
+            name: http
+        EOF
+            fi
+          fi
+        done
+
+        # Create README
+        cat > README.md <<EOF
+        # Environment: $NAMESPACE
+
+        **Owner:** $OWNER
+        **TTL:** $TTL hours
+        **Services:** $SERVICES
+
+        ## Description
+        This is an automatically provisioned test environment managed by ArgoCD.
+
+        ## Resources
+        - Namespace with resource quotas and limits
+        - Network policies for isolation
+        - Deployed services: $(echo $SERVICES | tr ',' ' ')
+
+        ## Management
+        This environment is managed by ArgoCD. Any changes pushed to this repository will be automatically synced to the cluster.
+
+        **Note:** This environment will be cleaned up after $TTL hours.
+        EOF
+
+        # Commit and push
+        git add .
+        git commit -m "Initial environment setup for $NAMESPACE by $OWNER"
+        git push origin main || git push origin master
+
+        echo "Git repository created and pushed successfully"
+        echo "Repository URL: $GITEA_URL/giteaAdmin/$REPO_NAME"
+  - inputs:
+      parameters:
+      - name: namespace
+      - name: owner
+    name: create-argo-app
+    script:
+      command:
+      - sh
+      env:
+      # Credentials from External Secrets (GCP Secret Manager)
+      - name: ARGOCD_SERVER
+        valueFrom:
+          secretKeyRef:
+            name: argocd-credentials
+            key: server
+      - name: ARGOCD_USERNAME
+        valueFrom:
+          secretKeyRef:
+            name: argocd-credentials
+            key: username
+      - name: ARGOCD_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: argocd-credentials
+            key: password
+      - name: GITEA_URL
+        valueFrom:
+          secretKeyRef:
+            name: gitea-credentials
+            key: url
+      image: argoproj/argocd:latest
+      source: |
+        #!/bin/sh
+        set -e
+
+        NAMESPACE="{{inputs.parameters.namespace}}"
+        OWNER="{{inputs.parameters.owner}}"
+
+        # Sanitize namespace name (replace spaces and special chars with hyphens)
+        NAMESPACE=$(echo "$NAMESPACE" | tr '[:upper:]' '[:lower:]' | tr ' _' '-' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')
+
+        # Sanitize owner name for label (Kubernetes labels have strict requirements)
+        OWNER_LABEL=$(echo "$OWNER" | tr '[:upper:]' '[:lower:]' | tr ' _' '-' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')
+
+        REPO_NAME="environment-$NAMESPACE"
+
+        echo "Creating ArgoCD application for namespace: $NAMESPACE"
+
+        # Login to ArgoCD (TLS enabled for production)
+        argocd login $ARGOCD_SERVER \
+          --username $ARGOCD_USERNAME \
+          --password $ARGOCD_PASSWORD \
+          --grpc-web
+
+        # Create ArgoCD application
+        argocd app create "$NAMESPACE-environment" \
+          --repo "$GITEA_URL/giteaAdmin/$REPO_NAME.git" \
+          --path manifests \
+          --dest-server https://kubernetes.default.svc \
+          --dest-namespace $NAMESPACE \
+          --sync-policy automated \
+          --auto-prune \
+          --self-heal \
+          --sync-option CreateNamespace=true \
+          --upsert \
+          --label "owner=$OWNER_LABEL" \
+          --label "provisioned-by=backstage" \
+          --label "managed-by=argocd"
+
+        echo "ArgoCD application created successfully"
+        echo "Auto-sync is enabled - ArgoCD will automatically sync the application"
+        echo "View in ArgoCD UI: https://$ARGOCD_SERVER/applications/$NAMESPACE-environment"
+  - inputs:
+      parameters:
+      - name: namespace
+      - name: services
+    name: register-hades-ips
+    script:
+      command:
+      - bash
+      image: bitnami/kubectl:latest
+      source: |
+        #!/bin/bash
+        set -e
+
+        NAMESPACE="{{inputs.parameters.namespace}}"
+        SERVICES="{{inputs.parameters.services}}"
+
+        # Sanitize namespace
+        NAMESPACE=$(echo "$NAMESPACE" | tr '[:upper:]' '[:lower:]' | tr ' _' '-' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')
+
+        # MySQL connection details for Hades
+        MYSQL_HOST="hades-mysql-db-service.${NAMESPACE}.svc.cluster.local"
+        MYSQL_PORT="3306"
+        MYSQL_USER="root"
+        MYSQL_PWD="local-root-pass"
+        MYSQL_DB="hades_db"
+
+        echo "=== Hades IP Registration ==="
+        echo "Namespace: $NAMESPACE"
+        echo "MySQL Host: $MYSQL_HOST"
+        echo ""
+
+        # kubectl is already available in this image
+
+        # Find the Hades MySQL pod
+        MYSQL_POD=$(kubectl get pods -n "$NAMESPACE" -l "app.kubernetes.io/name=mysql" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
+
+        if [ -z "$MYSQL_POD" ]; then
+          # Try alternative label
+          MYSQL_POD=$(kubectl get pods -n "$NAMESPACE" -o name 2>/dev/null | grep -i "mysql" | head -1 | sed 's/pod\///')
+        fi
+
+        if [ -z "$MYSQL_POD" ]; then
+          echo "Warning: Could not find Hades MySQL pod"
+          echo "Skipping IP registration - Hades may not be deployed"
+          exit 0
+        fi
+
+        echo "Found MySQL pod: $MYSQL_POD"
+
+        # Wait for Hades MySQL to be ready (should already be running from previous steps)
+        echo "Waiting for Hades MySQL to be ready..."
+        MAX_WAIT=120
+        WAIT_TIME=0
+        while [ $WAIT_TIME -lt $MAX_WAIT ]; do
+          if kubectl exec -n "$NAMESPACE" "$MYSQL_POD" -- mysql -u "$MYSQL_USER" -p"$MYSQL_PWD" -e "SELECT 1" "$MYSQL_DB" > /dev/null 2>&1; then
+            echo "Hades MySQL is ready"
+            break
+          fi
+          echo "  Waiting for MySQL... (${WAIT_TIME}s)"
+          sleep 5
+          WAIT_TIME=$((WAIT_TIME + 5))
+        done
+
+        if [ $WAIT_TIME -ge $MAX_WAIT ]; then
+          echo "Warning: Could not connect to Hades MySQL after ${MAX_WAIT}s"
+          echo "Skipping IP registration - Hades may not be deployed"
+          exit 0
+        fi
+
+        # Wait for Hades dbinit job to complete (creates the database schema)
+        echo "Waiting for Hades dbinit job to complete..."
+        DBINIT_MAX_WAIT=120
+        DBINIT_WAIT_TIME=0
+        while [ $DBINIT_WAIT_TIME -lt $DBINIT_MAX_WAIT ]; do
+          # Check if dbinit job exists and is complete
+          DBINIT_JOB=$(kubectl get jobs -n "$NAMESPACE" -o name 2>/dev/null | grep -i "hades.*dbinit" | head -1) || DBINIT_JOB=""
+
+          if [ -n "$DBINIT_JOB" ]; then
+            DBINIT_COMPLETE=$(kubectl get "$DBINIT_JOB" -n "$NAMESPACE" -o jsonpath='{.status.succeeded}' 2>/dev/null) || DBINIT_COMPLETE="0"
+            if [ "$DBINIT_COMPLETE" = "1" ]; then
+              echo "Hades dbinit job completed"
+              break
+            fi
+          fi
+
+          echo "  Waiting for dbinit job... (${DBINIT_WAIT_TIME}s)"
+          sleep 5
+          DBINIT_WAIT_TIME=$((DBINIT_WAIT_TIME + 5))
+        done
+
+        if [ $DBINIT_WAIT_TIME -ge $DBINIT_MAX_WAIT ]; then
+          echo "Warning: Hades dbinit job not completed after ${DBINIT_MAX_WAIT}s"
+          echo "Tables may not exist - attempting registration anyway"
+        fi
+
+        # Helper function to run MySQL commands via kubectl exec
+        run_mysql() {
+          kubectl exec -n "$NAMESPACE" "$MYSQL_POD" -- mysql -u "$MYSQL_USER" -p"$MYSQL_PWD" "$MYSQL_DB" -sN -e "$1" 2>/dev/null || true
+        }
+
+        # Function to register IP for an app
+        register_ip() {
+          local app_name="$1"
+          local ip="$2"
+
+          if [ -z "$app_name" ] || [ -z "$ip" ]; then
+            return 0
+          fi
+
+          # Get or create app_id
+          APP_ID=$(run_mysql "SELECT id FROM apps WHERE name='$app_name'" || echo "")
+
+          if [ -z "$APP_ID" ]; then
+            echo "  Creating app entry for: $app_name"
+            run_mysql "INSERT INTO apps (name, token, creation_date, last_update, is_deleted) VALUES ('$app_name', UUID(), NOW(), NOW(), 0)" || true
+            APP_ID=$(run_mysql "SELECT id FROM apps WHERE name='$app_name'" || echo "")
+          fi
+
+          if [ -n "$APP_ID" ]; then
+            # Check if IP already exists
+            EXISTING=$(run_mysql "SELECT id FROM app_ips_accesses WHERE app_id=$APP_ID AND ipaddress='$ip'" || echo "")
+
+            if [ -z "$EXISTING" ]; then
+              echo "  Registering IP $ip for app $app_name (app_id: $APP_ID)"
+              run_mysql "INSERT INTO app_ips_accesses (app_id, ipaddress, creation_date, last_update, is_deleted) VALUES ($APP_ID, '$ip', NOW(), NOW(), 0)" || true
+            else
+              echo "  IP $ip already registered for $app_name"
+            fi
+          fi
+          return 0
+        }
+
+        # 1. Collect common bridge/gateway IPs (handles SNAT across different clusters)
+        COMMON_BRIDGE_IPS="10.244.0.1 10.96.0.1 172.17.0.1"
+        echo "Common bridge IPs: $COMMON_BRIDGE_IPS"
+
+        # 2. Get node internal IPs
+        NODE_IPS=$(kubectl get nodes -o jsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null || echo "")
+        echo "Node IPs: $NODE_IPS"
+
+        # 3. Get pod network CIDR gateway
+        POD_CIDR=$(kubectl get nodes -o jsonpath='{.items[0].spec.podCIDR}' 2>/dev/null || echo "")
+        if [ -n "$POD_CIDR" ]; then
+          GATEWAY_IP=$(echo "$POD_CIDR" | sed 's/\.[0-9]*\/.*/.1/')
+          COMMON_BRIDGE_IPS="$COMMON_BRIDGE_IPS $GATEWAY_IP"
+          echo "Pod CIDR gateway: $GATEWAY_IP"
+        fi
+
+        # Combine all IPs to register
+        ALL_IPS="$COMMON_BRIDGE_IPS $NODE_IPS"
+        echo ""
+
+        # Process each service
+        SERVICES_LIST=$(echo "$SERVICES" | tr ',' ' ')
+        for service in $SERVICES_LIST; do
+          service=$(echo "$service" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+          [ -z "$service" ] && continue
+
+          # Skip resource references
+          if echo "$service" | grep -q "^resource:"; then
+            continue
+          fi
+
+          echo "Processing service: $service"
+
+          # Get actual pod IPs for this service
+          POD_IPS=$(kubectl get pods -n "$NAMESPACE" \
+            -l "app.kubernetes.io/instance=${service}" \
+            -o jsonpath='{.items[*].status.podIP}' 2>/dev/null) || POD_IPS=""
+
+          # Register pod IPs
+          for ip in $POD_IPS; do
+            [ -n "$ip" ] && register_ip "$service" "$ip"
+          done
+
+          # Register bridge/gateway/node IPs
+          for ip in $ALL_IPS; do
+            [ -n "$ip" ] && register_ip "$service" "$ip"
+          done
+
+          echo ""
+        done
+
+        # Register IPs for 'nagios' app (used by health probes)
+        echo "Registering IPs for nagios (health probes)..."
+        for ip in $ALL_IPS; do
+          [ -n "$ip" ] && register_ip "nagios" "$ip"
+        done
+
+        echo ""
+        echo "=== IP Registration Summary ==="
+        kubectl exec -n "$NAMESPACE" "$MYSQL_POD" -- mysql -u "$MYSQL_USER" -p"$MYSQL_PWD" "$MYSQL_DB" -e \
+          "SELECT a.name as app_name, i.ipaddress FROM apps a JOIN app_ips_accesses i ON a.id = i.app_id WHERE i.is_deleted = 0 ORDER BY a.name, i.ipaddress" 2>/dev/null || echo "Could not retrieve summary"
+
+        echo ""
+        echo "Hades IP registration completed"