diff --git a/environment-provisioner/workflow-template.yaml b/environment-provisioner/workflow-template.yaml index 01d78f4..7463257 100644 --- a/environment-provisioner/workflow-template.yaml +++ b/environment-provisioner/workflow-template.yaml @@ -739,30 +739,54 @@ spec: # Wait for Hades dbinit job to complete (creates the database schema) echo "Waiting for Hades dbinit job to complete..." - DBINIT_MAX_WAIT=120 + DBINIT_MAX_WAIT=300 DBINIT_WAIT_TIME=0 + DBINIT_COMPLETED=false + while [ $DBINIT_WAIT_TIME -lt $DBINIT_MAX_WAIT ]; do # Check if dbinit job exists and is complete DBINIT_JOB=$(kubectl get jobs -n "$NAMESPACE" -o name 2>/dev/null | grep -i "hades.*dbinit" | head -1) || DBINIT_JOB="" if [ -n "$DBINIT_JOB" ]; then + echo " Found dbinit job: $DBINIT_JOB" + + # Check if job failed + DBINIT_FAILED=$(kubectl get "$DBINIT_JOB" -n "$NAMESPACE" -o jsonpath='{.status.failed}' 2>/dev/null) || DBINIT_FAILED="0" + if [ "$DBINIT_FAILED" != "" ] && [ "$DBINIT_FAILED" != "0" ]; then + echo "ERROR: Hades dbinit job failed!" + kubectl logs "$DBINIT_JOB" -n "$NAMESPACE" --tail=50 2>/dev/null || true + exit 1 + fi + + # Check if job succeeded DBINIT_COMPLETE=$(kubectl get "$DBINIT_JOB" -n "$NAMESPACE" -o jsonpath='{.status.succeeded}' 2>/dev/null) || DBINIT_COMPLETE="0" if [ "$DBINIT_COMPLETE" = "1" ]; then - echo "Hades dbinit job completed" + echo "Hades dbinit job completed successfully" + DBINIT_COMPLETED=true break fi + + # Show job status + DBINIT_ACTIVE=$(kubectl get "$DBINIT_JOB" -n "$NAMESPACE" -o jsonpath='{.status.active}' 2>/dev/null) || DBINIT_ACTIVE="0" + echo " Job status - active: $DBINIT_ACTIVE, succeeded: $DBINIT_COMPLETE, failed: $DBINIT_FAILED" + else + echo " Waiting for dbinit job to be created... (${DBINIT_WAIT_TIME}s)" fi - echo " Waiting for dbinit job... (${DBINIT_WAIT_TIME}s)" - sleep 5 - DBINIT_WAIT_TIME=$((DBINIT_WAIT_TIME + 5)) + sleep 10 + DBINIT_WAIT_TIME=$((DBINIT_WAIT_TIME + 10)) done - if [ $DBINIT_WAIT_TIME -ge $DBINIT_MAX_WAIT ]; then - echo "Warning: Hades dbinit job not completed after ${DBINIT_MAX_WAIT}s" - echo "Tables may not exist - attempting registration anyway" + if [ "$DBINIT_COMPLETED" != "true" ]; then + echo "ERROR: Hades dbinit job not completed after ${DBINIT_MAX_WAIT}s" + echo "Cannot proceed without database schema. Exiting." + exit 1 fi + # Additional wait for database to be fully ready after dbinit + echo "Waiting 10s for database schema to be fully applied..." + sleep 10 + # Helper function to run MySQL commands via kubectl exec # Using -c to specify container explicitly to avoid kubectl container selection messages MYSQL_CONTAINER=$(kubectl get pod -n "$NAMESPACE" "$MYSQL_POD" -o jsonpath='{.spec.containers[0].name}' 2>/dev/null) @@ -840,24 +864,35 @@ spec: return 0 } - # Verify required tables exist + # Verify required tables exist (with retry) echo "Checking database tables..." - TABLES=$(run_mysql "SHOW TABLES") - echo "Available tables: $TABLES" + TABLE_CHECK_MAX_WAIT=60 + TABLE_CHECK_WAIT=0 + TABLES_FOUND=false - if ! echo "$TABLES" | grep -q "apps"; then - echo "ERROR: 'apps' table not found in database" - echo "Database may not be initialized. Exiting." + while [ $TABLE_CHECK_WAIT -lt $TABLE_CHECK_MAX_WAIT ]; do + TABLES=$(run_mysql "SHOW TABLES" 2>&1) || TABLES="" + echo "Available tables: $TABLES" + + if echo "$TABLES" | grep -q "apps" && echo "$TABLES" | grep -q "app_ips_accesses"; then + echo "Required tables found!" + TABLES_FOUND=true + break + fi + + echo " Required tables not yet available, retrying... (${TABLE_CHECK_WAIT}s)" + sleep 5 + TABLE_CHECK_WAIT=$((TABLE_CHECK_WAIT + 5)) + done + + if [ "$TABLES_FOUND" != "true" ]; then + echo "ERROR: Required tables not found after ${TABLE_CHECK_MAX_WAIT}s" + echo "Missing: apps and/or app_ips_accesses tables" + echo "Database may not be initialized properly. Exiting." exit 1 fi - if ! echo "$TABLES" | grep -q "app_ips_accesses"; then - echo "ERROR: 'app_ips_accesses' table not found in database" - echo "Database may not be initialized. Exiting." - exit 1 - fi - - echo "Required tables found. Proceeding with IP registration..." + echo "Proceeding with IP registration..." echo "" # 1. Collect common bridge/gateway IPs (handles SNAT across different clusters)