From 7c11aa222f4e791e68d688a003bc449b2f3f9631 Mon Sep 17 00:00:00 2001 From: Tohru Hasegawa Date: Fri, 7 Aug 2026 18:11:31 +0900 Subject: [PATCH] feat(storage): add multi-realm RGW support and phase-resume fixes Key changes ----------- - Add --rgw-realm option for multi-realm Ceph clusters (native mode). Required when the cluster has named realms instead of 'default'. - Rewrite detect_rgw_zone() for reliable zone resolution: - Use 'radosgw-admin zone get --rgw-realm=' when --rgw-realm is given (default_info in zone list contains a UUID, not the name). - Redirect debug/warning to stderr so the zone name captured via $() is not polluted. - Extract zone name from the 'zones' array in the fallback path. - Resolve RGW_HOST from short hostname to IP address: 'ceph orch ps' returns a short hostname that is typically not resolvable from an external tenant cluster. Use 'ceph orch host ls -f json' + jq to map the hostname to its public IP address automatically. If resolution fails the short hostname is kept as-is. - Auto-detect RGW protocol (http/https) from 'ceph config dump' by checking the rgw_frontends config for ssl_port / ssl_certificate. - Rename ceph-external-cluster-details-exporter.py references to create-external-cluster-resources.py (current Rook/ODF script name). - Restore RGW credentials and MAIN_RGW_ENDPOINT when resuming from phase 6, 7, or 8 (previously caused failures on --phase N reruns). Falls back to 'ceph orch ps --daemon-type rgw' detection if the credentials file does not yet contain the RGW_ENDPOINT entry. - Add MAIN_RGW_ENDPOINT_OVERRIDE env var to allow explicit endpoint override without modifying the script. - Add --connect-timeout/--max-time to backing-bucket curl call for reliable timeout behaviour on slow/unresponsive endpoints. - Delete state file in cleanup_artifacts() so the script can be re-run from scratch after --delete. --- scripts/setup-storage.sh | 477 +++++++++++++++++++++++++++++---------- 1 file changed, 363 insertions(+), 114 deletions(-) diff --git a/scripts/setup-storage.sh b/scripts/setup-storage.sh index 61fb528..da90979 100755 --- a/scripts/setup-storage.sh +++ b/scripts/setup-storage.sh @@ -99,6 +99,7 @@ LOG_LEVEL="INFO" CEPH_MON_HOST="" CEPH_FSID="" RGW_REGION="" # Will be set based on deployment or default to "default" +RGW_REALM="" # RGW realm for multi-realm clusters (e.g. realm-ocp1) RGW_PROTOCOL="http" MAIN_RGW_ENDPOINT="" ENABLE_OBJECT_STORAGE=false @@ -351,6 +352,8 @@ while [[ $# -gt 0 ]]; do STORAGECLUSTER_NAME="$2"; shift 2 ;; --region|--rgw-region) RGW_REGION="$2"; shift 2 ;; + --rgw-realm) + RGW_REALM="$2"; shift 2 ;; --ca-bundle-path) CUSTOM_CA_BUNDLE_PATH="$2"; shift 2 ;; --resume) @@ -394,6 +397,9 @@ ${COLOR_BOLD}OBJECT STORAGE (OPTIONAL)${COLOR_RESET} Note: Requires --rbd-quota for NooBaa PostgreSQL PVC --region REGION S3 region for RGW (default: us-east-1) --rgw-region REGION Alias for --region + --rgw-realm REALM RGW realm name for multi-realm clusters (native mode) + Required when the cluster has named realms (not "default") + Example: --rgw-realm realm-ocp1 ${COLOR_BOLD}HTTPS/TLS OPTIONS${COLOR_RESET} --ca-bundle-path PATH Path to custom CA certificate bundle for HTTPS endpoints @@ -1129,35 +1135,60 @@ convert_from_bytes() { # Detect the correct RGW zone for user creation detect_rgw_zone() { local zone_list - local default_zone - - print_debug "Detecting RGW zone..." - - # Get list of zones - zone_list=$(ceph_exec radosgw-admin zone list 2>&1 || echo "") - + local detected_zone + + print_debug "Detecting RGW zone..." >&2 + + # If --rgw-realm was specified, use `zone get` to obtain the default zone + # name for that realm. `zone list` returns UUIDs in default_info, not names, + # so `zone get --rgw-realm=` (which returns .name) is the reliable path. + if [ -n "${RGW_REALM}" ]; then + local zone_get_out + zone_get_out=$(ceph_exec radosgw-admin zone get --rgw-realm="${RGW_REALM}" 2>/dev/null || echo "") + detected_zone=$(echo "$zone_get_out" | sed -n 's/.*"name":[[:space:]]*"\([^"]*\)".*/\1/p' | head -1) + if [ -n "$detected_zone" ]; then + print_debug "Using default zone for realm '${RGW_REALM}': ${detected_zone}" >&2 + echo "$detected_zone" + else + # Fall back: list zones and pick the first matching zone-* name + zone_list=$(ceph_exec radosgw-admin zone list --rgw-realm="${RGW_REALM}" 2>/dev/null || echo "") + detected_zone=$(echo "$zone_list" | sed -n 's/.*"\(zone-[^"]*\)".*/\1/p' | head -1) + if [ -n "$detected_zone" ]; then + print_debug "Using first zone for realm '${RGW_REALM}': ${detected_zone}" >&2 + echo "$detected_zone" + else + print_warning "Could not detect zone for realm '${RGW_REALM}', deriving from realm name" >&2 + echo "${RGW_REALM/realm-/zone-}" + fi + fi + return + fi + + # No realm specified — use global zone list (works for single-realm / "default" zone clusters) + zone_list=$(ceph_exec radosgw-admin zone list 2>/dev/null || echo "") + if [ -z "$zone_list" ]; then - print_warning "Could not detect RGW zones, using default" + print_warning "Could not detect RGW zones, using default" >&2 echo "default" return fi - - # Check if storage cluster CephObjectStore zone exists + + # Check if storage cluster CephObjectStore zone exists (ODF mode) local rgw_zone_name="${STORAGECLUSTER_NAME}-cephobjectstore" if echo "$zone_list" | grep -q "$rgw_zone_name"; then - print_debug "Found ODF zone: ${rgw_zone_name}" + print_debug "Found ODF zone: ${rgw_zone_name}" >&2 echo "${rgw_zone_name}" return fi - - # Get default zone - default_zone=$(echo "$zone_list" | sed -n 's/.*"default_info":[[:space:]]*"\([^"]*\)".*/\1/p' | head -1) - - if [ -n "$default_zone" ]; then - print_debug "Using default zone from zone list" - echo "default" + + # Extract the first zone name from the "zones" array. + # NOTE: "default_info" contains a UUID, NOT the zone name — do not use it. + detected_zone=$(echo "$zone_list" | sed -n 's/.*"zones":[[:space:]]*\[.*"\([^"]*\)".*/\1/p' | head -1) + if [ -n "$detected_zone" ]; then + print_debug "Using first zone from zones array: ${detected_zone}" >&2 + echo "$detected_zone" else - print_debug "No specific zone detected, using 'default'" + print_debug "No specific zone detected, using 'default'" >&2 echo "default" fi } @@ -1632,7 +1663,7 @@ update_tenant_storage() { print_success "External config saved to: ${OUTPUT_JSON}" else - # Native Ceph mode: use ceph-external-cluster-details-exporter.py + # Native Ceph mode: use create-external-cluster-resources.py print_info "Generating external config for native Ceph..." # Check for Python 3 @@ -1647,11 +1678,11 @@ update_tenant_storage() { # Find the exporter script EXPORTER_SCRIPT="" COMMON_PATHS=( - "/root/ceph-external-cluster-details-exporter.py" - "/usr/share/ceph/ceph-external-cluster-details-exporter.py" - "/usr/local/share/ceph/ceph-external-cluster-details-exporter.py" - "/opt/ceph/ceph-external-cluster-details-exporter.py" - "/usr/share/ceph-common/ceph-external-cluster-details-exporter.py" + "/root/create-external-cluster-resources.py" + "/usr/share/ceph/create-external-cluster-resources.py" + "/usr/local/share/ceph/create-external-cluster-resources.py" + "/opt/ceph/create-external-cluster-resources.py" + "/usr/share/ceph-common/create-external-cluster-resources.py" ) for path in "${COMMON_PATHS[@]}"; do @@ -1663,11 +1694,11 @@ update_tenant_storage() { done if [ -z "$EXPORTER_SCRIPT" ]; then - EXPORTER_SCRIPT=$(find /usr -name "ceph-external-cluster-details-exporter.py" 2>/dev/null | head -1) + EXPORTER_SCRIPT=$(find /usr /root -name "create-external-cluster-resources.py" 2>/dev/null | head -1) fi if [ -z "$EXPORTER_SCRIPT" ]; then - print_error "ceph-external-cluster-details-exporter.py not found" + print_error "create-external-cluster-resources.py not found" print_warning "Configuration regeneration failed - you can manually run phase 7:" print_info " ./setup-storage.sh --tenant ${TENANT_NAME} --rbd-quota ${RBD_QUOTA} \\" print_info " --output-dir ${OUTPUT_DIR} --mode ${MODE} --phase 7" @@ -1684,7 +1715,7 @@ update_tenant_storage() { SCRIPT_EXIT_CODE=$? if [ $SCRIPT_EXIT_CODE -ne 0 ]; then - print_error "Failed to run ceph-external-cluster-details-exporter.py" + print_error "Failed to run create-external-cluster-resources.py" print_error "Exit code: ${SCRIPT_EXIT_CODE}" print_warning "Configuration regeneration failed - you can manually run phase 7:" print_info " ./setup-storage.sh --tenant ${TENANT_NAME} --rbd-quota ${RBD_QUOTA} \\" @@ -2016,6 +2047,13 @@ cleanup_artifacts() { rm -f "$config_file" fi + # Remove state file so the script can be re-run from scratch + local state_file="${OUTPUT_DIR}/msp-provision-state.txt" + if [ -f "$state_file" ]; then + print_info "Removing state file: ${state_file}" + rm -f "$state_file" + fi + print_success "Artifacts cleaned up" } @@ -2387,35 +2425,60 @@ echo "" # Detect the correct RGW zone for user creation detect_rgw_zone() { local zone_list - local default_zone - - print_debug "Detecting RGW zone..." - - # Get list of zones - zone_list=$(ceph_exec radosgw-admin zone list 2>&1 || echo "") - + local detected_zone + + print_debug "Detecting RGW zone..." >&2 + + # If --rgw-realm was specified, use `zone get` to obtain the default zone + # name for that realm. `zone list` returns UUIDs in default_info, not names, + # so `zone get --rgw-realm=` (which returns .name) is the reliable path. + if [ -n "${RGW_REALM}" ]; then + local zone_get_out + zone_get_out=$(ceph_exec radosgw-admin zone get --rgw-realm="${RGW_REALM}" 2>/dev/null || echo "") + detected_zone=$(echo "$zone_get_out" | sed -n 's/.*"name":[[:space:]]*"\([^"]*\)".*/\1/p' | head -1) + if [ -n "$detected_zone" ]; then + print_debug "Using default zone for realm '${RGW_REALM}': ${detected_zone}" >&2 + echo "$detected_zone" + else + # Fall back: list zones and pick the first matching zone-* name + zone_list=$(ceph_exec radosgw-admin zone list --rgw-realm="${RGW_REALM}" 2>/dev/null || echo "") + detected_zone=$(echo "$zone_list" | sed -n 's/.*"\(zone-[^"]*\)".*/\1/p' | head -1) + if [ -n "$detected_zone" ]; then + print_debug "Using first zone for realm '${RGW_REALM}': ${detected_zone}" >&2 + echo "$detected_zone" + else + print_warning "Could not detect zone for realm '${RGW_REALM}', deriving from realm name" >&2 + echo "${RGW_REALM/realm-/zone-}" + fi + fi + return + fi + + # No realm specified — use global zone list (works for single-realm / "default" zone clusters) + zone_list=$(ceph_exec radosgw-admin zone list 2>/dev/null || echo "") + if [ -z "$zone_list" ]; then - print_warning "Could not detect RGW zones, using default" + print_warning "Could not detect RGW zones, using default" >&2 echo "default" return fi - - # Check if storage cluster CephObjectStore zone exists + + # Check if storage cluster CephObjectStore zone exists (ODF mode) local rgw_zone_name="${STORAGECLUSTER_NAME}-cephobjectstore" if echo "$zone_list" | grep -q "$rgw_zone_name"; then - print_debug "Found ODF zone: $rgw_zone_name" - echo "$rgw_zone_name" + print_debug "Found ODF zone: ${rgw_zone_name}" >&2 + echo "${rgw_zone_name}" return fi - - # Get default zone - default_zone=$(echo "$zone_list" | sed -n 's/.*"default_info":[[:space:]]*"\([^"]*\)".*/\1/p' | head -1) - - if [ -n "$default_zone" ]; then - print_debug "Using default zone from zone list" - echo "default" + + # Extract the first zone name from the "zones" array. + # NOTE: "default_info" contains a UUID, NOT the zone name — do not use it. + detected_zone=$(echo "$zone_list" | sed -n 's/.*"zones":[[:space:]]*\[.*"\([^"]*\)".*/\1/p' | head -1) + if [ -n "$detected_zone" ]; then + print_debug "Using first zone from zones array: ${detected_zone}" >&2 + echo "$detected_zone" else - print_debug "No specific zone detected, using 'default'" + print_debug "No specific zone detected, using 'default'" >&2 echo "default" fi } @@ -2795,13 +2858,24 @@ else fi print_success "Ceph configuration found" - # Verify Ceph keyring - if [ ! -f /etc/ceph/ceph.client.admin.keyring ]; then - print_error "Ceph admin keyring not found at /etc/ceph/ceph.client.admin.keyring" - print_error "Ensure you have admin credentials" + # Verify Ceph keyring (check both naming conventions) + # cephadm shell environment uses /etc/ceph/ceph.keyring + # standard ceph-common install uses /etc/ceph/ceph.client.admin.keyring + admin_keyring="" + if [ -f /etc/ceph/ceph.client.admin.keyring ]; then + admin_keyring="/etc/ceph/ceph.client.admin.keyring" + elif [ -f /etc/ceph/ceph.keyring ]; then + admin_keyring="/etc/ceph/ceph.keyring" + fi + + if [ -z "$admin_keyring" ]; then + print_error "Ceph admin keyring not found." + print_error "Checked: /etc/ceph/ceph.client.admin.keyring" + print_error " /etc/ceph/ceph.keyring" + print_error "Ensure you have admin credentials (run inside 'cephadm shell' or install ceph-common)" exit 1 fi - print_success "Ceph admin keyring found" + print_success "Ceph admin keyring found: ${admin_keyring}" # Test Ceph connectivity if ! ceph status &>/dev/null; then @@ -2830,18 +2904,32 @@ else # Get all RGW services RGW_SERVICES=$(ceph orch ps --daemon-type rgw 2>/dev/null | grep -v "NAME" || echo "") - RGW_COUNT=$(echo "$RGW_SERVICES" | grep -c "rgw\." || echo "0") + RGW_COUNT=$(echo "$RGW_SERVICES" | grep -c "rgw\." 2>/dev/null || true) + RGW_COUNT=${RGW_COUNT:-0} - if [ "$RGW_COUNT" -gt 1 ]; then - print_warning "Multiple RGW services detected (${RGW_COUNT}). Using the first one." - print_warning "To specify a different RGW, set MAIN_RGW_ENDPOINT manually." - print_warning "Note: Ensure the selected RGW is network-accessible from the tenant plane." - print_warning "You may need to verify network connectivity and firewall rules." + if [ "${RGW_COUNT}" -gt 1 ] 2>/dev/null && [ "${RGW_COUNT}" != "" ]; then + print_warning "Multiple RGW services detected (${RGW_COUNT})." + if [ -n "${RGW_REALM}" ]; then + print_info "Selecting RGW service matching realm '${RGW_REALM}'..." + else + print_warning "Using the first one. To specify a different RGW, set MAIN_RGW_ENDPOINT manually." + fi fi - # Extract hostname and port from first RGW service - # Format: rgw.ocmirror.sc5-ceph01.deuuvy sc5-ceph01 *:8000 running... - RGW_SERVICE_INFO=$(echo "$RGW_SERVICES" | head -1) + # Extract hostname and port from matching RGW service. + # If --rgw-realm was specified, prefer the service whose name contains the realm. + # Format: rgw.realm-ocp1.zone-ocp1.hase-st2.xxx hase-st2 *:8080 running... + if [ -n "${RGW_REALM}" ]; then + RGW_SERVICE_INFO=$(echo "$RGW_SERVICES" | grep "${RGW_REALM}" | head -1) + if [ -z "$RGW_SERVICE_INFO" ]; then + print_warning "No RGW service found for realm '${RGW_REALM}', falling back to first RGW." + RGW_SERVICE_INFO=$(echo "$RGW_SERVICES" | head -1) + else + print_debug "Selected RGW service for realm '${RGW_REALM}': ${RGW_SERVICE_INFO}" + fi + else + RGW_SERVICE_INFO=$(echo "$RGW_SERVICES" | head -1) + fi if [ -n "$RGW_SERVICE_INFO" ]; then RGW_HOST=$(echo "$RGW_SERVICE_INFO" | awk '{print $2}') @@ -2858,6 +2946,35 @@ else fi print_debug "RGW service found via orchestrator - host: ${RGW_HOST}, port: ${RGW_PORT}" + + # ceph orch ps returns a short hostname which may not be resolvable + # from an external tenant cluster. Use ceph orch host ls to resolve + # it to the node's public IP address. + _host_ls_json=$(ceph orch host ls -f json 2>/dev/null || echo "") + if [ -n "$_host_ls_json" ]; then + _resolved_ip=$(echo "$_host_ls_json" \ + | jq -r --arg h "$RGW_HOST" \ + '.[] | select(.hostname == $h) | .addr' \ + 2>/dev/null \ + | head -1) + [ "$_resolved_ip" = "null" ] && _resolved_ip="" + if [ -n "$_resolved_ip" ]; then + print_debug "Resolved RGW host '${RGW_HOST}' → '${_resolved_ip}' via ceph orch host ls" + RGW_HOST="$_resolved_ip" + else + print_debug "Could not resolve RGW host '${RGW_HOST}' via ceph orch host ls — keeping hostname as-is" + fi + fi + unset _host_ls_json _resolved_ip + + # Detect protocol from ceph config dump for this daemon + # ceph orch ps gives us the port but not whether it's SSL + _rgw_frontends_config=$(ceph config dump 2>/dev/null | grep "rgw_frontends" | head -1 | awk '{$1=$2=$3=""; print $0}' || echo "") + if [[ "$_rgw_frontends_config" =~ ssl_port ]] || [[ "$_rgw_frontends_config" =~ ssl_certificate ]]; then + RGW_PROTOCOL="https" + print_debug "SSL detected in rgw_frontends config — using https" + fi + unset _rgw_frontends_config fi else print_debug "Ceph orchestrator not available, using config-based detection..." @@ -2919,9 +3036,18 @@ else print_debug "Using first monitor IP as RGW host: ${RGW_HOST}" else print_error "Could not determine RGW hostname from ceph.conf" - print_error "RGW endpoint must be accessible from external clusters (ODF client)" - print_error "Please ensure 'rgw_host' or 'mon_host' is configured in /etc/ceph/ceph.conf" - print_error "Or manually set MAIN_RGW_ENDPOINT environment variable before running this script" + print_error "" + print_error "Object storage requires a running RGW (RADOS Gateway) daemon." + print_error "No RGW daemons were found in this cluster." + print_error "" + print_error "To deploy RGW with cephadm:" + print_error " ceph orch apply rgw default" + print_error "" + print_error "Or skip object storage by removing --rgw-user-quota:" + print_error " ./setup-storage.sh --tenant TENANT --rbd-quota SIZE --mode native --output-dir DIR" + print_error "" + print_error "Or set the endpoint manually:" + print_error " export MAIN_RGW_ENDPOINT=:" exit 1 fi fi @@ -2940,8 +3066,12 @@ else exit 1 fi - # Construct endpoint - MAIN_RGW_ENDPOINT="${RGW_HOST}:${RGW_PORT}" + # Construct endpoint — honour pre-set MAIN_RGW_ENDPOINT env var if given + if [ -n "${MAIN_RGW_ENDPOINT_OVERRIDE:-}" ]; then + MAIN_RGW_ENDPOINT="$MAIN_RGW_ENDPOINT_OVERRIDE" + else + MAIN_RGW_ENDPOINT="${RGW_HOST}:${RGW_PORT}" + fi print_info "RGW endpoint detected: ${RGW_PROTOCOL}://${MAIN_RGW_ENDPOINT}" @@ -3194,16 +3324,26 @@ else # Detect the correct RGW zone RGW_ZONE=$(detect_rgw_zone) print_info "Detected RGW zone: ${RGW_ZONE}" - + + if [ -n "${RGW_REALM}" ]; then + print_info "Using RGW realm: ${RGW_REALM}" + fi + + # Build optional realm flag — passed to every radosgw-admin call in this phase + _realm_flag="" + [ -n "${RGW_REALM}" ] && _realm_flag="--rgw-realm=${RGW_REALM}" + print_info "Creating RGW user: ${RGW_USER_NAME} (in main RGW zone: ${RGW_ZONE})" - + # Check if user already exists first (check in the correct zone) - if ceph_exec radosgw-admin user info --uid="${RGW_USER_NAME}" --rgw-zone="${RGW_ZONE}" &>/dev/null; then + # shellcheck disable=SC2086 + if ceph_exec radosgw-admin user info --uid="${RGW_USER_NAME}" --rgw-zone="${RGW_ZONE}" ${_realm_flag} &>/dev/null; then print_warning "RGW user '${RGW_USER_NAME}' already exists in zone '${RGW_ZONE}' — fetching credentials" - RGW_USER_INFO=$(ceph_exec radosgw-admin user info --uid="${RGW_USER_NAME}" --rgw-zone="${RGW_ZONE}" 2>&1) + # shellcheck disable=SC2086 + RGW_USER_INFO=$(ceph_exec radosgw-admin user info --uid="${RGW_USER_NAME}" --rgw-zone="${RGW_ZONE}" ${_realm_flag} 2>&1) RGW_ACCESS_KEY=$(echo "$RGW_USER_INFO" | jq -r '.keys[0].access_key' 2>/dev/null || echo "") RGW_SECRET_KEY=$(echo "$RGW_USER_INFO" | jq -r '.keys[0].secret_key' 2>/dev/null || echo "") - + # Fallback to portable field extraction if jq fails if [ -z "$RGW_ACCESS_KEY" ] || [ -z "$RGW_SECRET_KEY" ]; then RGW_ACCESS_KEY=$(extract_json_string_field "$RGW_USER_INFO" "access_key") @@ -3212,42 +3352,45 @@ else else # Create new RGW user in the correct zone with full capabilities print_info "User does not exist, creating in zone '${RGW_ZONE}' with full capabilities..." + # shellcheck disable=SC2086 RGW_USER_OUTPUT=$(ceph_exec radosgw-admin user create \ --uid="${RGW_USER_NAME}" \ --display-name="${TENANT_NAME} NooBaa User" \ --max-buckets=1000 \ --rgw-zone="${RGW_ZONE}" \ + ${_realm_flag} \ --caps="buckets=*;users=*;usage=*;metadata=*" 2>&1 || echo "COMMAND_FAILED") - + if echo "$RGW_USER_OUTPUT" | grep -q "COMMAND_FAILED"; then print_error "radosgw-admin command failed or timed out" print_error "Output: $RGW_USER_OUTPUT" exit 1 fi - + if echo "$RGW_USER_OUTPUT" | grep -qi "error" && ! echo "$RGW_USER_OUTPUT" | grep -qi "already exists"; then print_error "Failed to create RGW user ${RGW_USER_NAME}" print_error "Output: $RGW_USER_OUTPUT" exit 1 fi - + # Extract access key and secret key using jq first. RGW_ACCESS_KEY=$(echo "$RGW_USER_OUTPUT" | jq -r '.keys[0].access_key' 2>/dev/null || echo "") RGW_SECRET_KEY=$(echo "$RGW_USER_OUTPUT" | jq -r '.keys[0].secret_key' 2>/dev/null || echo "") - + # Fallback to portable field extraction if jq fails if [ -z "$RGW_ACCESS_KEY" ] || [ -z "$RGW_SECRET_KEY" ]; then RGW_ACCESS_KEY=$(extract_json_string_field "$RGW_USER_OUTPUT" "access_key") RGW_SECRET_KEY=$(extract_json_string_field "$RGW_USER_OUTPUT" "secret_key") fi - + if [ -z "$RGW_ACCESS_KEY" ] || [ -z "$RGW_SECRET_KEY" ]; then # Try to get existing keys if creation reported "already exists" print_warning "Could not extract keys from creation output, fetching user info" - RGW_USER_INFO=$(ceph_exec radosgw-admin user info --uid="${RGW_USER_NAME}" --rgw-zone="${RGW_ZONE}" 2>&1) + # shellcheck disable=SC2086 + RGW_USER_INFO=$(ceph_exec radosgw-admin user info --uid="${RGW_USER_NAME}" --rgw-zone="${RGW_ZONE}" ${_realm_flag} 2>&1) RGW_ACCESS_KEY=$(echo "$RGW_USER_INFO" | jq -r '.keys[0].access_key' 2>/dev/null || echo "") RGW_SECRET_KEY=$(echo "$RGW_USER_INFO" | jq -r '.keys[0].secret_key' 2>/dev/null || echo "") - + # Final fallback to portable field extraction if [ -z "$RGW_ACCESS_KEY" ] || [ -z "$RGW_SECRET_KEY" ]; then RGW_ACCESS_KEY=$(extract_json_string_field "$RGW_USER_INFO" "access_key") @@ -3255,50 +3398,73 @@ else fi fi fi - + if [ -z "$RGW_ACCESS_KEY" ] || [ -z "$RGW_SECRET_KEY" ]; then print_error "Failed to get RGW credentials for user ${RGW_USER_NAME}" exit 1 fi - + # Ensure user has proper capabilities (in case it already existed without them) print_info "Ensuring user has full management capabilities..." + # shellcheck disable=SC2086 ceph_exec radosgw-admin caps add \ --uid="${RGW_USER_NAME}" \ --caps="buckets=*;users=*;usage=*;metadata=*" \ - --rgw-zone="${RGW_ZONE}" &>/dev/null || { + --rgw-zone="${RGW_ZONE}" ${_realm_flag} &>/dev/null || { print_warning "Could not add capabilities (user may already have them)" } - + print_success "RGW user created: ${RGW_USER_NAME}" print_info " Access Key: ${RGW_ACCESS_KEY}" print_info " Secret Key: [REDACTED]" - + # Set quota on RGW user print_info "Setting RGW user quota: ${RGW_USER_QUOTA}" RGW_QUOTA_BYTES=$(convert_to_bytes "$RGW_USER_QUOTA") - + + # shellcheck disable=SC2086 ceph_exec radosgw-admin quota set \ --quota-scope=user \ --uid="${RGW_USER_NAME}" \ --rgw-zone="${RGW_ZONE}" \ + ${_realm_flag} \ --max-size="${RGW_QUOTA_BYTES}" 2>&1 || { print_error "Failed to set quota for RGW user ${RGW_USER_NAME}" exit 1 } - + # Enable quota enforcement print_info "Enabling RGW quota enforcement" + # shellcheck disable=SC2086 ceph_exec radosgw-admin quota enable \ --quota-scope=user \ --uid="${RGW_USER_NAME}" \ - --rgw-zone="${RGW_ZONE}" 2>&1 || { + --rgw-zone="${RGW_ZONE}" \ + ${_realm_flag} 2>&1 || { print_error "Failed to enable quota for RGW user ${RGW_USER_NAME}" exit 1 } print_success "RGW quota set and enabled: ${RGW_USER_QUOTA}" + # If MAIN_RGW_ENDPOINT is still empty at this point (e.g. native mode resumed + # from phase 5), detect it via orchestrator using the realm hint. + if [ -z "${MAIN_RGW_ENDPOINT:-}" ]; then + print_info "MAIN_RGW_ENDPOINT not set, detecting from orchestrator..." + _rgw_svc="" + if [ -n "${RGW_REALM}" ]; then + _rgw_svc=$(ceph_exec ceph orch ps --daemon-type rgw 2>/dev/null | grep "${RGW_REALM}" | head -1 || true) + fi + [ -z "$_rgw_svc" ] && _rgw_svc=$(ceph_exec ceph orch ps --daemon-type rgw 2>/dev/null | grep -v "^NAME" | head -1 || true) + if [ -n "$_rgw_svc" ]; then + _h=$(echo "$_rgw_svc" | awk '{print $2}') + _p=$(echo "$_rgw_svc" | awk '{print $3}' | grep -oE '[0-9]+$') + [ -n "$_h" ] && [ -n "$_p" ] && MAIN_RGW_ENDPOINT="${_h}:${_p}" + print_info "Detected RGW endpoint: ${RGW_PROTOCOL}://${MAIN_RGW_ENDPOINT}" + fi + unset _rgw_svc _h _p + fi + # Save RGW credentials RGW_CREDS_FILE="${OUTPUT_DIR}/${TENANT_NAME}-rgw-credentials.txt" cat > "$RGW_CREDS_FILE" </dev/null | cut -d'=' -f2- | sed 's|^https\?://||') + if [ -n "$_saved_endpoint" ]; then + MAIN_RGW_ENDPOINT="$_saved_endpoint" + print_debug "Restored MAIN_RGW_ENDPOINT from credentials file: ${MAIN_RGW_ENDPOINT}" + else + # No endpoint in file yet — detect now via orchestrator + print_info "MAIN_RGW_ENDPOINT not set, detecting from orchestrator..." + _rgw_svc="" + if [ -n "${RGW_REALM}" ]; then + _rgw_svc=$(ceph_exec ceph orch ps --daemon-type rgw 2>/dev/null | grep "${RGW_REALM}" | head -1 || true) + fi + [ -z "$_rgw_svc" ] && _rgw_svc=$(ceph_exec ceph orch ps --daemon-type rgw 2>/dev/null | grep -v "^NAME" | head -1 || true) + if [ -n "$_rgw_svc" ]; then + _h=$(echo "$_rgw_svc" | awk '{print $2}') + _p=$(echo "$_rgw_svc" | awk '{print $3}' | grep -oE '[0-9]+$') + [ -n "$_h" ] && [ -n "$_p" ] && MAIN_RGW_ENDPOINT="${_h}:${_p}" + print_info "Detected RGW endpoint: ${RGW_PROTOCOL}://${MAIN_RGW_ENDPOINT}" + fi + fi + unset _saved_endpoint _rgw_svc _h _p + fi + print_info "Creating backing bucket: ${BACKING_BUCKET}" print_info "Note: Bucket will be created using S3 API" @@ -3462,7 +3669,8 @@ else STRING_TO_SIGN="PUT\n\n\n${DATE}\n/${BACKING_BUCKET}" SIGNATURE=$(echo -en "${STRING_TO_SIGN}" | openssl sha1 -hmac "${RGW_SECRET_KEY}" -binary | base64) - BUCKET_CREATE_RESULT=$(curl -k -s -w "\n%{http_code}" -X PUT "${RGW_PROTOCOL}://${MAIN_RGW_ENDPOINT}/${BACKING_BUCKET}" \ + BUCKET_CREATE_RESULT=$(curl -k -s -w "\n%{http_code}" --connect-timeout 10 --max-time 20 \ + -X PUT "${RGW_PROTOCOL}://${MAIN_RGW_ENDPOINT}/${BACKING_BUCKET}" \ -H "Host: ${MAIN_RGW_ENDPOINT}" \ -H "Date: ${DATE}" \ -H "Authorization: AWS ${RGW_ACCESS_KEY}:${SIGNATURE}") @@ -3581,8 +3789,8 @@ else print_success "External config saved to: ${OUTPUT_JSON}" else - # Native Ceph mode: use ceph-external-cluster-details-exporter.py - print_info "Generating external config for native Ceph using ceph-external-cluster-details-exporter.py..." + # Native Ceph mode: use create-external-cluster-resources.py + print_info "Generating external config for native Ceph using create-external-cluster-resources.py..." # Check for Python 3 if ! command -v python3 &>/dev/null; then @@ -3591,16 +3799,16 @@ else fi # Find the exporter script - print_info "Locating ceph-external-cluster-details-exporter.py..." + print_info "Locating create-external-cluster-resources.py..." EXPORTER_SCRIPT="" # Common locations for the script COMMON_PATHS=( - "/root/ceph-external-cluster-details-exporter.py" - "/usr/share/ceph/ceph-external-cluster-details-exporter.py" - "/usr/local/share/ceph/ceph-external-cluster-details-exporter.py" - "/opt/ceph/ceph-external-cluster-details-exporter.py" - "/usr/share/ceph-common/ceph-external-cluster-details-exporter.py" + "/root/create-external-cluster-resources.py" + "/usr/share/ceph/create-external-cluster-resources.py" + "/usr/local/share/ceph/create-external-cluster-resources.py" + "/opt/ceph/create-external-cluster-resources.py" + "/usr/share/ceph-common/create-external-cluster-resources.py" ) for path in "${COMMON_PATHS[@]}"; do @@ -3614,40 +3822,29 @@ else # If not found in common locations, try to find it if [ -z "$EXPORTER_SCRIPT" ]; then print_info "Script not found in common locations, searching..." - EXPORTER_SCRIPT=$(find /usr -name "ceph-external-cluster-details-exporter.py" 2>/dev/null | head -1) + EXPORTER_SCRIPT=$(find /usr /root -name "create-external-cluster-resources.py" 2>/dev/null | head -1) if [ -n "$EXPORTER_SCRIPT" ]; then print_success "Found exporter script at: ${EXPORTER_SCRIPT}" else - print_error "ceph-external-cluster-details-exporter.py not found" + print_error "create-external-cluster-resources.py not found" print_error "" print_error "This script is required for native Ceph external cluster configuration." - print_error "It should be included with ceph-common package." print_error "" print_error "Checked locations:" for path in "${COMMON_PATHS[@]}"; do print_error " - ${path}" done print_error "" - print_error "To install:" - print_error " RHEL/CentOS: yum install ceph-common" - print_error " Ubuntu/Debian: apt-get install ceph-common" - print_error "" - print_error "Or download from: https://github.com/rook/rook/tree/master/deploy/examples" + print_error "Download from: https://github.com/rook/rook/tree/master/deploy/examples" exit 1 fi fi # Run the exporter script - print_info "Running ceph-external-cluster-details-exporter.py..." + print_info "Running create-external-cluster-resources.py..." print_info "This will generate all necessary ConfigMaps and Secrets for ODF" - # The script needs these parameters: - # --rbd-data-pool-name: The RBD pool name - # --k8s-cluster-name: Cluster identifier (tenant name) - # --restricted-auth-permission: Use restricted permissions - # --format: Output format (json) - SCRIPT_OUTPUT=$(python3 "$EXPORTER_SCRIPT" \ --rbd-data-pool-name "$POOL_NAME" \ --k8s-cluster-name "$TENANT_NAME" \ @@ -3657,7 +3854,7 @@ else SCRIPT_EXIT_CODE=$? if [ $SCRIPT_EXIT_CODE -ne 0 ]; then - print_error "Failed to run ceph-external-cluster-details-exporter.py" + print_error "Failed to run create-external-cluster-resources.py" print_error "Exit code: ${SCRIPT_EXIT_CODE}" print_error "Output:" print_error "$SCRIPT_OUTPUT" @@ -3836,7 +4033,39 @@ else # Add RGW credentials to JSON if object storage is enabled if [ "$ENABLE_OBJECT_STORAGE" = true ]; then print_info "Adding RGW object storage resources to external config JSON..." - + + # Restore RGW credentials and endpoint when resuming from a saved state + # (Phase 1 was skipped so these variables may be unset). + _ph7_creds_file="${OUTPUT_DIR}/${TENANT_NAME}-rgw-credentials.txt" + if [ -z "${RGW_SECRET_KEY:-}" ] || [ -z "${RGW_ACCESS_KEY:-}" ]; then + if [ -f "$_ph7_creds_file" ]; then + RGW_ACCESS_KEY=$(grep "^RGW_ACCESS_KEY=" "$_ph7_creds_file" | cut -d'=' -f2-) + RGW_SECRET_KEY=$(grep "^RGW_SECRET_KEY=" "$_ph7_creds_file" | cut -d'=' -f2-) + print_debug "Phase 7: restored RGW credentials from file" + else + print_error "Cannot add RGW resources: credentials file not found: ${_ph7_creds_file}" + exit 1 + fi + fi + if [ -z "${MAIN_RGW_ENDPOINT:-}" ]; then + _saved_ep=$(grep "^RGW_ENDPOINT=" "$_ph7_creds_file" 2>/dev/null | cut -d'=' -f2- | sed 's|^https\?://||') + if [ -n "$_saved_ep" ]; then + MAIN_RGW_ENDPOINT="$_saved_ep" + else + _rgw_svc="" + [ -n "${RGW_REALM}" ] && _rgw_svc=$(ceph_exec ceph orch ps --daemon-type rgw 2>/dev/null | grep "${RGW_REALM}" | head -1 || true) + [ -z "$_rgw_svc" ] && _rgw_svc=$(ceph_exec ceph orch ps --daemon-type rgw 2>/dev/null | grep -v "^NAME" | head -1 || true) + if [ -n "$_rgw_svc" ]; then + _h=$(echo "$_rgw_svc" | awk '{print $2}') + _p=$(echo "$_rgw_svc" | awk '{print $3}' | grep -oE '[0-9]+$') + [ -n "$_h" ] && [ -n "$_p" ] && MAIN_RGW_ENDPOINT="${_h}:${_p}" + fi + fi + print_debug "Phase 7: MAIN_RGW_ENDPOINT=${MAIN_RGW_ENDPOINT}" + unset _saved_ep _rgw_svc _h _p + fi + unset _ph7_creds_file + # Prepare CA bundle content (always include, even if placeholder) CA_BUNDLE_CONTENT="" if [ -f "$CA_BUNDLE_FILE" ]; then @@ -3936,7 +4165,27 @@ else echo "" # Create summary file SUMMARY_FILE="${OUTPUT_DIR}/${TENANT_NAME}-summary.txt" - + # Ensure file-path variables are set when resuming from phase 8 + : "${RGW_CREDS_FILE:=${OUTPUT_DIR}/${TENANT_NAME}-rgw-credentials.txt}" + : "${OUTPUT_JSON:=${OUTPUT_DIR}/${TENANT_NAME}-external-config.json}" + # Restore MAIN_RGW_ENDPOINT if not set (from credentials file or orchestrator) + if [ "$ENABLE_OBJECT_STORAGE" = true ] && [ -z "${MAIN_RGW_ENDPOINT:-}" ]; then + _ep=$(grep "^RGW_ENDPOINT=" "$RGW_CREDS_FILE" 2>/dev/null | cut -d'=' -f2- | sed 's|^https\?://||') + if [ -n "$_ep" ]; then + MAIN_RGW_ENDPOINT="$_ep" + else + _s="" + [ -n "${RGW_REALM}" ] && _s=$(ceph_exec ceph orch ps --daemon-type rgw 2>/dev/null | grep "${RGW_REALM}" | head -1 || true) + [ -z "$_s" ] && _s=$(ceph_exec ceph orch ps --daemon-type rgw 2>/dev/null | grep -v "^NAME" | head -1 || true) + if [ -n "$_s" ]; then + _h=$(echo "$_s" | awk '{print $2}'); _p=$(echo "$_s" | awk '{print $3}' | grep -oE '[0-9]+$') + [ -n "$_h" ] && [ -n "$_p" ] && MAIN_RGW_ENDPOINT="${_h}:${_p}" + fi + unset _s _h _p + fi + unset _ep + fi + if [ "$MODE" = "odf" ]; then CLUSTER_INFO=$(oc whoami --show-server) else