#!/usr/bin/env bash
#	Filename:-		gpinitsystem
#	Status:-		Released	
#	Author:-		L Lonergan/G Coombe	
#	Contact:-		gcoombe@greenplum.com
#	Release date:-		March 2006
#	Release stat:-		Released
#                               Copyright (c) Metapa 2005. All Rights Reserved.
#                               Copyright Greenplum database cluster
#******************************************************************************
# Update History
#******************************************************************************
# 	Date		Who		Update
#	12/05/2006	G Coombe	Added a parallel create segment option
#	4/26/2007	cmcdevitt	renamed from gpcreatecluster.sh
#
#******************************************************************************
# Detailed Description
#******************************************************************************
#******************************************************************************
# Prep Code

WORKDIR=`dirname $0`

# Source required functions file, this required for script to run
# exit if cannot locate this file. Change location of FUNCTIONS variable
# as required.
FUNCTIONS=$WORKDIR/lib/gp_bash_functions.sh
if [ -f $FUNCTIONS ]; then
    . $FUNCTIONS
    . $WORKDIR/lib/gp_bash_version.sh
else
    echo "[FATAL]:-Cannot source $FUNCTIONS file, script Exits!"
    exit 1
fi

#******************************************************************************
# Script Specific Variables
#******************************************************************************
# Log file that will record script actions
CUR_DATE=`$DATE +%Y%m%d`
FILE_TIME=`$DATE +%H%M%S`
PROG_NAME=`$BASENAME $0`
HELP_DOC_NAME=`$ECHO $PROG_NAME|$AWK -F'.' '{print $1}'`_help
BACKOUT_FILE=$DEFLOGDIR/backout_gpinitsystem_${USER_NAME}_${CUR_DATE}_$FILE_TIME
# Level of script feedback 0=small 1=verbose
VERBOSE=1
INTERACTIVE=1
MIRRORING=0
INST_COUNT=0
# Apache Cloudberry specific parameters
GP_USER=$USER_NAME
# System table names
GP_TBL=gp_id
GP_CONFIG_TBL=gp_segment_configuration
unset PG_CONF_ADD_FILE
#unset QD_PRIMARY_ARRAY QE_PRIMARY_ARRAY QE_MIRROR_ARRAY
EXIT_STATUS=0
# SED_PG_CONF search text values
PORT_TXT="#port"
LOG_STATEMENT_TXT="#log_statement ="
LISTEN_ADR_TXT="listen_addresses"
CONTENT_ID_TXT="gp_contentid"
DBID_TXT="gp_dbid"
SINGLENODE_MODE_TXT="gp_internal_is_singlenode"
INIT_STANDBY_PROG=$GPINITSTANDBY
QE_MIRROR_ARRAY=""
GP_PASSWD=gparray
TMP_PG_HBA=/tmp/pg_hba_conf_coordinator.$$
TMP_FILE=/tmp/cluster_tmp_file.$$
TMP_HOSTNAME_FILE=/tmp/hostname_test_file.$$
PARALLEL_STATUS_FILE=/tmp/gpinitsystem_parallel_status_file.$$
GPCREATESEG=$WORKDIR/lib/gpcreateseg.sh
ULIMIT_WARN=0
MIRROR_TYPE=0 # 0 = group, 1 = spread
REMOTE_HOST_COUNT=0
SINGLE_HOST_BATCH_LIMIT=4
INPUT_CONFIG=""
OUTPUT_CONFIG=""
STANDBY_RET_CODE=0
CLUSTER_BOOT_MODE="PRODUCTION"
ETCD_HOST_CONFIG=""
FTS_HOST_CONFIG=""
ETCD_HOST_CONFIG_NUM=0
ETCD_CONFIG_FILE_PATH=$GPHOME/bin/config/cbdb_etcd_default.conf
FTS_HOST_CONFIG_NUM=0
FTS_HOST_STRING_LIST=""
declare -a ETCD_HOST_MACHINE_LIST
declare -a FTS_HOST_MACHINE_LIST
USE_EXTERNAL_FTS=0
DEMO_CLUSTER_HOST="127.0.0.1"
ETCD_EXIST="false"
SINGLENODE_MODE="false"

#******************************************************************************
# DCA Specific Variables
#******************************************************************************
DCA_VERSION_FILE="${__DCA_VERSION_FILE__:-/etc/gpdb-appliance-version}"

DCA_RESQUEUE_PRIORITY_NAME="gp_resqueue_priority"
DCA_RESQUEUE_PRIORITY_CPUCORES_PER_SEGMENT_NAME="gp_resqueue_priority_cpucores_per_segment"
DCA_RESQUEUE_PRIORITY_SWEEPER_INTERVAL_NAME="gp_resqueue_priority_sweeper_interval"

DCA_COORDINATOR_RESQUEUE_PRIORITY_VAL="on"
DCA_COORDINATOR_RESQUEUE_PRIORITY_CPUCORES_PER_SEGMENT_VAL=24
DCA_COORDINATOR_RESQUEUE_PRIORITY_SWEEPER_INTERVAL_VAL=1000

DCA_SEGMENT_RESQUEUE_PRIORITY_VAL="on"
DCA_SEGMENT_RESQUEUE_PRIORITY_CPUCORES_PER_SEGMENT_VAL=4
DCA_SEGMENT_RESQUEUE_PRIORITY_SWEEPER_INTERVAL_VAL=1000

#******************************************************************************
# Functions
#******************************************************************************
USAGE () {
	if [ "$1" == "print_doc" ] && [ -f ${GPDOCDIR}/$HELP_DOC_NAME ]; then
		$LESSCMD ${GPDOCDIR}/$HELP_DOC_NAME
		exit 0
	else
		$ECHO
		$ECHO "      `basename $0` -c gp_config_file [OPTIONS]"
		$ECHO
		$ECHO "      Creates a new Apache Cloudberry instance on a Coordinator host and a number of"
		$ECHO "      segment instance hosts."
		$ECHO
		$ECHO "      General options:"
		$ECHO "      -v, display version information & exit"
		$ECHO
		$ECHO "      Logging options:"
		$ECHO "      -a, don't ask to confirm instance creation [default:- ask]"
		$ECHO "      -D, set log output to debug level, shows all function calls"
		$ECHO "      -l, logfile_directory [optional]"
		$ECHO "          Alternative logfile directory"
		$ECHO "      -q, quiet mode, do not log progress to screen [default:- verbose output to screen]"
		$ECHO
		$ECHO "      Configuration options:"
		$ECHO "      -b, shared_buffers per instance [default $DEFAULT_BUFFERS]"
		$ECHO "          Specify either the number of database I/O buffers (without suffix) or the"
		$ECHO "          amount of memory to use for buffers (with suffix 'kB', 'MB' or 'GB')."
		$ECHO "          Applies to coordinator and all segments."
		$ECHO "      -B, <number> run this batch of create segment processes in parallel [default $BATCH_DEFAULT]"
		$ECHO "      -c, gp_config_file [mandatory]"
		$ECHO "          Supplies all Cloudberry configuration information required by this utility."
		$ECHO "          Full description of all parameters contained within the example file"
		$ECHO "          supplied with this distribution."
		$ECHO "          Also see gpinitsystem_INSTRUCTIONS file for greater detail on"
		$ECHO "          the operation and configuration of this script"
		$ECHO "      -e, <password>, password to set for Cloudberry superuser in database [default $GP_PASSWD]"
		$ECHO "      -S, standby_datadir [optional]"
		$ECHO "      -h, gp_hostlist_file [optional]"
		$ECHO "          Contains a list of all segment instance hostnames required to participate in"
		$ECHO "          the new Cloudberry instance. Normally set in gp_config_file."
		$ECHO "      -I, <input_configuration_file>"
		$ECHO "          The full path and filename of an input configuration file, which defines the"
		$ECHO "          Apache Cloudberry members and segments using the QD_PRIMARY_ARRAY and"
		$ECHO "          PRIMARY_ARRAY parameters. The input configuration file is typically created by"
		$ECHO "          using gpinitsystem with the -O <output_configuration_file> option. You must"
		$ECHO "          provide either the -c <cluster_configuration_file> option or the -I"
		$ECHO "          <input_configuration_file> option to gpinitsystem."
		$ECHO "      -m, maximum number of connections for coordinator instance [default ${DEFAULT_QD_MAX_CONNECT}]"
		$ECHO "      -n, <locale>, setting for locale to be set when database initialized [defaults to system locale]"
		$ECHO "      -O, <output_configuration_file>"
		$ECHO "          When used with the -O option, gpinitsystem does not create a new Cloudberry"
		$ECHO "          Database cluster but instead writes the supplied cluster configuration"
		$ECHO "          information to the specified output_configuration_file. This file defines"
		$ECHO "          Apache Cloudberry members and segments using the QD_PRIMARY_ARRAY and"
		$ECHO "          PRIMARY_ARRAY parameters, and can be later used with -I"
		$ECHO "          <input_configuration_file> to initialize a new cluster."
		$ECHO "      -p, postgresql_conf_gp_additions [optional]"
		$ECHO "          List of additional PostgreSQL parameters to be applied to each Coordinator/Segment"
		$ECHO "          postgresql.conf file during Apache Cloudberry initialization."
		$ECHO "      -P, standby_port [optional]"
		$ECHO "      -s, standby_hostname [optional]"
		$ECHO "      -U, Use external FTS, default false"
		$ECHO "      -E, ETCD cluster host list, default deployed on master/standby/segment0 node [optional]"
		$ECHO "      -F, FTS cluster host list, default deployed on master/standby/segment0 node [optional]"
		$ECHO
		$ECHO "      Return codes:"
		$ECHO "      0 - No problems encountered with requested operation"
		$ECHO "      1 - Fatal error, instance not created/started, or in an inconsistent state,"
		$ECHO "          see log file for failure reason."
		$ECHO "      -T, cluster file encryption method, such as AES256, SM4"
		$ECHO "      -C, enable cluster file encryption and set command"
		$ECHO "          to obtain the cluster key"
		$ECHO
		exit $EXIT_STATUS
	fi
}

# derive proper options for gpstart/gpstop based on VERBOSE and DEBUG_LEVEL.  
# sample use:
#     GPSTOP_OPTS=$(OUTPUT_LEVEL_OPTS)
#
OUTPUT_LEVEL_OPTS () {
    if [ $VERBOSE ]; then
        if [ $DEBUG_LEVEL -eq 1 ]; then
    	    echo ' -v '
        else
    	    echo ''
        fi
    else
        echo ' -q '
    fi
}

# Check whether two intervals ($1, $2), ($3, $4) overlap
# Returns 0 if the intervals overlap, 1 otherwise
CHK_OVERLAP() {
	if [[ $1 -ge $3 && $1 -le $4 ]]; then
	   return 0;
	elif [[ $3 -ge $1 && $3 -le $2 ]]; then
	   return 0;
	else
       return 1;
    fi
}

# Accept a list of variables and output the first one that has
# a value, or an empty string if none are set.
USE_FIRST_SET_ARG (){
	for var in "$@"; do
		if [ x"" != x"$var" ]; then
			echo "$var"
			return
		fi
	done
	echo ""
}

CHK_PARAMS () {
		LOG_MSG "[INFO]:-Start Function $FUNCNAME"
		LOG_MSG "[INFO]:-Checking configuration parameters, please wait..." 1
		if [ "$EUID" -eq 0 ] || [ $(id -u) = 0 ]; then
				ERROR_EXIT "[FATAL]:-Unable to run this script as root user: $USER."
		fi
		if [ x"" = x"$GPHOME" ]; then
				LOG_MSG "[FATAL]:-Environment variable \$GPHOME not set" 1
				ERROR_EXIT "[FATAL]:-Unable to continue"
		fi
		# Check that we can see initdb
		if [ x"$INITDB" = x"" ] || [ ! -x $INITDB ];then
			ERROR_EXIT "[FATAL]:-Unable to locate initdb"
		fi
		# Make sure that script has been supplied a config file
		if [ x"$CLUSTER_CONFIG" = x"" ] && [ x"$INPUT_CONFIG" = x"" ]; then
			ERROR_EXIT "[FATAL]:-At least one of two options, [-c] or [-I], is required."
		fi

		if [ -n "$CLUSTER_CONFIG" ] && [ -n "$INPUT_CONFIG" ]; then
			ERROR_EXIT "[FATAL]:-Options [-c] and [-I] cannot be used at the same time."
		fi
		
		if [ x"" != x"$CLUSTER_CONFIG" ] ; then
		    # Check that we have a non-zero configuration file
		    CHK_FILE $CLUSTER_CONFIG
		    
		    if [ $EXISTS -ne 0 ]; then
		        ERROR_EXIT "[FATAL]:-Configuration file $CLUSTER_CONFIG does not exist."
		    fi
		    
		    # Make sure old CLUSTER_CONFIG settings are not hanging around.
		    unset PORT_BASE SEG_PREFIX DATA_DIRECTORY HEAP_CHECKSUM HBA_HOSTNAMES

		    # Make sure it is not a dos file with CTRL M at end of each line
		    $TR -d '\r' < $CLUSTER_CONFIG > $TMP_FILE
		    $MV $TMP_FILE $CLUSTER_CONFIG
		    LOG_MSG "[INFO]:-Dumping $CLUSTER_CONFIG to logfile for reference"
		    $CAT $CLUSTER_CONFIG|$GREP -v "^\s*\(#.*\)\?$" >> $LOG_FILE
		    LOG_MSG "[INFO]:-Completed $CLUSTER_CONFIG dump to logfile"
		    # Source the cluster configuration file
		    LOG_MSG "[INFO]:-Reading Cloudberry configuration file $CLUSTER_CONFIG" 1
			. $CLUSTER_CONFIG
		    ASSIGN_COORDINATOR_VARS
		    
		    if [ x"" != x"$QD_PRIMARY_ARRAY" ] ; then
			    ERROR_EXIT "[FATAL]:-Cannot specify QD_PRIMARY_ARRAY in '-c <config file>'.  Only valid with '-I <input file>'"
		    fi

		    if [ x"" = x"$PORT_BASE" ]; then
			    ERROR_EXIT "[FATAL]:-PORT_BASE not specified in $CLUSTER_CONFIG file, is this the correct instance configuration file."
		    fi

		    if [ $MIRROR_PORT_BASE ]; then
			    MIRRORING=1
		    fi

		else
		    LOG_MSG "[INFO]:-Reading Cloudberry input configuration file $INPUT_CONFIG"
		    READ_INPUT_CONFIG
		fi

		if [ x"" != x"$REQ_LOCALE_SETTING" ];then LOCALE_SETTING=$REQ_LOCALE_SETTING;fi

		
		if [ x"" == x"$INPUT_CONFIG" ] ; then 
		    if [ x"" != x"$MACHINE_LIST_FILE_ALT" ];then
                        MACHINE_LIST_FILE=$MACHINE_LIST_FILE_ALT
                    fi
		    # Now check to see if MACHINE_LIST_FILE is still empty
		    if [ x"" == x"$MACHINE_LIST_FILE" ];then
				LOG_MSG "[FATAL]:-MACHINE_LIST_FILE variable is unset, and -h option not supplied" 1
				ERROR_EXIT "[FATAL]:-Unable to continue"
		    else
		    	MACHINE_LIST_NUM=${#MACHINE_LIST_FILE[@]}
			if [ $MACHINE_LIST_NUM -eq 1 ];then
				local HOSTFILE=${MACHINE_LIST_FILE[0]}
				local SEG_HOSTNAME=(`$CAT $HOSTFILE`)
				if [ x"$COORDINATOR_HOSTNAME" == x"$SEG_HOSTNAME" ];then
					CLUSTER_BOOT_MODE="DEMO"
				fi
			fi
		    fi
		fi

		if [ x"$ETCD_HOST_CONFIG" = x"" ] || [ x"$CLUSTER_BOOT_MODE" = x"DEMO" ]; then
			LOG_MSG "[WARN]:-No ETCD cluster host config provided, use default configuration."
		else
			ETCD_HOST_MACHINE_LIST=(`$CAT $ETCD_HOST_CONFIG`)
			ETCD_HOST_CONFIG_NUM=${#ETCD_HOST_MACHINE_LIST[@]}
			if [ $ETCD_HOST_CONFIG_NUM -lt 2 ]; then
				ERROR_EXIT "[FATAL]:-ETCD host number should be larger or equal to 2 nodes."
			fi
		fi

		if [ x"$FTS_HOST_CONFIG" = x"" ] || [ x"$CLUSTER_BOOT_MODE" = x"DEMO" ]; then
			LOG_MSG "[WARN]:-No FTS cluster host config provided, use default configuration."
		else
			FTS_HOST_MACHINE_LIST=(`$CAT $FTS_HOST_CONFIG`)
			FTS_HOST_CONFIG_NUM=${#FTS_HOST_MACHINE_LIST[@]}
			if [ $FTS_HOST_CONFIG_NUM -lt 2 ]; then
				ERROR_EXIT "[FATAL]:-FTS host number should be larger or equal to 2 nodes."
			fi
		fi

		# Check for required definitions in the CLUSTER_CONFIG file
		if [ x"" = x"$SEG_PREFIX" ]; then
			ERROR_EXIT "[FATAL]:-SEG_PREFIX not specified in $CLUSTER_CONFIG file, is this the correct instance configuration file."
		fi
		if [ "$SINGLENODE_MODE" == "false" ] && [ x"" = x"$DATA_DIRECTORY" ]; then
			ERROR_EXIT "[FATAL]:-DATA_DIRECTORY not specified in $CLUSTER_CONFIG file, is this the correct instance configuration file."
		fi

		if [ x"" = x"$HEAP_CHECKSUM" ]; then
			HEAP_CHECKSUM=on
			LOG_MSG "[INFO]:-Could not find HEAP_CHECKSUM in cluster config, defaulting to on."
		fi

		if [ x"" = x"$HBA_HOSTNAMES" ]; then
			HBA_HOSTNAMES=0
			LOG_MSG "[INFO]:-Could not find HBA_HOSTNAMES in cluster config, defaulting to 0."
		fi

		if [ x"" == x"$LOCALE_SETTING" ];then
			LOG_MSG "[INFO]:-Locale has not been set in $CLUSTER_CONFIG, will set to default value" 1
		else
			LOG_MSG "[INFO]:-Locale set to $LOCALE_SETTING"
			CHK_LOCALE_KNOWN
		fi
		LOG_MSG "[INFO]:-Dump current system locale to log file"
		$LOCALE >> $LOG_FILE
		LOG_MSG "[INFO]:-End of system locale dump"
		if [ ! -f $GPCREATESEG ];then
			ERROR_EXIT "[FATAL]:-No $GPCREATESEG exists"
		fi

		if [ x"" == x"$INPUT_CONFIG" ] ; then
		    # Check the other files that are required to be able to continue
		    FILE_LIST=($MACHINE_LIST_FILE)
		    for FILE in "${FILE_LIST[@]}"
		    do
				CHK_FILE $FILE
			if [ $EXISTS -ne 0 ]; then
			    ERROR_EXIT "[FATAL]:-Expected file \"$FILE\" not found"
			else
			    LOG_MSG "[INFO]:-Completed check of file $FILE"
			fi
		    done
		    CHK_DUPLICATES
		    # Set up the machine list array
		    LOG_MSG "[INFO]:-Setting up segment instance list array"
		    declare -a MACHINE_LIST=(`$CAT $MACHINE_LIST_FILE|$SORT`)
		    MACHINE_HOST_NUM=${#MACHINE_LIST[@]}
			if [ x"$FTS_HOST_CONFIG" = x"" ] && [ x"$CLUSTER_BOOT_MODE" = x"PRODUCTION" ]; then
				FTS_HOST_MACHINE_LIST[1]=${MACHINE_LIST[0]}
				if [ x"$STANDBY_HOSTNAME" = x"" ] && [ $MACHINE_HOST_NUM -ge 2 ];then
					FTS_HOST_MACHINE_LIST[2]=${MACHINE_LIST[1]}
				fi
			fi
			if [ x"$ETCD_HOST_CONFIG" = x"" ] && [ x"$CLUSTER_BOOT_MODE" = x"PRODUCTION" ]; then
				ETCD_HOST_MACHINE_LIST[1]=${MACHINE_LIST[0]}
				if [ x"$STANDBY_HOSTNAME" = x"" ] && [ $MACHINE_HOST_NUM -ge 2 ];then
					ETCD_HOST_MACHINE_LIST[2]=${MACHINE_LIST[1]}
				fi
			fi
		fi
		# Now process contents for the configuration file
		# Make sure that this script is running on the QD host
		if [ $COORDINATOR_HOSTNAME != `$HOSTNAME` ]; then
			LOG_MSG "[WARN]:-Coordinator hostname $COORDINATOR_HOSTNAME does not match hostname output" 1
			LOG_MSG "[INFO]:-Checking to see if $COORDINATOR_HOSTNAME can be resolved on this host" 1
			$TRUSTED_SHELL $COORDINATOR_HOSTNAME "$TOUCH $TMP_HOSTNAME_FILE"
			if [ -f $TMP_HOSTNAME_FILE ];then
				LOG_MSG "[INFO]:-Can resolve $COORDINATOR_HOSTNAME to this host" 1
				$RM -f $TMP_HOSTNAME_FILE
			else
				$TRUSTED_SHELL $COORDINATOR_HOSTNAME "$RM -f $TMP_HOSTNAME_FILE"
				LOG_MSG "[FATAL]:-Coordinator hostname in configuration file is ${COORDINATOR_HOSTNAME}" 1
				LOG_MSG "[FATAL]:-Operating system command returns `$HOSTNAME`" 1
				LOG_MSG "[FATAL]:-Unable to resolve $COORDINATOR_HOSTNAME on this host" 1
				ERROR_EXIT "[FATAL]:-Coordinator hostname in gpinitsystem configuration file must be $COORDINATOR_HOSTNAME"
			fi
		fi

		# DATA_DIRECTORY
		((QE_PRIMARY_COUNT=${#DATA_DIRECTORY[@]}))
		if [ "$SINGLENODE_MODE" == "false" ] && [ $QE_PRIMARY_COUNT -eq 0 ]; then
			ERROR_EXIT "[FATAL]:-Number of primary directories 0 (zero)"
		fi

		# Check that we have write access to the proposed coordinator data directory
		W_DIR=$COORDINATOR_DIRECTORY
		LOG_MSG "[INFO]:-Checking write access to $W_DIR on coordinator host"
		$TOUCH ${W_DIR}/tmp_file_test
		RETVAL=$?
		if [ $RETVAL -ne 0 ];then
			ERROR_EXIT "[FATAL]:-Cannot write to $W_DIR on coordinator host "
		else
			$RM -f ${W_DIR}/tmp_file_test
			LOG_MSG "[INFO]:-Write test passed $W_DIR directory on coordinator host"
		fi	
		# Check that the coordinator segment directory does not exist
		if [ -d ${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1 ];then
			ERROR_EXIT "[FATAL]:-Coordinator host data directory ${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1 already exists"
		fi
		# DATABASE_PREFIX
		if [ x"" = x"$DATABASE_NAME" ]; then
			LOG_MSG "[INFO]:-No DATABASE_NAME set, will exit following $DEFAULTDB updates" 1
		else
			LOG_MSG "[INFO]:-Will create database $DATABASE_NAME"
		fi
		if [ ! $TRUSTED_SHELL ]; then
				ERROR_EXIT "[FATAL]:-TRUSTED_SHELL variable not set"
		fi
		# ENCODING
		if [ ! $ENCODING ]; then
			LOG_MSG "[WARN]:-ENCODING variable not set, will set to default UTF-8" 1
			ENCODING="UTF-8"
		fi
		if [ x"SQL_ASCII" = x"$ENCODING" ]; then
			ERROR_EXIT "[FATAL]:-SQL_ASCII is no longer supported as a server encoding"
		fi
		# COORDINATOR_MAX_CONNECT
		if [ ! $COORDINATOR_MAX_CONNECT ];then
			LOG_MSG "[INFO]:-COORDINATOR_MAX_CONNECT not set, will set to default value $DEFAULT_QD_MAX_CONNECT" 1
			COORDINATOR_MAX_CONNECT=$DEFAULT_QD_MAX_CONNECT
		else
			if [ $COORDINATOR_MAX_CONNECT -lt 1 ];then
				ERROR_EXIT "[FATAL]:-COORDINATOR_MAX_CONNECT less than 1"
			fi
		fi
		((QE_MAX_CONNECT=$COORDINATOR_MAX_CONNECT*$QE_CONNECT_FACTOR))
		LOG_MSG "[INFO]:-Setting segment instance MAX_CONNECTIONS to $QD_MAX_CONNECT"
		if [ x"" == x"$NEW_BUFFERS" ];then
			COORDINATOR_SHARED_BUFFERS=$DEFAULT_BUFFERS
			QE_SHARED_BUFFERS=$DEFAULT_BUFFERS
		else
			# removed code that forced at least 1000 buffers -kh 3/14/07
			COORDINATOR_SHARED_BUFFERS=$NEW_BUFFERS
			QE_SHARED_BUFFERS=$NEW_BUFFERS
			LOG_MSG "[INFO]:-Set shared buffers to $NEW_BUFFERS"
		fi
		# Number of QE hosts in this configuration
		((NUM_QES=${#MACHINE_LIST[*]}))
		if [ "$SINGLENODE_MODE" == "false" ] && [ $NUM_QES -eq 0 ]; then
			ERROR_EXIT "[FATAL]:-Number of Segment instances's 0 (zero)"
		else
			LOG_MSG "[INFO]:-Number of segment instance hosts = $NUM_QES"
			if [ $NUM_QES -eq 1 ];then
				# This is a single host array, re-tune the BATCH_DEFAULT to 4
				if [ $BATCH_DEFAULT -gt $SINGLE_HOST_BATCH_LIMIT ];then
				LOG_MSG "[INFO]:-Detected a single host Cloudberry array build, reducing value of BATCH_DEFAULT from $BATCH_DEFAULT to $SINGLE_HOST_BATCH_LIMIT" 1
				BATCH_DEFAULT=$SINGLE_HOST_BATCH_LIMIT
				fi
			fi
		fi

		# Mirror configuration
		if [ $MIRRORING -eq 1 ]; then
			((NUM_MIRROR_DIRECTORY=${#MIRROR_DATA_DIRECTORY[@]}))
			if [ x"" == x"$INPUT_CONFIG" ] ; then
				if [ $NUM_MIRROR_DIRECTORY -ne $QE_PRIMARY_COUNT ]; then
					ERROR_EXIT "[FATAL]:-Number of primary directories does not match number of mirror directories"
				fi
			fi
			
			for dir in "${DATA_DIRECTORY[@]}"
			do
				for mirrordir in "${MIRROR_DATA_DIRECTORY[@]}"
				do
					if [ $dir == $mirrordir ] ; then
						ERROR_EXIT "[FATAL]:-Conflict between data directory and mirror data directory ($dir)."
					fi
				done
			done
		fi

		# Check open files value
		COORDINATOR_OPEN=`ulimit -n`
		if [ $COORDINATOR_OPEN -lt $OS_OPENFILES ];then
			LOG_MSG "[WARN]:-Coordinator open file limit is $COORDINATOR_OPEN should be >= $OS_OPENFILES" 1
			ULIMIT_WARN=1
		fi
		# Get IP address of the coordinator host

		COORDINATOR_IP_ADDRESS_ALL=(`$IPV4_ADDR_LIST_CMD | $GREP inet | $GREP -v 127.0.0 | $AWK '{print $2}' | $CUT -d'/' -f1`)
		ERROR_CHK $? "obtain IP address of Coordinator host" 2
		COORDINATOR_IPV6_LOCAL_ADDRESS_ALL=(`$IPV6_ADDR_LIST_CMD | $GREP inet6 | $AWK '{print $2}' | $CUT -d'/' -f1`)
		COORDINATOR_IP_ADDRESS=(`$ECHO ${COORDINATOR_IP_ADDRESS_ALL[@]} ${COORDINATOR_IPV6_LOCAL_ADDRESS_ALL[@]}|$TR ' ' '\n'|$SORT -u|$TR '\n' ' '`)
		LOG_MSG "[INFO]:-Coordinator IP address array = ${COORDINATOR_IP_ADDRESS[@]}"
		if [ x"" != x"$STANDBY_HOSTNAME" ];then
			STANDBY_IP_ADDRESS_ALL=( $( REMOTE_EXECUTE_AND_GET_OUTPUT $STANDBY_HOSTNAME "$IPV4_ADDR_LIST_CMD |$GREP inet|$GREP -v \"127.0.0\"|$AWK '{print \$2}'|$CUT -d'/' -f1" 2>>$LOG_FILE ))
			STANDBY_IPV6_ADDRESS_ALL=( $( REMOTE_EXECUTE_AND_GET_OUTPUT $STANDBY_HOSTNAME "$IPV6_ADDR_LIST_CMD |$GREP inet6|$AWK '{print \$2}' |$CUT -d'/' -f1" 2>>$LOG_FILE ))
			STANDBY_IP_ADDRESS=(`$ECHO ${STANDBY_IP_ADDRESS_ALL[@]} ${STANDBY_IPV6_ADDRESS_ALL[@]}|$TR ' ' '\n'|$SORT -u|$TR '\n' ' '`)
			ERROR_CHK $? "obtain IP address of standby Coordinator host" 2
			LOG_MSG "[INFO]:-Standby IP address array = ${STANDBY_IP_ADDRESS[@]}"
			#Check open files value
			STANDBY_COORDINATOR_OPEN=$( REMOTE_EXECUTE_AND_GET_OUTPUT $STANDBY_HOSTNAME "ulimit -n" )
			if [ $STANDBY_COORDINATOR_OPEN -lt $OS_OPENFILES ];then
				LOG_MSG "[WARN]:-Standby Coordinator open file limit is $STANDBY_COORDINATOR_OPEN should be >= $OS_OPENFILES" 1
				ULIMIT_WARN=1
			fi
		fi
		
		# Validate that the different locale settings are available on the system, first
		# using passed locale arguments and then falling back to standard environment
		# variables if necessary.
		# Note: This check is performed on the coordinator only.  There is an assumption
		# being made that the locales available on the coordinator are available on the
		# segment hosts.
		LOCALE_SETTING=$(USE_FIRST_SET_ARG $REQ_LOCALE_SETTING $LC_ALL)
		if [ x"" != x"$LOCALE_SETTING" ]; then
			LOCALE_IS_AVAILABLE $LOCALE_SETTING
			if [ $? -eq 0 ]; then
				ERROR_EXIT "[FATAL]-Value $LOCALE_SETTING is not a valid value for --locale on this system."
			fi
		fi

		LCCOLLATE=$(USE_FIRST_SET_ARG $LCCOLLATE $LC_COLLATE $LOCALE_SETTING)
		if [ x"" != x"$LCCOLLATE" ]; then
			LOCALE_IS_AVAILABLE $LCCOLLATE
			if [ $? -eq 0 ]; then
				ERROR_EXIT "[FATAL]-Value $LCCOLLATE is not a valid value for --lc-collate on this system."
			fi	
		fi

		LCCTYPE=$(USE_FIRST_SET_ARG $LCCTYPE $LC_CTYPE $LOCALE_SETTING)
		if [ x"" != x"$LCCTYPE" ]; then
			LOCALE_IS_AVAILABLE $LCCTYPE
			if [ $? -eq 0 ]; then
				ERROR_EXIT "[FATAL]-Value $LCCTYPE is not a valid value for --lc-ctype on this system."
			fi
		fi

		LCMESSAGES=$(USE_FIRST_SET_ARG $LCMESSAGES $LC_MESSAGES $LOCALE_SETTING)
		if [ x"" != x"$LCMESSAGES" ]; then
			LOCALE_IS_AVAILABLE $LCMESSAGES
			if [ $? -eq 0 ]; then
				ERROR_EXIT "[FATAL]-Value $LCMESSAGES is not a valid value for --lc-messages on this system."
			fi
		fi
		
		LCMONETARY=$(USE_FIRST_SET_ARG $LCMONETARY $LC_MONETARY $LOCALE_SETTING)
		if [ x"" != x"$LCMONETARY" ]; then
			LOCALE_IS_AVAILABLE $LCMONETARY
			if [ $? -eq 0 ]; then
				ERROR_EXIT "[FATAL]-Value $LCMONETARY is not a valid value for --lc-monetary on this system."
			fi
		fi
		
		LCNUMERIC=$(USE_FIRST_SET_ARG $LCNUMERIC $LC_NUMERIC $LOCALE_SETTING)
		if [ x"" != x"$LCNUMERIC" ]; then
			LOCALE_IS_AVAILABLE $LCNUMERIC
			if [ $? -eq 0 ]; then
				ERROR_EXIT "[FATAL]-Value $LCNUMERIC is not a valid value for --lc-numeric on this system."
			fi
		fi
		
		LCTIME=$(USE_FIRST_SET_ARG $LCTIME $LC_TIME $LOCALE_SETTING)
		if [ x"" != x"$LCTIME" ]; then
			LOCALE_IS_AVAILABLE $LCTIME
			if [ $? -eq 0 ]; then
				ERROR_EXIT "[FATAL]-Value $LCTIME is not a valid value for --lc-time on this system."
			fi
		fi

		LOG_MSG "[INFO]:-Checking configuration parameters, Completed" 1
		LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

CHK_MULTI_HOME () {
	LOG_MSG "[INFO]:-Start Function $FUNCNAME"
	LOG_MSG "[INFO]:-Commencing multi-home checks, please wait..." 1

	MACHINE_LIST=(`$CAT $MACHINE_LIST_FILE`)
	M_HOST_ARRAY=()
	MCOUNT=0
	for MHOST in ${MACHINE_LIST[@]}
	do
		if [ $DEBUG_LEVEL -eq 0 ] && [ x"" != x"$VERBOSE" ];then $NOLINE_ECHO ".\c";fi
		PING_HOST $MHOST
		SEG_HOSTNAME=$( REMOTE_EXECUTE_AND_GET_OUTPUT $MHOST "$HOSTNAME" )
		if [ $? -ne 0 ];then
			LOG_MSG "[FATAL]:-Remote command to host $MHOST failed to get value of hostname" 1
			LOG_MSG "[FATAL]:-Check to see that you have setup trusted remote ssh on all hosts" 1
			ERROR_EXIT "[FATAL]:-Unable to get hostname output for $MHOST"
		fi
		if [ `$ECHO ${T_SEG_ARRAY[@]}|$TR ' ' '\n'|$GREP -c "^${SEG_HOSTNAME}$"` -eq 0 ];then
			T_SEG_ARRAY=(${T_SEG_ARRAY[@]} $SEG_HOSTNAME)
		fi
		T_HOST_ARRAY=(${T_HOST_ARRAY[@]} ${MHOST}~$SEG_HOSTNAME)	
		((MCOUNT=$MCOUNT+1))
	done
	if [ $DEBUG_LEVEL -eq 0 ] && [ x"" != x"$VERBOSE" ];then $ECHO;fi
	# Now sort the array to ensure that all SEG_HOSTNAME values together
	for S_HOST in ${T_SEG_ARRAY[@]}
	do
		M_HOST_ARRAY=(${M_HOST_ARRAY[@]} `$ECHO ${T_HOST_ARRAY[@]}|$TR ' ' '\n'|$GREP "~${S_HOST}$"|$TR '\n' ' '`)
	done
	. $CLUSTER_CONFIG
	ASSIGN_COORDINATOR_VARS
	NUM_DATADIR=${#DATA_DIRECTORY[@]}
	if [ `$ECHO ${M_HOST_ARRAY[@]}|$TR ' ' '\n'|$AWK -F"~" '{print $2}'|$SORT -u|$WC -l` -ne $MCOUNT ];then
		LOG_MSG "[INFO]:-Configuring build for multi-home array" 1
		MULTI_HOME=1
		# Now make sure that we have same number of unique hostnames as there are data directories declared
		HOST1=`$ECHO ${M_HOST_ARRAY[0]}|$AWK -F"~" '{print $2}'`
		NUM_MHOST_NODE=`$ECHO ${M_HOST_ARRAY[@]}|$TR ' ' '\n'|$GREP "~${HOST1}$"|$WC -l`

                ((REMAINDER=$NUM_DATADIR % $NUM_MHOST_NODE))                
                ((MULTIPLE=$NUM_DATADIR / $NUM_MHOST_NODE))
                if [ $REMAINDER -ne 0 ] || [ $MULTIPLE -eq 0 ] ;then                        
                    LOG_MSG "[FATAL]:-Inconsistency between number of multi-home hostnames and number of segments per host" 1
                    LOG_MSG "[INFO]:-Have $NUM_DATADIR data directories and $NUM_MHOST_NODE multi-home hostnames for each host" 1
                    LOG_MSG "[INFO]:-For multi-home configuration, number of segment instance data directories per host must be multiple of" 1
                    LOG_MSG "[INFO]:-the number of multi-home hostnames within the Cloudberry array" 1
                    ERROR_EXIT "[FATAL]:-Unable to continue"
                fi

		if [ $MULTIPLE -gt 1 ] ; then
			# Now need to increase the size of M_HOME_ARRAY to fake a multi-home array
			HOME_COUNT=1
			FAKE_M_HOST_ARRAY=(${M_HOST_ARRAY[@]})                
			while [ $HOME_COUNT -lt $MULTIPLE ]
			do
				FAKE_M_HOST_ARRAY=(${FAKE_M_HOST_ARRAY[@]} ${M_HOST_ARRAY[@]})
				((HOME_COUNT=$HOME_COUNT+1))
			done
			M_HOST_ARRAY=()
			for S_HOST in ${T_SEG_ARRAY[@]}
			do
				M_HOST_ARRAY=(${M_HOST_ARRAY[@]} `$ECHO ${FAKE_M_HOST_ARRAY[@]}|$TR ' ' '\n'|$GREP "~${S_HOST}$"| $SORT | $TR '\n' ' '`)
			done
		fi
	else
		LOG_MSG "[INFO]:-Configuring build for standard array" 1
		MULTI_HOME=0
		# Now need to increase the size of M_HOME_ARRAY to fake a multi-home array
		HOME_COUNT=1
		FAKE_M_HOST_ARRAY=(${M_HOST_ARRAY[@]})
		while [ $HOME_COUNT -lt $NUM_DATADIR ]
		do
			FAKE_M_HOST_ARRAY=(${FAKE_M_HOST_ARRAY[@]} ${M_HOST_ARRAY[@]})
			((HOME_COUNT=$HOME_COUNT+1))
		done
		M_HOST_ARRAY=(`$ECHO ${FAKE_M_HOST_ARRAY[@]}|$TR ' ' '\n'|$SORT |$TR '\n' ' '`)
	fi
	NUM_SEP_HOSTS=`$ECHO ${M_HOST_ARRAY[@]}|$TR ' ' '\n'|$AWK -F"~" '{print $2}'|$SORT -u|$WC -l`
	if [ $MIRRORING -eq 1 ] && [ $MIRROR_TYPE -eq 1 ];then
	# Mirroring is on and spread mirror configuration has been requested, make sure sufficient hosts
		if [ $NUM_SEP_HOSTS -eq 1 ] || [ $NUM_SEP_HOSTS -le $NUM_DATADIR ];then
			LOG_MSG "[FATAL]:-Request made for spread mirroring via --mirror-mode option, but insufficient hosts available" 1
			LOG_MSG "[INFO]:-Number of separate hosts must be greater than number of segment instances per host" 1
			ERROR_EXIT "[FATAL]:-Unable to continue"
		else
			LOG_MSG "[INFO]:-Sufficient hosts for spread mirroring request" 1
		fi 
	fi
	if [ $MIRRORING -eq 0 ] && [ $MIRROR_TYPE -eq 1 ];then
		LOG_MSG "[WARN]:-Option --mirror-mode supplied, but no mirrors have been defined, ignoring --mirror-mode option" 1
		MIRROR_TYPE=0
	fi
	LOG_MSG "[INFO]:-Commencing multi-home checks, Completed" 1
	LOG_MSG "[INFO]:-End Function $FUNCNAME"
}


CHK_LOCALE_KNOWN () {
	if [ x"" == x"$LOCALE_SETTING" ]; then
		return # no explicit locale set, no check needed
	fi

	LOG_MSG "[INFO]:-Start Function $FUNCNAME"
	if [ $# -eq 0 ];then
		if [ `$LOCALE -a |$GREP -ic $(NORMALIZE_CODESET_IN_LOCALE $LOCALE_SETTING)` -eq 0 ] \
			&& [`$LOCALE -a |$GREP -ic $LOCALE_SETTING` -eq 0];then
			LOG_MSG "[INFO]:-Coordinator host available locale values"
			$LOCALE -a >> $LOG_FILE
			LOG_MSG "[FATAL]:-Unable to locate locale value $LOCALE_SETTING on this host" 1
			LOG_MSG "[INFO]:-Select another locale value via -n option to this utility" 1
			LOG_MSG "[INFO]:-Available locale values have been dumped to the log file" 1
			ERROR_EXIT "[FATAL]:-Unable to continue"
		else
			LOG_MSG "[INFO]:-Locale check passed on this host" 
		fi
	else
		# Hostname has been passed so check remote locale value
		if [ $( REMOTE_EXECUTE_AND_GET_OUTPUT $1 "$LOCALE -a|$GREP -ic '$(NORMALIZE_CODESET_IN_LOCALE $LOCALE_SETTING)'" ) -eq 0 ] \
			&& [ $( REMOTE_EXECUTE_AND_GET_OUTPUT $1 "$LOCALE -a|$GREP -ic '$LOCALE_SETTING'" ) -eq 0 ] ;then
			LOG_MSG "[INFO]:-Host $1 available locale values"
			`$TRUSTED_SHELL $1 "$LOCALE -a"` >> $LOG_FILE
			LOG_MSG "[FATAL]:-Unable to locate locale value $LOCALE_SETTING on $1" 1
			LOG_MSG "[INFO]:-Select another locale value via -n option to this utility" 1
			LOG_MSG "[INFO]:-Available locale values have been dumped to the log file" 1 
			ERROR_EXIT "[FATAL]:-Unable to continue"
		else
			LOG_MSG "[INFO]:-Locale check passed on $1" 
		fi
	fi
	LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

CHK_DUPLICATES () {
		LOG_MSG "[INFO]:-Start Function $FUNCNAME"
		if [ `$ECHO | $CAT $MACHINE_LIST_FILE - |$GREP -v $"^$"|$WC -l` -ne `$CAT $MACHINE_LIST_FILE|$GREP -v "^$"|$SORT -u|$WC -l` ]
		then
			ERROR_EXIT "[FATAL]:-Duplicate segment instance hostname exists in $MACHINE_LIST_FILE"
		fi
		LOG_MSG "[INFO]:-No duplicate segment instance hostnames found, will proceed"
		LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

CHK_OPEN_FILES () {
	LOG_MSG "[INFO]:-Start Function $FUNCNAME"
	LOG_MSG "[INFO]:-Checking $1"
	OPEN_VALUE=$( REMOTE_EXECUTE_AND_GET_OUTPUT $1 "ulimit -n" )
	if [ $OPEN_VALUE -lt $OS_OPENFILES ];then
		LOG_MSG "[WARN]:-Host $1 open files limit is $OPEN_VALUE should be >= $OS_OPENFILES" 1
		ULIMIT_WARN=1
	fi
	LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

CHK_QES () {
	LOG_MSG "[INFO]:-Start Function $FUNCNAME"
	LOG_MSG "[INFO]:-Checking Coordinator host" 1
	SET_VAR $QD_PRIMARY_ARRAY
	CHK_DIR $GP_DIR
	if [ $EXISTS -eq 0 ]; then
		ERROR_EXIT "[FATAL]:-Coordinator directory $GP_DIR already exists"
	fi
	GET_PG_PID_ACTIVE $GP_PORT
	if [ $PID -ne 0 ];then
		ERROR_EXIT "[FATAL]:-Found indication of postmaster process on port $GP_PORT on Coordinator host"
	fi
	LOG_MSG "[INFO]:-Checking new segment hosts, please wait..." 1

    CHECK_LIST=(`$ECHO ${MACHINE_LIST[@]} | $TR ' ' '\n' | $GREP -v "${HOSTNAME}\$" | $SORT | $TR '\n' ' '`)
    for QE_ID in ${CHECK_LIST[@]}
    do
        CHK_LOCALE_KNOWN $QE_ID
        CHK_OPEN_FILES $GP_HOSTADDRESS
        POSTGRES_VERSION_CHK $GP_HOSTADDRESS
        if [ $VERSION_MATCH -ne 1 ] ; then
            ERROR_EXIT "Postgres version does not match"
        fi

        # Check to ensure have required QE directories
        for DIR in "${DATA_DIRECTORY[@]}"
        do
            if [ $DEBUG_LEVEL -eq 0 ] && [ x"" != x"$VERBOSE" ];then $NOLINE_ECHO ".\c";fi
            LOG_MSG "[INFO]:-Checking segment instance $QE_ID directory $DIR"
            CHK_DIR $DIR $QE_ID
            if [ $EXISTS -ne 0 ]; then
                ERROR_EXIT "[FATAL]:-No $DIR on segment instance $QE_ID"
            else
                LOG_MSG "[INFO]:-$QE_ID $DIR checked"
            fi
        done
        # Check that we have a GP_LIBRARY_PATH directory on the QE
        CHK_DIR $GP_LIBRARY_PATH $QE_ID
        if [ $EXISTS -ne 0 ]; then
            ERROR_EXIT "[FATAL]:-No $GP_LIBRARY_PATH on segment instance $QE_ID"
        else
            LOG_MSG "[INFO]:-Segment instance $QE_ID $GP_LIBRARY_PATH checked"
        fi
    done

    # Check to ensure that instance directories do not exist on hosts
    LOG_MSG "[INFO]:-Primary segment instance directory check"
    for I in "${QE_PRIMARY_ARRAY[@]}"
    do
        if [ $DEBUG_LEVEL -eq 0 ] && [ x"" != x"$VERBOSE" ];then $NOLINE_ECHO ".\c";fi
        SET_VAR $I
        LOG_MSG "[INFO]:-Checking $GP_HOSTADDRESS for dir $GP_DIR"
        CHK_DIR $GP_DIR $GP_HOSTADDRESS
        if [ $EXISTS -eq 0 ]; then
            ERROR_EXIT "[FATAL]:-Instance directory $GP_DIR exists on segment instance $GP_HOSTADDRESS"
        fi
        # Check that the hostname is not associated with local host
        LOG_MSG "[INFO]:-Checking $GP_HOSTADDRESS $HOSTFILE for localhost set as $GP_HOSTADDRESS"
        LOCAL_COUNT=$( REMOTE_EXECUTE_AND_GET_OUTPUT $GP_HOSTADDRESS "$GREP $GP_HOSTADDRESS $HOSTFILE|$GREP -c localhost")
        if [ $LOCAL_COUNT -ne 0 ];then
            LOG_MSG "[WARN]:-----------------------------------------------------------" 1
            LOG_MSG "[WARN]:-Host $GP_HOSTADDRESS is assigned as localhost in $HOSTFILE"  1
            LOG_MSG "[WARN]:-This will cause segment->coordinator communication failures" 1
            LOG_MSG "[WARN]:-Remove $GP_HOSTADDRESS from local host line in /etc/hosts" 1
            LOG_MSG "[WARN]:-----------------------------------------------------------" 1
        fi
        # Check that we can write to the QE directory
        W_DIR=`$DIRNAME $GP_DIR`
        LOG_MSG "[INFO]:-Checking write access to $W_DIR on $GP_HOSTADDRESS"
        $TRUSTED_SHELL $GP_HOSTADDRESS "$TOUCH ${W_DIR}/tmp_file_test"
        RETVAL=$?
        if [ $RETVAL -ne 0 ];then
            ERROR_EXIT "[FATAL]:-Cannot write to $W_DIR on $GP_HOSTADDRESS "
        else
            $TRUSTED_SHELL $GP_HOSTADDRESS "$RM -f ${W_DIR}/tmp_file_test"
            LOG_MSG "[INFO]:-Write test passed on $GP_HOSTADDRESS $W_DIR directory"
            $ECHO "$ECHO \"Stopping segment instance on $GP_HOSTADDRESS\"" >> $BACKOUT_FILE
            $ECHO "$TRUSTED_SHELL $GP_HOSTADDRESS \"if [ -f $GP_DIR/postmaster.pid ]; then  ${EXPORT_LIB_PATH};export PGPORT=${GP_PORT}; $PG_CTL -w -D $GP_DIR -o \"-i -p ${GP_PORT}\" -m immediate  stop; fi\"" >> $BACKOUT_FILE
            $ECHO "$ECHO \"removing directory $GP_DIR on $GP_HOSTADDRESS\"" >> $BACKOUT_FILE
            $ECHO "$TRUSTED_SHELL ${GP_HOSTADDRESS} \"$RM -rf $GP_DIR > /dev/null 2>&1\"" >> $BACKOUT_FILE
        fi

    done
    # Check for mirror directories if mirroring configured
    if [ $MIRRORING -ne 0 ]; then
        LOG_MSG "[INFO]:-Mirror segment instance directory check"
        for I in "${QE_MIRROR_ARRAY[@]}"
        do
            if [ $DEBUG_LEVEL -eq 0 ] && [ x"" != x"$VERBOSE" ];then $NOLINE_ECHO ".\c";fi
            SET_VAR $I
            LOG_MSG "[INFO]:-Checking $GP_HOSTADDRESS for dir $GP_DIR"
            CHK_DIR $GP_DIR $GP_HOSTADDRESS
            if [ $EXISTS -eq 0 ]; then
                ERROR_EXIT "[FATAL]:-Instance directory $GP_DIR exists on segment instance $GP_HOSTADDRESS"
            fi
            # Check that we can write to the QE directory
            W_DIR=`$DIRNAME $GP_DIR`
            LOG_MSG "[INFO]:-Checking write access to $W_DIR on $GP_HOSTADDRESS"
            $TRUSTED_SHELL $GP_HOSTADDRESS "$TOUCH ${W_DIR}/tmp_file_test"
            RETVAL=$?
            if [ $RETVAL -ne 0 ];then
                ERROR_EXIT "[FATAL]:-Cannot write to $W_DIR on $GP_HOSTADDRESS "
            else
                $TRUSTED_SHELL $GP_HOSTADDRESS "$RM -f ${W_DIR}/tmp_file_test"
                LOG_MSG "[INFO]:-Write test passed on $GP_HOSTADDRESS $W_DIR directory"
                $ECHO "$ECHO \"Stopping segment instance on $GP_HOSTADDRESS\"" >> $BACKOUT_FILE
                $ECHO "$TRUSTED_SHELL $GP_HOSTADDRESS \"if [ -f $GP_DIR/postmaster.pid ]; then  ${EXPORT_LIB_PATH};export PGPORT=${GP_PORT}; $PG_CTL -w -D $GP_DIR -o \"-i -p ${GP_PORT}\" -m immediate  stop; fi\"" >> $BACKOUT_FILE
                $ECHO "$ECHO \"removing directory $GP_DIR on $GP_HOSTADDRESS\"" >> $BACKOUT_FILE
                $ECHO "$TRUSTED_SHELL ${GP_HOSTADDRESS} \"$RM -rf $GP_DIR > /dev/null 2>&1\"" >> $BACKOUT_FILE
            fi
        done
    else
        LOG_MSG "[INFO]:-Mirror segment instance directory check skipped, mirroring=Off"
    fi

    if [ x"" != x"$STANDBY_HOSTNAME" ];then
        PING_HOST $STANDBY_HOSTNAME 
        CHK_LOCALE_KNOWN $STANDBY_HOSTNAME
        CHK_OPEN_FILES $GP_HOSTADDRESS
        # Check standby host directory
        CHK_DIR $COORDINATOR_DIRECTORY $STANDBY_HOSTNAME
        if [ $EXISTS -ne 0 ];then
            ERROR_EXIT "[FATAL]:-Standby host directory $COORDINATOR_DIRECTORY, does not exist"
        fi
        if [ x"$FTS_HOST_CONFIG" = x"" ] && [ x"$CLUSTER_BOOT_MODE" = x"PRODUCTION" ]; then
            FTS_HOST_MACHINE_LIST[2]=$STANDBY_HOSTNAME
        fi
        if [ x"$ETCD_HOST_CONFIG" = x"" ] && [ x"$CLUSTER_BOOT_MODE" = x"PRODUCTION" ]; then
            ETCD_HOST_MACHINE_LIST[2]=$STANDBY_HOSTNAME
        fi
    fi

	if [ $DEBUG_LEVEL -eq 0 ] && [ x"" != x"$VERBOSE" ];then $ECHO;fi

	LOG_MSG "[INFO]:-Checking new segment hosts, Completed" 1
	LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

CHK_SEG () {
    LOG_MSG "[INFO]:-Start Function $FUNCNAME"
    SET_VAR $1
    
    # We need QE_ID to be able to re-use some functions.
    QE_ID=$GP_HOSTADDRESS
    CHK_LOCALE_KNOWN $QE_ID
	CHK_OPEN_FILES $GP_HOSTADDRESS
	POSTGRES_VERSION_CHK $GP_HOSTADDRESS
	if [ $VERSION_MATCH -ne 1 ] ; then
	   ERROR_EXIT "Postgres version does not match"
	fi
	
	#make sure we can write to it.
	# Check that we can write to the QE directory
    W_DIR=`$DIRNAME $GP_DIR`
    LOG_MSG "[INFO]:-Checking write access to $W_DIR on $GP_HOSTADDRESS"
    $TRUSTED_SHELL $GP_HOSTADDRESS "$TOUCH ${W_DIR}/tmp_file_test"
    RETVAL=$?
    if [ $RETVAL -ne 0 ];then
         ERROR_EXIT "[FATAL]:-Cannot write to $W_DIR on $GP_HOSTADDRESS "
    else
         $TRUSTED_SHELL $GP_HOSTADDRESS "$RM -f ${W_DIR}/tmp_file_test"
         LOG_MSG "[INFO]:-Write test passed on $GP_HOSTADDRESS $W_DIR directory"
    fi
    
	# Check that we have a GP_LIBRARY_PATH directory on the QE
	CHK_DIR $GP_LIBRARY_PATH $GP_HOSTADDRESS
	if [ $EXISTS -ne 0 ]; then
		ERROR_EXIT "[FATAL]:-No $GP_LIBRARY_PATH on segment $GP_HOSTADDRESS:$GP_DIR"
	else
		LOG_MSG "[INFO]:-Segment instance $GP_HOSTADDRESS:$GP_DIR $GP_LIBRARY_PATH checked"
	fi
	LOG_MSG "[INFO]:-End Function $FUNCNAME"

}

CHK_QES_FROM_INPUTFILE () {
	LOG_MSG "[INFO]:-Start Function $FUNCNAME"
	LOG_MSG "[INFO]:-Checking Coordinator host" 1
	SET_VAR $QD_PRIMARY_ARRAY
	CHK_DIR $GP_DIR
	if [ $EXISTS -eq 0 ]; then
		ERROR_EXIT "[FATAL]:-Coordinator directory $GP_DIR already exists"
	fi
	GET_PG_PID_ACTIVE $GP_PORT
	if [ $PID -ne 0 ];then
		ERROR_EXIT "[FATAL]:-Found indication of postmaster process on port $GP_PORT on Coordinator host"
	fi
	LOG_MSG "[INFO]:-Checking new segment hosts, please wait..." 1
	
	for QE in ${QE_PRIMARY_ARRAY[@]}
	do
		CHK_SEG $QE            
	done
	
	for QE in ${QE_MIRROR_ARRAY[@]}
	do
		CHK_SEG $QE            
	done
	        
	if [ x"" != x"$STANDBY_HOSTNAME" ];then
		PING_HOST $STANDBY_HOSTNAME 
		CHK_LOCALE_KNOWN $STANDBY_HOSTNAME
		CHK_OPEN_FILES $GP_HOSTADDRESS
		#Check standby host directory
		CHK_DIR $COORDINATOR_DIRECTORY $STANDBY_HOSTNAME
		if [ $EXISTS -ne 0 ];then
			ERROR_EXIT "[FATAL]:-Standby host directory $COORDINATOR_DIRECTORY, does not exist"
		fi
 		if [ x"$FTS_HOST_CONFIG" = x"" ] && [ x"$CLUSTER_BOOT_MODE" = x"PRODUCTION" ]; then
			FTS_HOST_MACHINE_LIST[2]=$STANDBY_HOSTNAME
		fi
		if [ x"$ETCD_HOST_CONFIG" = x"" ] && [ x"$CLUSTER_BOOT_MODE" = x"PRODUCTION" ]; then
			ETCD_HOST_MACHINE_LIST[2]=$STANDBY_HOSTNAME
		fi
	fi
	
	if [ $DEBUG_LEVEL -eq 0 ] && [ x"" != x"$VERBOSE" ];then $ECHO;fi
	LOG_MSG "[INFO]:-Checking new segment hosts, Completed" 1
	LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

POSTGRES_PORT_CHK () {
	LOG_MSG "[INFO]:-Start Function $FUNCNAME"
	GET_PG_PID_ACTIVE $1 $2
	if [ x"$PID" != x0 ];then
		ERROR_EXIT "[FATAL]:-Host $2 has an active database process on port = $1"
	fi
	LOG_MSG "[INFO]:-End Function $FUNCNAME"
}


CREATE_QE_ARRAY () {
	LOG_MSG "[INFO]:-Start Function $FUNCNAME"
	LOG_MSG "[INFO]:-Building primary segment instance array, please wait..." 1
	#Set up initial variables
	. $CLUSTER_CONFIG
	if [ x"" != x"$REQ_LOCALE_SETTING" ];then LOCALE_SETTING=$REQ_LOCALE_SETTING;fi
	if [ `$ECHO ${M_HOST_ARRAY[@]}|$TR ' ' '\n'|$GREP -c "~${COORDINATOR_HOSTNAME}\$"` -gt 0 ]; then
		COORDINATOR_LIST=(`$ECHO ${M_HOST_ARRAY[@]}|$TR ' ' '\n'|$GREP "~${COORDINATOR_HOSTNAME}\$"`)
		M_HOST_ARRAY=(${COORDINATOR_LIST[@]} `$ECHO ${M_HOST_ARRAY[@]}|$TR ' ' '\n'|$GREP -v "~${COORDINATOR_HOSTNAME}\$"`)
	fi
	if [ $MIRRORING -eq 1 ] && [ $MIRROR_TYPE -eq 0 ];then
		# Move first host to other end of the array
		# Get first hostname
		HOST_MOVE=`$ECHO ${M_HOST_ARRAY[0]}|$AWK -F"~" '{print $2}'`
		M_MIR_HOST_ARRAY=(`$ECHO ${M_HOST_ARRAY[@]}|$TR ' ' '\n'|$GREP -v "~${HOST_MOVE}"|$TR '\n' ' '` `$ECHO ${M_HOST_ARRAY[@]}|$TR ' ' '\n'|$GREP "~${HOST_MOVE}"|$TR '\n' ' '`)
	fi
	DBID_COUNT=1
	CONTENT_COUNT=-1
	PORT_COUNT=0
	LAST_HOST=X
	SEG_DIR_VECTOR=0
	MIR_COUNT=-1
	COORDINATOR_HOST=$COORDINATOR_HOSTNAME
	QD_PRIMARY_ARRAY=${COORDINATOR_HOST}~${COORDINATOR_HOSTNAME}~${COORDINATOR_PORT}~${COORDINATOR_DIRECTORY}/${SEG_PREFIX}${CONTENT_COUNT}~${DBID_COUNT}~${CONTENT_COUNT}~0
	((DBID_COUNT=$DBID_COUNT+1));((CONTENT_COUNT=$CONTENT_COUNT+1))
	for QE_PAIR in ${M_HOST_ARRAY[@]}
	do
		if [ $DEBUG_LEVEL -eq 0 ] && [ x"" != x"$VERBOSE" ];then $NOLINE_ECHO ".\c";fi
		QE_NAME=`$ECHO $QE_PAIR|$AWK -F"~" '{print $1}'`
		QE_HOST=`$ECHO $QE_PAIR|$AWK -F"~" '{print $2}'`
		if [ $LAST_HOST == $QE_HOST ];then
			((PORT_COUNT=$PORT_COUNT+1))
		else
			SEG_DIR_VECTOR=0
			PORT_COUNT=0
		fi
		((GP_PORT=$PORT_BASE+$PORT_COUNT))
		
		GP_DIR=${DATA_DIRECTORY[$SEG_DIR_VECTOR]}
		QE_PRIMARY_ARRAY=(${QE_PRIMARY_ARRAY[@]} ${QE_HOST}~${QE_NAME}~${GP_PORT}~${GP_DIR}/${SEG_PREFIX}${CONTENT_COUNT}~${DBID_COUNT}~$CONTENT_COUNT)
		POSTGRES_PORT_CHK $GP_PORT $QE_NAME
		((DBID_COUNT=$DBID_COUNT+1))
		((CONTENT_COUNT=$CONTENT_COUNT+1))
		((SEG_DIR_VECTOR=$SEG_DIR_VECTOR+1))
		LAST_HOST=$QE_HOST
	done
	if [ $DEBUG_LEVEL -eq 0 ] && [ x"" != x"$VERBOSE" ];then $ECHO;fi
	if [ $MIRRORING -eq 1 ] && [ "$SINGLENODE_MODE" == "false" ] ;then
		((MIRROR_OFFSET=$MIRROR_PORT_BASE-$PORT_BASE))
		if [ $MIRROR_TYPE -eq 1 ];then
			CREATE_SPREAD_MIRROR_ARRAY
		else
			CREATE_GROUP_MIRROR_ARRAY
		fi
	fi 
	((TOTAL_SEG=${#QE_PRIMARY_ARRAY[@]}))
	LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

ARRAY_REORDER() {
    LOG_MSG "[INFO]:-Start Function $FUNCNAME"

    # Now re-order the array so that it is ordered by PORT to spread out the 
    # number of parallel processes initiated on each host
    #
    # MPP-13617: If segment contains a ~, we assume ~ is the field delimiter.
    # Otherwise we assume : is the delimiter.  This allows us to easily 
    # handle IPv6 addresses which may contain a : by using a ~ as a delimiter. 
    # 
    case `$ECHO ${QE_PRIMARY_ARRAY[@]}` in 
        *~*)
	    S="~"
            ;;
        *)
	    S=":"
            ;;
    esac

    QE_REORDER_ARRAY=(`$ECHO ${QE_PRIMARY_ARRAY[@]}|$TR ' ' '\n'|$SORT -t$S -n -k3,3|$TR '\n' ' '`)
    QE_PRIMARY_ARRAY=(${QE_REORDER_ARRAY[@]})
    if [ $MIRROR_TYPE -eq 1 ];then
	QE_REORDER_ARRAY=(`$ECHO ${QE_MIRROR_ARRAY[@]}|$TR ' ' '\n'|$SORT -t$S -n -k3,3|$TR '\n' ' '`)
	QE_MIRROR_ARRAY=(${QE_REORDER_ARRAY[@]})
    fi
    LOG_MSG "[INFO]:-End Function $FUNCNAME"
}


CREATE_ARRAY_SORTED_ON_CONTENT_ID() {
    LOG_MSG "[INFO]:-Start Function $FUNCNAME"

    local REORDERING_ON_CONTENT

    REORDERING_ON_CONTENT=(`$ECHO ${QE_PRIMARY_ARRAY[@]}|$TR ' ' '\n'|$SORT -t$S -n -k6,6|$TR '\n' ' '`)
    QE_PRIMARY_ARRAY_SORTED_ON_CONTENT_ID=(${REORDERING_ON_CONTENT[@]})
    if [ $MIRRORING -ne 0 ] ; then
      REORDERING_ON_CONTENT=(`$ECHO ${QE_MIRROR_ARRAY[@]}|$TR ' ' '\n'|$SORT -t$S -n -k6,6|$TR '\n' ' '`)
      QE_MIRROR_ARRAY_SORTED_ON_CONTENT_ID=(${REORDERING_ON_CONTENT[@]})
    fi
    LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

DISPLAY_CONFIG () {
		LOG_MSG "[INFO]:-Start Function $FUNCNAME"
		if [ x"" == x"$INPUT_CONFIG" ] ; then
		    . $CLUSTER_CONFIG
		    if [ x"" != x"$REQ_LOCALE_SETTING" ];then
			LOCALE_SETTING=$REQ_LOCALE_SETTING;
		    fi
		    if [ x"" != x"$MACHINE_LIST_FILE_ALT" ];then
			MACHINE_LIST_FILE=$MACHINE_LIST_FILE_ALT
		    fi

		    if [ `$CAT ${MACHINE_LIST_FILE}| $GREP -c "${COORDINATOR_HOSTNAME}\$"` -eq 0 ]; then
			declare -a MACHINE_LIST=(`$CAT $MACHINE_LIST_FILE`)
		    else
			declare -a MACHINE_LIST=($COORDINATOR_HOSTNAME `$CAT $MACHINE_LIST_FILE|$GREP -v "${COORDINATOR_HOSTNAME}\$"`)
		    fi
		fi
		((NUM_QES=${#MACHINE_LIST[*]}))
		if [  x"" == x"$PG_CONF_ADD_FILE" ]; then
				PG_ADD=Off
		else
				PG_ADD=On
		fi
		export QD_DIR=$GP_DIR
	if [ x"" != x"$INTERACTIVE" ];then
		SET_VAR $QD_PRIMARY_ARRAY
		LOG_MSG "[INFO]:-Apache Cloudberry Creation Parameters" 1
		LOG_MSG "[INFO]:---------------------------------------" 1
		LOG_MSG "[INFO]:-Coordinator Configuration" 1
		LOG_MSG "[INFO]:---------------------------------------" 1
		LOG_MSG "[INFO]:-Coordinator hostname       = $GP_HOSTADDRESS" 1
		LOG_MSG "[INFO]:-Coordinator port           = $COORDINATOR_PORT" 1
		LOG_MSG "[INFO]:-Coordinator instance dir   = $GP_DIR" 1
		LOG_MSG "[INFO]:-Coordinator LOCALE         = $LOCALE_SETTING" 1
		LOG_MSG "[INFO]:-Cloudberry segment prefix   = $SEG_PREFIX" 1
		LOG_MSG "[INFO]:-Coordinator Database       = $DATABASE_NAME" 1
		LOG_MSG "[INFO]:-Coordinator connections    = $COORDINATOR_MAX_CONNECT" 1
		LOG_MSG "[INFO]:-Coordinator buffers        = $COORDINATOR_SHARED_BUFFERS" 1
		LOG_MSG "[INFO]:-Segment connections        = $QE_MAX_CONNECT" 1
		LOG_MSG "[INFO]:-Segment buffers            = $QE_SHARED_BUFFERS" 1
		LOG_MSG "[INFO]:-Encoding                   = $ENCODING" 1
		LOG_MSG "[INFO]:-Postgres param file        = $PG_ADD" 1
		LOG_MSG "[INFO]:-Initdb to be used          = $INITDB" 1
		LOG_MSG "[INFO]:-GP_LIBRARY_PATH is         = $GP_LIBRARY_PATH" 1
		LOG_MSG "[INFO]:-HEAP_CHECKSUM is           = $HEAP_CHECKSUM" 1
		LOG_MSG "[INFO]:-HBA_HOSTNAMES is           = $HBA_HOSTNAMES" 1
		if [ $ULIMIT_WARN -eq 1 ];then
			LOG_MSG "[WARN]:-Ulimit check               = Warnings generated, see log file $WARN_MARK" 1
		else
			LOG_MSG "[INFO]:-Ulimit check               = Passed" 1
		fi
		if [ $MULTI_HOME -eq 0 ];then
			LOG_MSG "[INFO]:-Array host connect type    = Single hostname per node" 1
		else
			LOG_MSG "[INFO]:-Array host connect type    = Multi hostname per node" 1
		fi
		IP_COUNT=1
		for COORDINATOR_IP in "${COORDINATOR_IP_ADDRESS[@]}"
		do
			LOG_MSG "[INFO]:-Coordinator IP address [$IP_COUNT]      = $COORDINATOR_IP" 1
			((IP_COUNT=$IP_COUNT+1))
		done

        if [ x"" != x"$STANDBY_HOSTNAME" ];then
            LOG_MSG "[INFO]:-Standby Coordinator             = $STANDBY_HOSTNAME" 1
        else
            LOG_MSG "[INFO]:-Standby Coordinator             = Not Configured" 1
        fi
        LOG_MSG "[INFO]:-Number of primary segments = $QE_PRIMARY_COUNT" 1
        if [ x"" != x"$STANDBY_HOSTNAME" ];then
            for STANDBY_IP in "${STANDBY_IP_ADDRESS[@]}"
            do
                LOG_MSG "[INFO]:-Standby IP address         = $STANDBY_IP" 1
            done
        fi
        LOG_MSG "[INFO]:-Total Database segments    = $TOTAL_SEG" 1
        LOG_MSG "[INFO]:-Trusted shell              = $TRUSTED_SHELL" 1
        ((NUM_QES=${#MACHINE_LIST[*]}))
        LOG_MSG "[INFO]:-Number segment hosts       = $NUM_QES" 1
        if [ $MIRROR_PORT_BASE ]; then
            LOG_MSG "[INFO]:-Mirror port base           = $MIRROR_PORT_BASE" 1
            ((NUM_MIRROR_DIRECTORY=${#MIRROR_DATA_DIRECTORY[@]}))
            LOG_MSG "[INFO]:-Number of mirror segments  = $NUM_MIRROR_DIRECTORY" 1
            LOG_MSG "[INFO]:-Mirroring config           = ON" 1
            if [ x"$INPUT_CONFIG" != x"" ];then
                LOG_MSG "[INFO]:-Mirroring type             = Custom" 1
            else
                if [ $MIRROR_TYPE -eq 0 ];then
                    LOG_MSG "[INFO]:-Mirroring type             = Group" 1
                else
                    LOG_MSG "[INFO]:-Mirroring type             = Spread" 1
                fi
            fi
        else
            LOG_MSG "[INFO]:-Mirroring config           = OFF" 1
        fi
        LOG_MSG "[INFO]:----------------------------------------" 1
        LOG_MSG "[INFO]:-Cloudberry Primary Segment Configuration" 1
        LOG_MSG "[INFO]:----------------------------------------" 1
        for I in "${QE_PRIMARY_ARRAY[@]}"
        do
            TXT=`$ECHO $I|$AWK -F"~" '{print $1" \t"$3" \t"$2" \t"$4" \t"$5}'`
            LOG_MSG "[INFO]:-$TXT" 1
        done
        if [ $MIRRORING -ne 0 ]; then
            LOG_MSG "[INFO]:---------------------------------------" 1
            LOG_MSG "[INFO]:-Cloudberry Mirror Segment Configuration" 1
            LOG_MSG "[INFO]:---------------------------------------" 1
            for I in  "${QE_MIRROR_ARRAY[@]}"
            do
                TXT=`$ECHO $I|$AWK -F"~" '{print $1" \t"$3" \t"$2" \t"$4" \t"$5}'`
                LOG_MSG "[INFO]:-$TXT" 1
            done
        fi

		GET_REPLY "Continue with Cloudberry creation" 
	fi
	LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

# Retrieve locale settings from the coordinator and set locale variables
# appropriately so those values can be used when initializing the segments.
SET_LOCALE_VARS_BASED_ON_COORDINATOR() {
	# The below command exports a set of database values like
	#
	#     name    |  setting
	# ------------+------------
	#  lc_collate | en_US.utf8
	#  lc_ctype   | en_US.utf8
	#  ...        | ...
	#
	# in the format
	#
	# --lc-collate=en_US.utf8 --lc-ctype=en_US.utf8 ...
	psql_cmd="SELECT '--' || REPLACE(name, '_', '-') || '=' || setting FROM pg_settings WHERE name LIKE 'lc_%' ORDER BY name"
	export LC_ALL_SETTINGS=$(env PGOPTIONS="-c gp_role=utility" $PSQL -d template1 -X -A -t -c "$psql_cmd")
}

CREATE_QD_DB () {
		LOG_MSG "[INFO]:-Start Function $FUNCNAME"
		LOG_MSG "[INFO]:-Building the Coordinator instance database, please wait..." 1
		SET_VAR $QD_PRIMARY_ARRAY
		LOG_MSG "[INFO]:-Initializing Coordinator Postgres instance $GP_DIR"
		$EXPORT_LIB_PATH

		if [ x"" != x"$LCCOLLATE" ]; then
			LC_COLLATE_SETTING="--lc-collate=$LCCOLLATE"
		fi
		
		if [ x"" != x"$LCCTYPE" ]; then
			LC_CTYPE_SETTING="--lc-ctype=$LCCTYPE"
		fi
		
		if [ x"" != x"$LCMESSAGES" ]; then
			LC_MESSAGES_SETTING="--lc-messages=$LCMESSAGES"
		fi
		
		if [ x"" != x"$LCMONETARY" ]; then
			LC_MONETARY_SETTING="--lc-monetary=$LCMONETARY"
		fi
		
		if [ x"" != x"$LCNUMERIC" ]; then
			LC_NUMERIC_SETTING="--lc-numeric=$LCNUMERIC"
		fi
		
		if [ x"" != x"$LCTIME" ]; then
			LC_TIME_SETTING="--lc-time=$LCTIME"
		fi

		# Since we explicitly set each LC_* setting, we do not need to pass a --locale option to initdb
		LC_ALL_SETTINGS=" $LC_COLLATE_SETTING $LC_CTYPE_SETTING $LC_MESSAGES_SETTING $LC_MONETARY_SETTING $LC_NUMERIC_SETTING $LC_TIME_SETTING"

		# build initdb command
		cmd="$INITDB"
		cmd="$cmd -E $ENCODING"
		cmd="$cmd -D $GP_DIR"
		if [ x"" != x"$ENCRYPT_METHOD" ]; then
			cmd="$cmd -K $ENCRYPT_METHOD"
		fi
		if [ x"" != x"$CLUSTER_KEY_CMD" ]; then
			cmd="$cmd -c $CLUSTER_KEY_CMD -R"
		fi
		
		cmd="$cmd $LC_ALL_SETTINGS"
		cmd="$cmd --max_connections=$COORDINATOR_MAX_CONNECT"
		cmd="$cmd --shared_buffers=$COORDINATOR_SHARED_BUFFERS"
		if [ x"$HEAP_CHECKSUM" == x"on" ]; then
			cmd="$cmd --data-checksums"
		fi
		LOG_MSG "[INFO]:-Commencing local $cmd"
		$cmd >> $LOG_FILE 2>&1
		RETVAL=$?

		if [ $RETVAL -ne 0 ]; then
		    ERROR_EXIT "[FATAL]:- Command $cmd failed with error status $RETVAL, see log file $LOG_FILE for more detail"
		fi

		BACKOUT_COMMAND "if [ -d $GP_DIR ]; then $RM -Rf $GP_DIR; fi"
		BACKOUT_COMMAND "$ECHO Removing Coordinator data directory files"
		LOG_MSG "[INFO]:-Completed Coordinator instance initialization"
		$ECHO "#Cloudberry specific configuration parameters for Coordinator instance database" >> ${GP_DIR}/$PG_CONF
		$ECHO "#------------------------------------------------------------------------" >> ${GP_DIR}/$PG_CONF
		LOG_MSG "[INFO]:-Setting the Coordinator port to $GP_PORT"
		SED_PG_CONF ${GP_DIR}/$PG_CONF "$PORT_TXT" port=$GP_PORT 0
		ERROR_CHK $? "set Coordinator port=$GP_PORT in $PG_CONF" 2
		LOG_MSG "[INFO]:-Completed setting the Coordinator port to $COORDINATOR_PORT"
		LOG_MSG "[INFO]:-Setting the Coordinator listen addresses to '*'"
		SED_PG_CONF ${GP_DIR}/$PG_CONF "$LISTEN_ADR_TXT" listen_addresses=\'*\' 0
		ERROR_CHK $? "set Coordinator listen addresses to '*' in $PG_CONF" 2
		LOG_MSG "[INFO]:-Completed setting the listen addresses to '*'"
		LOG_MSG "[INFO]:-Setting Coordinator logging option"
		SED_PG_CONF ${GP_DIR}/$PG_CONF "$LOG_STATEMENT_TXT" log_statement=all 0
		ERROR_CHK $? "set log_statement=all in ${GP_DIR}/$PG_CONF" 1
		LOG_MSG "[INFO]:-Setting Coordinator instance check point segments"

		LOG_MSG "[INFO]:-Setting Coordinator instance content id"
		SED_PG_CONF ${GP_DIR}/$PG_CONF "$CONTENT_ID_TXT" "gp_contentid=$GP_CONTENT" 0
		ERROR_CHK $? "set gp_contentid=$GP_CONTENT in ${GP_DIR}/$PG_CONF" 1

		LOG_MSG "[INFO]:-Setting Coordinator instance db id"
		SED_PG_CONF ${GP_DIR}/$PG_INTERNAL_CONF "$DBID_TXT" "gp_dbid=$GP_DBID" 0
		ERROR_CHK $? "set gp_dbid=$GP_DBID in ${GP_DIR}/$PG_INTERNAL_CONF" 1

		if [ "$SINGLENODE_MODE" == "true" ]; then
				LOG_MSG "[INFO]:-Setting Coordinator singlenodeMode"
				SED_PG_CONF ${GP_DIR}/$PG_CONF "$SINGLENODE_MODE_TXT" "gp_internal_is_singlenode=on" 0
				ERROR_CHK $? "set gp_internal_is_singlenode=on in ${GP_DIR}/$PG_CONF" 1
		fi

		if [  x"" != x"$PG_CONF_ADD_FILE" ]; then
				LOG_MSG "[INFO]:-Processing additional configuration parameters"
				for NEW_PARAM in `$CAT $PG_CONF_ADD_FILE|$TR -s ' '|$TR -d ' '|$GREP -v "^#"`
						do
						LOG_MSG "[INFO]:-Adding config $NEW_PARAM to Coordinator"
						SEARCH_TXT=`$ECHO $NEW_PARAM |$CUT -d"=" -f1`
						SED_PG_CONF ${GP_DIR}/$PG_CONF $SEARCH_TXT $NEW_PARAM 0
						ERROR_CHK $? "set $NEW_PARAM ${GP_DIR}/$PG_CONF" 1
				done
		fi
		LOG_MSG "[INFO]:-Adding gp_dumpall access to $PG_HBA for coordinator host"
		BUILD_COORDINATOR_PG_HBA_FILE $GP_DIR $HBA_HOSTNAMES
		LOG_MSG "[INFO]:-Creating gpssh configuration file"
		BUILD_GPSSH_CONF $GP_DIR
		LOG_MSG "[INFO]:-Starting the Coordinator in admin mode" 1
		export PGPORT=$GP_PORT;$PG_CTL -w -l $GP_DIR/log/startup.log -D $GP_DIR -o "-i -p $GP_PORT -c gp_role=utility \
		-m" start >> ${LOG_FILE} 2>&1
		RET_TEXT="`$PG_CTL status -D $GP_DIR`"
		RUNNING=`$ECHO $RET_TEXT|$GREP -E -c "not running|neither"`
		if [ $RUNNING -ne 0 ]; then
				$CAT ${GP_DIR}.log|$TEE -a $LOG_FILE
				ERROR_EXIT "[FATAL]:-Failed to start the Coordinator database in admin mode"
		fi
		BACKOUT_COMMAND "$RM -f /tmp/.s.PGSQL.${GP_PORT}*"
		BACKOUT_COMMAND "$ECHO \"Removing Coordinator lock files\""
		BACKOUT_COMMAND "$RM -f ${GP_DIR}.log"
		BACKOUT_COMMAND "$ECHO Removing Coordinator log file"
		BACKOUT_COMMAND "if [ -d $GP_DIR ]; then $EXPORT_LIB_PATH;export PGPORT=$GP_PORT; $PG_CTL -D $GP_DIR stop; fi"
		BACKOUT_COMMAND "$ECHO \"Stopping Coordinator instance\""
		LOG_MSG "[INFO]:-Completed starting the Coordinator in admin mode"
		GP_HOSTNAME=$GP_HOSTADDRESS
		PING_HOST $GP_HOSTNAME
		RETVAL=$?
		if [ $RETVAL -ne 0 ]; then
			ERROR_EXIT "[FATAL]:-Could not establish connection to hostname $GP_HOSTNAME. Please check your configuration."
		fi
		UPDATE_GPCONFIG $GP_PORT $GP_DBID $GP_CONTENT $GP_HOSTNAME $GP_HOSTADDRESS $GP_PORT $GP_DIR p
		SET_LOCALE_VARS_BASED_ON_COORDINATOR
		if [ "$SINGLENODE_MODE" == "false" ]; then
			LOAD_QE_SYSTEM_DATA $DEFAULTDB
		fi
		SET_VAR $QD_PRIMARY_ARRAY
		LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

UPDATE_GPCONFIG () {
		LOG_MSG "[INFO]:-Start Function $FUNCNAME"
		COORDINATOR_PORT=$1
		U_DBID=$2
		U_CONTENT=$3
		U_HOSTNAME=$4
		U_ADDRESS=$5
		U_PORT=$6
		U_DIR=$7
		U_ROLE=$8

		U_DB=$DEFAULTDB
		CHK_COUNT=`env PGOPTIONS="-c gp_role=utility" $PSQL -p $COORDINATOR_PORT -d "$U_DB" -X -A -t -c "SELECT count(*) FROM $GP_CONFIG_TBL WHERE content=${U_CONTENT} AND preferred_role='${U_ROLE}';" 2>>${LOG_FILE}` >> $LOG_FILE 2>&1
		ERROR_CHK $? "obtain psql count Coordinator $GP_CONFIG_TBL" 2
		if [ $CHK_COUNT -eq 0 ]; then
				LOG_MSG "[INFO]:-Adding $U_CONTENT on $U_HOSTNAME, path $U_DIR to Coordinator gp_segment_configuration"
				env PGOPTIONS="-c gp_role=utility" $PSQL -p $COORDINATOR_PORT -d "$U_DB" -c "SELECT pg_catalog.gp_add_segment(${U_DBID}::int2, ${U_CONTENT}::int2, '${U_ROLE}', '${U_ROLE}', 's', 'u', ${U_PORT}, '${U_HOSTNAME}', '${U_ADDRESS}', '${U_DIR}');" >> $LOG_FILE 2>&1
				ERROR_CHK $? "add $U_CONTENT on $U_HOSTNAME in dir $U_DIR to Coordinator gp_segment_configuration" 2
		else
				LOG_MSG "[INFO]:-Content $U_CONTENT already exists in gp_segment_configuration system table"
		fi
		LOG_MSG "[INFO]:-End Function $FUNCNAME"
}


LOAD_QE_SYSTEM_DATA () {
		LOG_MSG "[INFO]:-Start Function $FUNCNAME"
		SET_VAR $QD_PRIMARY_ARRAY
		TARGET_DB=$1
		QD_DBNAME=$DEFAULTDB
		for I in "${QE_PRIMARY_ARRAY[@]}"
		do
				SET_VAR $I
				LOG_MSG "[INFO]:-Adding segment $GP_HOSTADDRESS to Coordinator system tables"
				GP_HOSTNAME=$GP_HOSTADDRESS
				PING_HOST $GP_HOSTNAME
				RETVAL=$?
				if [ $RETVAL -ne 0 ]; then
					ERROR_EXIT "[FATAL]:-Could not establish connection to hostname $GP_HOSTNAME. Please check your configuration."
				fi
				UPDATE_GPCONFIG $COORDINATOR_PORT $GP_DBID $GP_CONTENT $GP_HOSTNAME $GP_HOSTADDRESS $GP_PORT $GP_DIR p
				LOG_MSG "[INFO]:-Successfully added segment $GP_HOSTADDRESS to Coordinator system tables"
		done
		LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

CREATE_SEGMENT () {
		LOG_MSG "[INFO]:-Start Function $FUNCNAME"
		local PREFERRED_ROLE=$1
		#
		# PARALLEL BUILD OF SEGMENTS
		# SEGMENT_ARRAY contains the array of segments which will be created in this invocation
		local ROLE_TEXT
		if [ x"IS_PRIMARY" == x"$PREFERRED_ROLE" ]; then
			ROLE_TEXT="primary segment"
			SEGMENT_ARRAY=(${QE_PRIMARY_ARRAY_SORTED_ON_CONTENT_ID[@]})
		else
			ROLE_TEXT="mirror segment"
			SEGMENT_ARRAY=(${QE_MIRROR_ARRAY_SORTED_ON_CONTENT_ID[@]})
		fi

		LOG_MSG "[INFO]:-Commencing parallel build of $ROLE_TEXT instances" 1

		BATCH_LIMIT=${#QE_PRIMARY_ARRAY[@]}
		PARALLEL_SETUP  $PARALLEL_STATUS_FILE
		HAS_MIRRORS_OPTION=no
		if [ $MIRRORING -ne 0 ]; then
			HAS_MIRRORS_OPTION=yes
			I_INDEX=0
		fi

		for SEGMENT_INFO in "${SEGMENT_ARRAY[@]}"
		do
			export PG_CONF_ADD_FILE
			export STANDBY_HOSTNAME
			export ENCODING
			export LOCALE_SETTING
			export LC_ALL_SETTINGS
			export BACKOUT_FILE
			export LOG_FILE
			export PARALLEL_STATUS_FILE
			export TOTAL_SEG
			export QE_MAX_CONNECT
			export QE_SHARED_BUFFERS
			export SEG_PREFIX
			export ENCRYPT_METHOD
			export CLUSTER_KEY_CMD
			if [ $DEBUG_LEVEL -eq 0 ] && [ x"" != x"$VERBOSE" ];then $NOLINE_ECHO ".\c";fi
				FLAG=""
			if [ x"" != x"$PG_CONF_ADD_FILE" ] ; then
				FLAG="-p $PG_CONF_ADD_FILE"
			fi

			# PAIR_SEGMENT_INFO contains the corresponding pair with same content id if mirroring is enabled
			PAIR_SEGMENT_INFO=""
			if [ x"IS_PRIMARY" == x"$PREFERRED_ROLE" ]  && [ $MIRRORING -ne 0 ]; then
				# ex: contains the corresponding pair with same content id if mirroring is enabled
				PAIR_SEGMENT_INFO=${QE_MIRROR_ARRAY_SORTED_ON_CONTENT_ID[$I_INDEX]}
			elif [ x"IS_MIRROR" == x"$PREFERRED_ROLE" ]; then
				PAIR_SEGMENT_INFO=${QE_PRIMARY_ARRAY_SORTED_ON_CONTENT_ID[$I_INDEX]}
			fi

			((I_INDEX++))
			# some of the params passed to gpcreateseg.sh listed below:
			# FLAG: -p clusterConfigPostgresAddonsFile
			# PREFERRED_ROLE: IS_PRIMARY or IS_MIRROR
			# SEGMENT_INFO: ~ separated value, ex: host1~port1~data_dir~dbid1~contentid
			# PAIR_SEGMENT_INFO: corresponding pair, ex: host2~port2~data_dir~dbid2~contentid
			# INST_COUNT: Number of parallel script count
			COORDINATOR_HOSTNAME=${COORDINATOR_HOSTNAME} HBA_HOSTNAMES=${HBA_HOSTNAMES} TRUSTED_SHELL=${TRUSTED_SHELL} $GPCREATESEG $FLAG $$ $PREFERRED_ROLE $SEGMENT_INFO "${PAIR_SEGMENT_INFO}" $INST_COUNT  $LOG_FILE $HEAP_CHECKSUM \
`$ECHO ${COORDINATOR_IP_ADDRESS[@]}|$TR ' ' '~'` \
`$ECHO ${STANDBY_IP_ADDRESS[@]}|$TR ' ' '~'` &
			PARALLEL_COUNT $BATCH_LIMIT $BATCH_DEFAULT
		done
		PARALLEL_SUMMARY_STATUS_REPORT $PARALLEL_STATUS_FILE
		if [ $REPORT_FAIL -ne 0 ];then
			$CAT $PARALLEL_STATUS_FILE >> $LOG_FILE
		fi
		$RM -f $PARALLEL_STATUS_FILE

		LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

REGISTER_MIRRORS () {
		LOG_MSG "[INFO]:-Start Function $FUNCNAME"
		for I in "${QE_MIRROR_ARRAY_SORTED_ON_CONTENT_ID[@]}"
		do
			SET_VAR $I
			GP_HOSTNAME=$GP_HOSTADDRESS
			dbid=`env PGOPTIONS="-c gp_role=utility" $PSQL -p $COORDINATOR_PORT -d "${DEFAULTDB}" -X -A -t -c "select pg_catalog.gp_add_segment_mirror(${GP_CONTENT}::int2, '${GP_HOSTNAME}', '${GP_HOSTADDRESS}', ${GP_PORT}, '${GP_DIR}');" 2>>${LOG_FILE}` >> $LOG_FILE 2>&1
			ERROR_CHK $? "failed to register mirror for contentid=${GP_CONTENT}" 2
			MIRRORS_UPDATED_DBID=(${MIRRORS_UPDATED_DBID[@]} ${GP_HOSTNAME}~${GP_HOSTADDRESS}~${GP_PORT}~${GP_DIR}~${dbid}~${GP_CONTENT})
		done

		QE_MIRROR_ARRAY_SORTED_ON_CONTENT_ID=(${MIRRORS_UPDATED_DBID[@]})
		LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

STOP_QD_PRODUCTION () {
    LOG_MSG "[INFO]:-Start Function $FUNCNAME"
    LOG_MSG "[INFO]:-Restarting the Cloudberry instance in production mode" 1
    if [ -f $GPSTOP ]; then
        GPSTOP_OPTS=$(OUTPUT_LEVEL_OPTS)
        export COORDINATOR_DATA_DIRECTORY=${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1
        $GPSTOP -a -l $LOG_DIR -m -d $COORDINATOR_DATA_DIRECTORY $GPSTOP_OPTS
        RETVAL=$?
        if [ $RETVAL -eq 0 ]; then
            LOG_MSG "[INFO]:-Successfully shutdown the new Cloudberry instance"
        else
            ERROR_EXIT "[FATAL]:-Error from Cloudberry instance shutdown, check log files"
        fi
    else
        ERROR_EXIT "[FATAL]:-$GPSTOP not located"
    fi
    LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

CREATE_STANDBY_QD () {
	LOG_MSG "[INFO]:-Start Function $FUNCNAME"
	if [ ! -f $INIT_STANDBY_PROG ]; then
		LOG_MSG "[WARN]:-Unable to locate $INIT_STANDBY_PROG, hence unable to initialize standby coordinator $STANDBY_HOSTNAME"
		STANDBY_RET_CODE=1
	else
		LOG_MSG "[INFO]:-Starting initialization of standby coordinator $STANDBY_HOSTNAME" 1
		INIT_STANDBY_ARGS=""
		if [ x"" != x"$STANDBY_PORT" ]; then
			INIT_STANDBY_ARGS+=" -P $STANDBY_PORT"
		fi
		if [ x"" != x"$STANDBY_DATADIR" ]; then
			INIT_STANDBY_ARGS+=" -S $STANDBY_DATADIR"
		fi
		if [ $HBA_HOSTNAMES -eq 1 ]; then
		    INIT_STANDBY_ARGS+=" --hba-hostnames"
		fi
		if [ x"$CLUSTER_BOOT_MODE" = x"PRODUCTION" ]; then
		    export COORDINATOR_DATA_DIRECTORY=${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1;$INIT_STANDBY_PROG -s $STANDBY_HOSTNAME $INIT_STANDBY_ARGS -a -f $FTS_HOST_STRING_LIST
		else
		    export COORDINATOR_DATA_DIRECTORY=${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1;$INIT_STANDBY_PROG -s $STANDBY_HOSTNAME $INIT_STANDBY_ARGS -a
		fi
		STANDBY_RET_CODE=$?
		if [ $STANDBY_RET_CODE -eq 0 ]; then
			LOG_MSG "[INFO]:-Successfully completed standby coordinator initialization" 1
		else
			LOG_MSG "[WARN]:-Failed to complete standby coordinator initialization" 1
		fi
		BACKOUT_COMMAND "$TRUSTED_SHELL ${STANDBY_HOSTNAME} \"$RM -Rf ${QD_DIR}\""
		BACKOUT_COMMAND "$ECHO \"Removing standby directory ${QD_DIR} on $STANDBY_HOSTNAME\""
		BACKOUT_COMMAND "$TRUSTED_SHELL $STANDBY_HOSTNAME \"if [ -d $GP_DIR ]; then ${EXPORT_LIB_PATH};export PGPORT=${COORDINATOR_PORT}; $PG_CTL -w -D ${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1 -o \"-i -p ${COORDINATOR_PORT}\" -m immediate  stop; fi\""
		BACKOUT_COMMAND "$ECHO \"Stopping standby instance on $STANDBY_HOSTNAME\""
	fi 
	LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

START_QD_PRODUCTION () {
    LOG_MSG "[INFO]:-Start Function $FUNCNAME"
    if [ -f $GPSTART ]; then
	GPSTART_OPTS=$(OUTPUT_LEVEL_OPTS)
	export COORDINATOR_DATA_DIRECTORY=${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1
	$GPSTART -a -l $LOG_DIR -d $COORDINATOR_DATA_DIRECTORY $GPSTART_OPTS

        if [ $? -eq 0 ];then
            LOG_MSG "[INFO]:-Successfully started new Cloudberry instance"
        else

            # this text is duplicated below
            LOG_MSG "[WARN]:" 1
            LOG_MSG "[WARN]:-Failed to start Cloudberry instance; review gpstart output to" 1
            LOG_MSG "[WARN]:- determine why gpstart failed and reinitialize cluster after resolving" 1
            LOG_MSG "[WARN]:- issues.  Not all initialization tasks have completed so the cluster" 1
            LOG_MSG "[WARN]:- should not be used." 1

            LOG_MSG "[WARN]:-gpinitsystem will now try to stop the cluster" 1
            LOG_MSG "[WARN]:" 1
            if [ -f $GPSTOP ]; then
		GPSTOP_OPTS=$(OUTPUT_LEVEL_OPTS)
                export COORDINATOR_DATA_DIRECTORY=${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1
                $GPSTOP -a -l $LOG_DIR -i -d $COORDINATOR_DATA_DIRECTORY $GPSTOP_OPTS

                RETVAL=$?
                if [ $RETVAL -eq 0 ]; then
                    LOG_MSG "[INFO]:-Successfully shutdown the Cloudberry instance" 1
                else
                    ERROR_EXIT "[FATAL]:-Error from Cloudberry instance shutdown, check log files"
                fi
            else
                LOG_MSG "[WARN]:-$GPSTOP not located" 1
            fi

            # this text is duplicated above
            LOG_MSG "[WARN]:" 1
            LOG_MSG "[WARN]:-Failed to start Cloudberry instance; review gpstart output to" 1
            LOG_MSG "[WARN]:- determine why gpstart failed and reinitialize cluster after resolving" 1
            LOG_MSG "[WARN]:- issues.  Not all initialization tasks have completed so the cluster" 1
            LOG_MSG "[WARN]:- should not be used." 1
            LOG_MSG "[WARN]:" 1
            ERROR_EXIT "[FATAL]: starting new instance failed;"
        fi
    else
	ERROR_EXIT "[FATAL]:-$GPSTART not located"
    fi
    LOG_MSG "[INFO]:-Completed restart of Cloudberry instance in production mode" 1
    LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

CREATE_DATABASE () {
		LOG_MSG "[INFO]:-Start Function $FUNCNAME"
		SET_VAR $QD_PRIMARY_ARRAY
		$PSQL -p $GP_PORT -d "$DEFAULTDB" -c"create database \"${DATABASE_NAME}\";" >> $LOG_FILE 2>&1
		ERROR_CHK $? "create database $DATABASE_NAME" 2
		LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

# Creating core GPDB extension(s) in 'template1' and 'postgres' databases
CREATE_GPEXTENSIONS() {
		LOG_MSG "[INFO]:-Start Function $FUNCNAME"
		LOG_MSG "[INFO]:-Creating core Cloudberry extensions" 1
		declare -a extnames=("gp_toolkit")
		for ext in ${extnames[@]};
		do
			$PSQL -d template1 -X -A -t -c "CREATE EXTENSION $ext" >> $LOG_FILE 2>&1;
			$PSQL -d postgres -X -A -t -c "CREATE EXTENSION $ext" >> $LOG_FILE 2>&1;
		done
}


FORCE_FTS_PROBE () {
    LOG_MSG "[INFO]:-Start Function $FUNCNAME"
    SET_VAR $QD_PRIMARY_ARRAY
    # loop on gp_segment_configuration to make sure primary/mirror pairs are up and in sync
    for i in {1..60}; do
        if [ $i == 60 ]; then
            $PSQL -p $GP_PORT -d "$DEFAULTDB" -A -t -c "select * from gp_segment_configuration where (mode = 'n' or status = 'd') and content != -1;" >> $LOG_FILE 2>&1
            $PSQL -p $GP_PORT -d "$DEFAULTDB" -A -t -c "select * from gp_stat_replication where sync_error != 'none' or sync_state != 'sync';" >> $LOG_FILE 2>&1
            LOG_MSG "[WARN]:" 1
            LOG_MSG "[WARN]:-Failed to start Cloudberry instance; please review gpinitsystem log to determine failure." 1
			# FTS_TODO: should support notify fts, then re-enable this function 
            # ERROR_EXIT "[FATAL]:-Some primary/mirror segment pairs were found to be not in sync"
            break;
        fi

        RESULT=$( $PSQL -p $GP_PORT -d "$DEFAULTDB" -X -A -t -c "select count(*) > 0 from gp_segment_configuration where (mode = 'n' or status = 'd') and content != -1;" )
        if [ x"$RESULT" == x"f" ]; then
        	break
        fi
    done
    LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

SCAN_LOG () {
	LOG_MSG "[INFO]:-Start Function $FUNCNAME"
	LOG_MSG "[INFO]:-Scanning utility log file for any warning messages" 1
	SCAN_STRING="\[WARN\]|invalid|warning:|fatal|error"
	if [ `$GREP -E -i -w "$SCAN_STRING" $LOG_FILE|$GREP -v "\[INFO\]"|$GREP -v "[Start|End] Function ERROR_CHK"|$GREP -v "warning: enabling \"trust\" authentication for local connections"|$GREP -v "to the list of known hosts"|$GREP -v "LOG"|$WC -w` -ne 0 ];then
		LOG_MSG "[WARN]:-*******************************************************" 1
		LOG_MSG "[WARN]:-Scan of log file indicates that some warnings or errors" 1
		LOG_MSG "[WARN]:-were generated during the array creation" 1
		LOG_MSG "[INFO]:-Please review contents of log file" 1
		LOG_MSG "[INFO]:-$LOG_FILE" 1
		LOG_MSG "[INFO]:-To determine level of criticality" 1
		if [ `$GREP -ci "\[INFO]:-Start Main" $LOG_FILE` -gt 1 ];then
		LOG_MSG "[INFO]:-These messages could be from a previous run of the utility" 1
		LOG_MSG "[INFO]:-that was called today!" 1
		fi
		LOG_MSG "[WARN]:-*******************************************************" 1
	else
		LOG_MSG "[INFO]:-Log file scan check passed" 1
	fi
	LOG_MSG "[INFO]:-End Function $FUNCNAME"
}

DUMP_OUTPUT_CONFIG () {
    if [ x"" != x"$TRUSTED_SHELL" ] ; then
	$ECHO "TRUSTED_SHELL=$TRUSTED_SHELL" >> $OUTPUT_CONFIG
    fi
    if [ x"" != x"$ENCODING" ] ; then
	$ECHO "ENCODING=$ENCODING" >> $OUTPUT_CONFIG
    fi
    
    $ECHO "SEG_PREFIX=$SEG_PREFIX" >> $OUTPUT_CONFIG

    if [ x"" != x"$HEAP_CHECKSUM" ] ; then
	$ECHO "HEAP_CHECKSUM=$HEAP_CHECKSUM" >> $OUTPUT_CONFIG
    fi

    if [ x"" != x"$HBA_HOSTNAMES" ] ; then
	$ECHO "HBA_HOSTNAMES=$HBA_HOSTNAMES" >> $OUTPUT_CONFIG
    fi

    $ECHO "QD_PRIMARY_ARRAY=$QD_PRIMARY_ARRAY" >> $OUTPUT_CONFIG
    $ECHO "declare -a PRIMARY_ARRAY=(" >> $OUTPUT_CONFIG
    for qe in ${QE_PRIMARY_ARRAY[@]}
    do
	$ECHO "$qe" >> $OUTPUT_CONFIG
    done
    $ECHO ")" >> $OUTPUT_CONFIG
    if [ $MIRRORING -ne 0 ] ; then
	$ECHO "declare -a MIRROR_ARRAY=(" >> $OUTPUT_CONFIG
	for qe in ${QE_MIRROR_ARRAY[@]}
	do
	    $ECHO "$qe" >> $OUTPUT_CONFIG
	done    
	$ECHO ")" >> $OUTPUT_CONFIG
    fi
    

}

ASSIGN_COORDINATOR_VARS() {
	# Use COORDINATOR_* variables by default, fall back to MASTER_* variables if necessary
	if [ x"" = x"$COORDINATOR_HOSTNAME" ]; then
		if [ x"" = x"$MASTER_HOSTNAME" ]; then
			ERROR_EXIT "[FATAL]:COORDINATOR_HOSTNAME variable not set"
		else
			export COORDINATOR_HOSTNAME=$MASTER_HOSTNAME
		fi
	fi

	if [ x"" = x"$COORDINATOR_PORT" ]; then
		if [ x"" = x"$MASTER_PORT" ]; then
			ERROR_EXIT "[FATAL]:COORDINATOR_PORT variable not set"
		else
			export COORDINATOR_PORT=$MASTER_PORT
		fi
	fi

	if [ x"" = x"$COORDINATOR_DIRECTORY" ]; then
		if [ x"" = x"$MASTER_DIRECTORY" ]; then
			ERROR_EXIT "[FATAL]:COORDINATOR_DIRECTORY variable not set"
		else
			export COORDINATOR_DIRECTORY=$MASTER_DIRECTORY
		fi
	fi
	if [ x"$FTS_HOST_CONFIG" = x"" ] && [ x"$CLUSTER_BOOT_MODE" = x"PRODUCTION" ]; then
		FTS_HOST_MACHINE_LIST[0]=$COORDINATOR_HOSTNAME
	fi
	if [ x"$ETCD_HOST_CONFIG" = x"" ] && [ x"$CLUSTER_BOOT_MODE" = x"PRODUCTION" ]; then
		ETCD_HOST_MACHINE_LIST[0]=$COORDINATOR_HOSTNAME
	fi
}

READ_INPUT_CONFIG () {
   # Check that we have a non-zero configuration file
    CHK_FILE $INPUT_CONFIG
    if [ $EXISTS -ne 0 ]; then
        ERROR_EXIT "[FATAL]:-Problem with $INPUT_CONFIG file"
    fi
    
    # Make sure old CLUSTER_CONFIG settings are not hanging around.
    unset PORT_BASE SEG_PREFIX DATA_DIRECTORY

    # Validation
    if [ x"" != x"${COORDINATOR_PORT}" ] ; then
        ERROR_EXIT "[FATAL]:-Problem with configuration. Cannot specify COORDINATOR_PORT and QD_PRIMARY_ARRAY"
    fi

    if [ x"" != x"${COORDINATOR_HOSTNAME}" ] ; then
        ERROR_EXIT "[FATAL]:-Problem with configuration file. Cannot specify COORDINATOR_HOSTNAME and QD_PRIMARY_ARRAY"
    fi

    if [ x"" != x"${COORDINATOR_DIRECTORY}" ] ; then
        ERROR_EXIT "[FATAL]:-Problem with configuration file. Cannot specify COORDINATOR_DIRECTORY and QD_PRIMARY_ARRAY"
    fi

    # Make sure it is not a dos file with CTRL M at end of each line
    $TR -d '\r' < $INPUT_CONFIG > $TMP_FILE
    $MV $TMP_FILE $INPUT_CONFIG
    LOG_MSG "[INFO]:-Dumping $INPUT_CONFIG to logfile for reference"
    $CAT $INPUT_CONFIG|$GREP -v "^\s*\(#.*\)\?$" >> $LOG_FILE
    LOG_MSG "[INFO]:-Completed $INPUT_CONFIG dump to logfile"
    # Source the cluster configuration file
    LOG_MSG "[INFO]:-Reading Cloudberry configuration file $INPUT_CONFIG"

    . $INPUT_CONFIG
    SET_VAR $QD_PRIMARY_ARRAY
    COORDINATOR_HOSTNAME=$GP_HOSTADDRESS
    COORDINATOR_DIRECTORY=`$DIRNAME $GP_DIR`
    COORDINATOR_PORT=$GP_PORT
    SEG_PREFIX=`$BASENAME $GP_DIR -1`

    MACHINE_LIST=()
    DATA_DIRECTORY=()
    CONTENT_COUNT=0
    for QE in ${PRIMARY_ARRAY[@]}
    do
        SET_VAR $QE

        DATA_DIRECTORY=(${DATA_DIRECTORY[@]} `$DIRNAME $GP_DIR`)
        MACHINE_LIST=(${MACHINE_LIST[@]} $GP_HOSTADDRESS)

        ((CONTENT_COUNT=$CONTENT_COUNT+1))
    done

    DATA_DIRECTORY=(`$ECHO ${DATA_DIRECTORY[@]} | $TR ' ' '\n' | $SORT | $TR '\n' ' '`)
    
    if [ x"" != x"${MIRROR_ARRAY}" ] ; then
        MIRRORING=1
        MIRROR_DATA_DIRECTORY=()
        for QE in ${MIRROR_ARRAY[@]}
        do
            SET_VAR $QE

            MIRROR_DATA_DIRECTORY=(${MIRROR_DATA_DIRECTORY[@]} `$DIRNAME $GP_DIR`)
            MACHINE_LIST=(${MACHINE_LIST[@]} $GP_HOSTADDRESS)
        done

        MIRROR_DATA_DIRECTORY=(`$ECHO ${MIRROR_DATA_DIRECTORY[@]} | $TR ' ' '\n' | $SORT | $TR '\n' ' '`)
    fi

    MACHINE_LIST=(`$ECHO ${MACHINE_LIST[@]} | $TR ' ' '\n' | $SORT -u | $TR '\n' ' '`)
    if [ `$ECHO ${MACHINE_LIST[@]}| $TR ' ' '\n' | $GREP -c "${COORDINATOR_HOSTNAME}\$"` -gt 0 ]; then
        MACHINE_LIST=($COORDINATOR_HOSTNAME `$ECHO ${MACHINE_LIST[@]} | $TR ' ' '\n' | $GREP -v "${COORDINATOR_HOSTNAME}\$" | $TR '\n' ' '`)
    fi

    QE_PRIMARY_ARRAY=(${PRIMARY_ARRAY[@]})
    QE_MIRROR_ARRAY=(${MIRROR_ARRAY[@]})
    ((TOTAL_SEG=${#QE_PRIMARY_ARRAY[@]}))
    ((TOTAL_MIRRORS=${#MIRROR_ARRAY[@]}))
    MIRROR_TYPE=0
    MULTI_HOME=0

    if [ $TOTAL_MIRRORS -ne 0 ] ; then
        if [ $TOTAL_SEG -ne $TOTAL_MIRRORS ] ; then
            ERROR_EXIT "[FATAL]:-Problem with configuration file. Cannot specify different number of primary and mirror segments."
        fi
    fi
}


CHK_QE_ARRAY_PORT_RANGES () {

    #
    # calculate port ranges
    #

    MIN_PORT=1000000
    MAX_PORT=0
    for QE in ${QE_PRIMARY_ARRAY[@]}
    do
        SET_VAR $QE
        if [ $GP_PORT -lt $MIN_PORT ] ; then
            MIN_PORT=$GP_PORT
        fi
        if [ $GP_PORT -gt $MAX_PORT ] ; then
            MAX_PORT=$GP_PORT
        fi
    done

    PORT_BASE=$MIN_PORT

    if [ x"" != x"${QE_MIRROR_ARRAY}" ] ; then
        MIN_MIRROR_PORT=100000000
        MAX_MIRROR_PORT=0
        for QE in ${QE_MIRROR_ARRAY[@]}
        do
            SET_VAR $QE
            if [ $GP_PORT -lt $MIN_MIRROR_PORT ] ; then
                MIN_MIRROR_PORT=$GP_PORT
            fi
            if [ $GP_PORT -gt $MAX_MIRROR_PORT ] ; then
                MAX_MIRROR_PORT=$GP_PORT
            fi
        done
    fi

    MIRROR_PORT_BASE=$MIN_MIRROR_PORT

    #
    # now look for range conflicts
    #

    if CHK_OVERLAP $COORDINATOR_PORT $COORDINATOR_PORT $MIN_PORT $MAX_PORT; then
	ERROR_EXIT "[FATAL]:-COORDINATOR_PORT overlaps with PORT_BASE."
    fi
    
    # Mirror configuration
    if [ $MIRROR_PORT_BASE ]; then
	
	if CHK_OVERLAP $COORDINATOR_PORT $COORDINATOR_PORT $MIN_MIRROR_PORT $MAX_MIRROR_PORT; then
	    ERROR_EXIT "[FATAL]:-COORDINATOR_PORT overlaps with MIRROR_PORT_BASE."
	fi
	
	if CHK_OVERLAP $MIN_PORT $MAX_PORT $MIN_MIRROR_PORT $MAX_MIRROR_PORT; then
	    ERROR_EXIT "[FATAL]:-PORT_BASE AND MIRROR_PORT_BASE define overlapping ranges."
	fi
    fi

    # PORT_BASE
    if [ x"" = x"$PORT_BASE" ]; then
	    ERROR_EXIT "[FATAL]:-PORT_BASE variable not set"
    fi
    QE_PORT=$PORT_BASE
}

SET_DCA_CONFIG_SETTINGS () {
	LOG_MSG "[INFO]:-Setting DCA specific configuration values..." 1
	local coordinator_data_directory="${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1"

	LOG_MSG "[INFO]:-COORDINATOR_DATA_DIRECTORY=$coordinator_data_directory $GPCONFIG -c $DCA_RESQUEUE_PRIORITY_NAME -v $DCA_SEGMENT_RESQUEUE_PRIORITY_VAL -m $DCA_COORDINATOR_RESQUEUE_PRIORITY_VAL" 1
	COORDINATOR_DATA_DIRECTORY=$coordinator_data_directory $GPCONFIG -c $DCA_RESQUEUE_PRIORITY_NAME -v $DCA_SEGMENT_RESQUEUE_PRIORITY_VAL -m $DCA_COORDINATOR_RESQUEUE_PRIORITY_VAL
	RETVAL=$?
	if [ $RETVAL -ne 0 ]; then
		LOG_MSG "[WARN]:-Failed to set value for $DCA_RESQUEUE_PRIORITY_NAME" 1
	fi
	
	LOG_MSG "[INFO]:-COORDINATOR_DATA_DIRECTORY=$coordinator_data_directory $GPCONFIG -c $DCA_RESQUEUE_PRIORITY_CPUCORES_PER_SEGMENT_NAME -v $DCA_SEGMENT_RESQUEUE_PRIORITY_CPUCORES_PER_SEGMENT_VAL -m $DCA_COORDINATOR_RESQUEUE_PRIORITY_CPUCORES_PER_SEGMENT_VAL" 1
	COORDINATOR_DATA_DIRECTORY=$coordinator_data_directory $GPCONFIG -c $DCA_RESQUEUE_PRIORITY_CPUCORES_PER_SEGMENT_NAME -v $DCA_SEGMENT_RESQUEUE_PRIORITY_CPUCORES_PER_SEGMENT_VAL -m $DCA_COORDINATOR_RESQUEUE_PRIORITY_CPUCORES_PER_SEGMENT_VAL
	RETVAL=$?
	if [ $RETVAL -ne 0 ]; then
		LOG_MSG "[WARN]:-Failed to set value for $DCA_RESQUEUE_PRIORITY_CPUCORES_PER_SEGMENT_NAME" 1
	fi

	LOG_MSG "[INFO]:-COORDINATOR_DATA_DIRECTORY=$coordinator_data_directory $GPCONFIG -c $DCA_RESQUEUE_PRIORITY_SWEEPER_INTERVAL_NAME -v $DCA_SEGMENT_RESQUEUE_PRIORITY_SWEEPER_INTERVAL_VAL -m $DCA_COORDINATOR_RESQUEUE_PRIORITY_SWEEPER_INTERVAL_VAL" 1
	COORDINATOR_DATA_DIRECTORY=$coordinator_data_directory $GPCONFIG -c $DCA_RESQUEUE_PRIORITY_SWEEPER_INTERVAL_NAME -v $DCA_SEGMENT_RESQUEUE_PRIORITY_SWEEPER_INTERVAL_VAL -m $DCA_COORDINATOR_RESQUEUE_PRIORITY_SWEEPER_INTERVAL_VAL
	RETVAL=$?
	if [ $RETVAL -ne 0 ]; then
		LOG_MSG "[WARN]:-Failed to set value for $DCA_RESQUEUE_PRIORITY_SWEEPER_INTERVAL_NAME" 1
	fi
}

SET_MIRROR_MODE() {
	local mode=$1
	if [ "$mode" == 'group' ]; then
		MIRROR_TYPE=0
	elif [ "$mode" == 'spread' ]; then
		MIRROR_TYPE=1
	else
		ERROR_EXIT "[FATAL]:-'$mode' is an invalid mirroring mode.  Valid options are 'group' and 'spread'."
	fi
}

GENERATE_ETCD_URL() {
	if [ "$#" -ne 1 ]; then
		LOG_MSG "[ERROR] GENERATE_ETCD_URL invalid params..."
	fi
	ETCD_PORT=$1
	ETCD_CMD=""
	local etcd_num=0
	for etcd in ${ETCD_HOST_MACHINE_LIST[*]}
	do
		local etcd_ip=`RESOLVE_HOSTNAME ${etcd}`
		if [ -z "$ETCD_CMD" ]; then
			ETCD_CMD="etcd-$etcd_num=http://$etcd_ip:$ETCD_PORT"
		else
			ETCD_CMD="$ETCD_CMD,etcd-$etcd_num=http://$etcd_ip:$ETCD_PORT"
		fi
		etcd_num=$(( $etcd_num + 1 ))
	done
	echo "$ETCD_CMD"
}

GENERATE_ETCD_CLIENT_URL() {
	if [ "$#" -ne 1 ];then
		LOG_MSG "[ERROR] GENERATE_ETCD_URL invalid params..."
	fi
	ETCD_PORT=$1
	ETCD_CMD=""
	local etcd_num=0
	for etcd in ${ETCD_HOST_MACHINE_LIST[*]}
	do
		local etcd_ip=`RESOLVE_HOSTNAME ${etcd}`
		if [ -z "$ETCD_CMD" ]; then
			ETCD_CMD="http://$etcd_ip:$ETCD_PORT"
		else
			ETCD_CMD="$ETCD_CMD,http://$etcd_ip:$ETCD_PORT"
		fi
		etcd_num=$(( $etcd_num + 1 ))
	done
	echo "$ETCD_CMD"
}

GENERATE_ETCD_HOST_CONFIG() {
	LOG_MSG "[INFO]: GENERATE ETCD HOST CONFIG in $ETCD_HOST_FILE"
	for etcd in ${ETCD_HOST_MACHINE_LIST[*]}
	do
		OUTPUT_HOST_TO_FILE $ETCD_HOST_FILE $etcd
	done
}

GENERATE_FTS_HOST_CONFIG() {
	LOG_MSG "[INFO]: GENERATE FTS HOST CONFIG in $FTS_HOST_FILE"
	for fts in ${FTS_HOST_MACHINE_LIST[*]}
	do
		OUTPUT_HOST_TO_FILE $FTS_HOST_FILE $fts
	done
}

CBDB_PRECHECK() {
        if [ x"$CLUSTER_BOOT_MODE" = x"DEMO" ];then
            FTS_HOST_MACHINE_LIST[0]=$DEMO_CLUSTER_HOST
            ETCD_HOST_MACHINE_LIST[0]=$DEMO_CLUSTER_HOST
        fi
        for fts in ${FTS_HOST_MACHINE_LIST[*]}
        do
                CHECK_FTS $fts
                if [ $? -eq 0 ];then
                        ERROR_EXIT "[FATAL]: FTS HOST $fts has alived fts process, please cleanup the environment first."
                fi
                if [ -z FTS_HOST_STRING_LIST ];then
                         FTS_HOST_STRING_LIST="$fts"
                else
                         FTS_HOST_STRING_LIST="$FTS_HOST_STRING_LIST,$fts"
                fi
        done

        for etcd in ${ETCD_HOST_MACHINE_LIST[*]}
        do
                CHECK_ETCD $etcd $CLUSTER_BOOT_MODE
                if [ $? -eq 0 ];then
                        LOG_MSG "[INFO]: ETCD HOST $etcd has alived etcd process."
                        ETCD_EXIST="true";
                fi
        done
        PRECHECK_CBDB_CONFIG_FILE
}

CREATE_ETCD() {
        if [[ `uname -i` = aarch64 ]]; then
            export ETCD_UNSUPPORTED_ARCH=arm64
        fi
        if [ x"$CLUSTER_BOOT_MODE" = x"PRODUCTION" ];then
            ETCD_CONFIG_FILE_PATH=`readlink -f $PG_CONF_ADD_FILE`
            if [ ! -x ${GPHOME}/bin/etcd ];then
                ERROR_EXIT "[FATAL]: No mgmt executables(etcd) found for Cloudberry installation in ${GPHOME}/bin"
            fi
        fi
        local etcd_url=`GENERATE_ETCD_URL 2380`
        local etcd_num=0
        if [ "$ETCD_EXIST" = "false" ];then
            for etcd in ${ETCD_HOST_MACHINE_LIST[*]}
            do
                local etcd_host_num=etcd-${etcd_num}
                SETUP_ETCD $etcd $etcd_url $etcd_host_num $CLUSTER_BOOT_MODE
                etcd_num=$(( $etcd_num + 1 ))
            done
        else
            LOG_MSG "[WARN]: The ETCD cluster already exists and skip installing ETCD."
        fi
        sleep 5
        for etcd in ${ETCD_HOST_MACHINE_LIST[*]}
        do
               CHECK_ETCD $etcd $CLUSTER_BOOT_MODE
               if [ $? -eq 1 ];then
                       ERROR_EXIT "[FATAL]: ETCD HOST $etcd error check process alive failed!"
               fi
        done
        etcd_num_res=$(( $etcd_num + 1 ))
        local etcd_client_url=`GENERATE_ETCD_CLIENT_URL 2379`
        local health_check_res=`export ETCDCTL_API=3;etcdctl --endpoints $etcd_client_url endpoint health | grep 'healthy'`
        if [ !"$health_check_res" = "$etcd_num_res" ]; then
                ERROR_EXIT "[FATAL]: ETCD HOST health check failed with result: $health_check_res"
        fi
        CLEANUP_ETCD $ETCD_CONFIG_FILE_PATH
}

CREATE_FTS() {
        if [ ! -x ${GPHOME}/bin/gpfts ]; then
            ERROR_EXIT "[FATAL]: No mgmt executables(gpfts) found for Cloudberry installation in ${GPHOME}/bin"
        fi
        for fts in ${FTS_HOST_MACHINE_LIST[*]}
        do
                local log_dir=${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1
                SETUP_FTS $fts $log_dir $CLUSTER_BOOT_MODE
        done
        sleep 5
        for fts in ${FTS_HOST_MACHINE_LIST[*]}
        do
                CHECK_FTS $fts
                if [ $? -eq 1 ];then
                        ERROR_EXIT "[FATAL]: FTS HOST $etcd failed to start!"
                fi
        done

        if [ "$FTS_START_MODE" = "DEMO" ];then
            local health_check_res=`$GPHOME/bin/gpfts -F $ETCD_CONFIG_FILE_PATH -d ${FTS_LOG_DIR}/log/fts -W 2 | wc -l`
        else
            FTS_CMD="mkdir -p ${FTS_LOG_DIR}/log/fts;${GPHOME}/bin/gpfts -F ${ETCD_CONFIG_TMP_FILE} -d ${FTS_LOG_DIR}/log/fts -W 2"
            local health_check_res=`gpssh -h $fts -e "${FTS_CMD}" | wc -l`
        fi
        if [ "$health_check_res" = "1" ]; then
                ERROR_EXIT "[FATAL]: FTS node health check failed!"
        fi
}

#******************************************************************************
# Main Section
#******************************************************************************
trap 'ERROR_EXIT "[FATAL]:-Received INT or TERM signal"' INT TERM
while getopts ":vaqe:c:l:-:p:m:h:n:s:P:S:b:DB:I:O:E:F:M:U:T:C" opt
		do
		case $opt in
				v ) print_version ;;
				a ) unset INTERACTIVE ;;
				q ) unset VERBOSE ;;
				c ) CLUSTER_CONFIG=$OPTARG ;;
				l ) LOG_DIR=$OPTARG ;;
				e ) GP_PASSWD=$OPTARG ;;
				p ) PG_CONF_ADD_FILE=$OPTARG ;;
				h ) MACHINE_LIST_FILE_ALT=$OPTARG ;;
				m ) COORDINATOR_MAX_CONNECT=$OPTARG ;;
				b ) NEW_BUFFERS=$OPTARG ;;
				s ) STANDBY_HOSTNAME=$OPTARG ;;
				P ) STANDBY_PORT=$OPTARG ;;
				S ) STANDBY_DATADIR=$OPTARG ;;
				n ) REQ_LOCALE_SETTING=$OPTARG ;;
				D ) DEBUG_LEVEL=1 ;;
				B ) BATCH_DEFAULT=$OPTARG ;;
				I ) INPUT_CONFIG=$OPTARG ;;
				O ) OUTPUT_CONFIG=$OPTARG ;;
				U ) USE_EXTERNAL_FTS=$OPTARG ;;
				E ) ETCD_HOST_CONFIG=$OPTARG ;;
				F ) FTS_HOST_CONFIG=$OPTARG ;;
				T ) ENCRYPT_METHOD=$OPTARG ;;
				C ) CLUSTER_KEY_CMD=$OPTARG ;;
				- ) # Long options ...
					NAME=${OPTARG%%=*}
					VAL=${OPTARG#*=}
					case $NAME in
						"singlenodeMode"  ) SINGLENODE_MODE=$VAL ;;
						"max_connections" ) COORDINATOR_MAX_CONNECT=$VAL ;;
						"shared_buffers"  ) NEW_BUFFERS=$VAL ;;
						"su_password"     ) GP_PASSWD=$VAL ;;
						"locale"          ) REQ_LOCALE_SETTING=$VAL ;;
						"lc-collate"      ) LCCOLLATE=$VAL ;;
						"lc-ctype"        ) LCCTYPE=$VAL ;;
						"lc-messages"     ) LCMESSAGES=$VAL ;;
						"lc-monetary"     ) LCMONETARY=$VAL ;;
						"lc-numeric"      ) LCNUMERIC=$VAL ;;
						"lc-time"         ) LCTIME=$VAL ;;
						"mirror-mode"     ) SET_MIRROR_MODE $VAL ;;
						"standby-datadir" ) STANDBY_DATADIR=$VAL ;;
						"encrypt-method"  ) ENCRYPT_METHOD=$VAL ;;
						"cluster-key-cmd" ) CLUSTER_KEY_CMD=$VAL ;;
						"help"            ) USAGE "print_doc" ;;
						"version"         ) print_version ;;
						* ) LOG_MSG "[ERROR]:-Unknown option --$NAME" 1; USAGE ;;
					esac

					# Check if we are missing the long option value.  Currently all
					# long options require a value so we can just do the check once.
					if [ x"$NAME" == x"$VAL" ]; then
						LOG_MSG "[FATAL]:-Missing value for option --$NAME" 1;
						USAGE;
					fi
					;;
				* ) USAGE ;;
		esac
done

if [ x"" != x"$PG_CONF_ADD_FILE" ]; then
	CHK_FILE $PG_CONF_ADD_FILE
	if [ $EXISTS -ne 0 ]; then
		ERROR_EXIT "[FATAL]:-Issue with postgres additions file $PG_CONF_ADD_FILE"
	fi
fi
if [ x"" != x"$LOG_DIR" ];then
        CHK_DIR $LOG_DIR "" 1
        if [ $EXISTS -eq 1 ]; then
                ERROR_EXIT "[FATAL]:-Log directory $LOG_DIR does not exist!"
        else
                LOG_FILE=$LOG_DIR/${PROG_NAME}_${CUR_DATE}.log
                BACKOUT_FILE=$LOG_DIR/backout_${PROG_NAME}_${USER_NAME}_${CUR_DATE}_$FILE_TIME
        fi
else
        BACKOUT_FILE=$DEFLOGDIR/backout_${PROG_NAME}_${USER_NAME}_${CUR_DATE}_$FILE_TIME
        # set $LOG_DIR to $DEFLOGDIR, so $LOG_DIR can be used everywhere
        LOG_DIR=$DEFLOGDIR
fi

LOG_MSG "[INFO]:-Start Main"
LOG_MSG "[INFO]:-Command line options passed to utility = $*"
if [ x"$INPUT_CONFIG" != x"" ] ; then
    CHK_GPDB_ID
    CHK_PARAMS
    CHK_QE_ARRAY_PORT_RANGES
    CHK_QES_FROM_INPUTFILE

    if [ x"" != x"$OUTPUT_CONFIG" ] ; then
        DUMP_OUTPUT_CONFIG
        exit 0
    fi

    DISPLAY_CONFIG

else
    CHK_GPDB_ID
    CHK_PARAMS
    CHK_MULTI_HOME
    CREATE_QE_ARRAY
    CHK_QE_ARRAY_PORT_RANGES

    if [ "$SINGLENODE_MODE" == "false" ] ; then
        CHK_QES
    fi

    if [ x"" != x"$OUTPUT_CONFIG" ] ; then
        DUMP_OUTPUT_CONFIG
        exit 0
    fi

    DISPLAY_CONFIG
	if [ "$SINGLENODE_MODE" == "false" ] ; then
		ARRAY_REORDER
	fi
fi

if [ $USE_EXTERNAL_FTS -ne 0 ];then
	CBDB_PRECHECK

	LOG_MSG "[INFO]:-------------------------------------------------------" 1
	LOG_MSG "[INFO]:-Start to install ETCD cluster" 1
	CREATE_ETCD
	LOG_MSG "[INFO]:-ETCD cluster successfully created" 1
	LOG_MSG "[INFO]:-------------------------------------------------------" 1
fi

CREATE_QD_DB

CREATE_ARRAY_SORTED_ON_CONTENT_ID

if [ "$SINGLENODE_MODE" == "false" ]; then
	CREATE_SEGMENT IS_PRIMARY
	if [ $REPORT_FAIL -ne 0 ];then
		LOG_MSG "[FATAL]:-Errors generated from parallel processes" 1
		LOG_MSG "[INFO]:-Dumped contents of status file to the log file" 1
		ERROR_EXIT "[FATAL]:-Failures detected, see log file $LOG_FILE for more detail"
	else
		LOG_MSG "[INFO]:-Removing back out file" 1
		$RM -f $BACKOUT_FILE
		LOG_MSG "[INFO]:-No errors generated from parallel processes" 1
	fi

	if [ -f $DCA_VERSION_FILE ]; then
		SET_DCA_CONFIG_SETTINGS
	fi
fi

if [ $USE_EXTERNAL_FTS -ne 0 ];then
	GENERATE_FTS_HOST_CONFIG
	GENERATE_ETCD_HOST_CONFIG
	if [ x"$CLUSTER_BOOT_MODE" = x"DEMO" ];then
		ETCD_CONFIG_PATH=`readlink -f $GPHOME/bin/config/cbdb_etcd_default.conf`
	else
		ETCD_CONFIG_PATH=`readlink -f $PG_CONF_ADD_FILE`
	fi
	LOG_MSG "[INFO]: Copy etcd config file from $ETCD_CONFIG_PATH to ${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1/config/cbdb_etcd.conf"
	cp  $ETCD_CONFIG_PATH ${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1/config/cbdb_etcd.conf
fi

STOP_QD_PRODUCTION
START_QD_PRODUCTION

CREATE_GPEXTENSIONS

if [ x"" != x"$DATABASE_NAME" ]; then
	CREATE_DATABASE
fi

SET_GP_USER_PW

if [ $MIRRORING -ne 0 ]; then
    REGISTER_MIRRORS
    CREATE_SEGMENT IS_MIRROR
    FORCE_FTS_PROBE
fi

if [ x"" != x"$STANDBY_HOSTNAME" ];then
	CREATE_STANDBY_QD
fi

SCAN_LOG
LOG_MSG "[INFO]:-Apache Cloudberry instance successfully created" 1
LOG_MSG "[INFO]:-------------------------------------------------------" 1
LOG_MSG "[INFO]:-To complete the environment configuration, please " 1
LOG_MSG "[INFO]:-update $USER_NAME .bashrc file with the following" 1
LOG_MSG "[INFO]:-1. Ensure that the greenplum_path.sh file is sourced" 1
LOG_MSG "[INFO]:-2. Add \"export COORDINATOR_DATA_DIRECTORY=${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1\"" 1
LOG_MSG "[INFO]:-   to access the Cloudberry scripts for this instance:" 1
LOG_MSG "[INFO]:-   or, use -d ${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1 option for the Cloudberry scripts" 1
LOG_MSG "[INFO]:-   Example gpstate -d ${COORDINATOR_DIRECTORY}/${SEG_PREFIX}-1" 1
LOG_MSG "[INFO]:-Script log file = $LOG_FILE" 1
LOG_MSG "[INFO]:-To remove instance, run gpdeletesystem utility" 1

if [ x"" != x"$STANDBY_HOSTNAME" ];then
	case $STANDBY_RET_CODE in
		0|1)
			LOG_MSG "[INFO]:-Standby Coordinator $STANDBY_HOSTNAME has been configured" 1
			LOG_MSG "[INFO]:-To activate the Standby Coordinator Segment in the event of Coordinator" 1
			LOG_MSG "[INFO]:-failure review options for gpactivatestandby" 1 ;;
		*) LOG_MSG "[WARN]:-Standby Coordinator failed to initialize" 1
	esac
else
    LOG_MSG "[INFO]:-To initialize a Standby Coordinator Segment for this Cloudberry instance" 1
    LOG_MSG "[INFO]:-Review options for gpinitstandby" 1
fi

LOG_MSG "[INFO]:-------------------------------------------------------" 1
LOG_MSG "[INFO]:-The Coordinator ${COORDINATOR_DATA_DIRECTORY}/$PG_HBA post gpinitsystem" 1
LOG_MSG "[INFO]:-has been configured to allow all hosts within this new" 1
LOG_MSG "[INFO]:-array to intercommunicate. Any hosts external to this" 1
LOG_MSG "[INFO]:-new array must be explicitly added to this file" 1
LOG_MSG "[INFO]:-------------------------------------------------------" 1

# Make sure that the user sees that there's an error with the standby
if [ $STANDBY_RET_CODE -ne 0 ] && [ $STANDBY_RET_CODE -ne 1 ]; then
	LOG_MSG "[WARN]:-*******************************************************" 1
	LOG_MSG "[WARN]:-Cluster setup finished, but Standby Coordinator failed to initialize. Review contents of log files for errors." 1
	LOG_MSG "[INFO]:-Use gpinitstandby to create a Standby Coordinator" 1
	LOG_MSG "[WARN]:-*******************************************************" 1
fi

LOG_MSG "[INFO]:-End Main"
exit $EXIT_STATUS
