#!/bin/sh
#
# Copyright (C) 1994-2018 Altair Engineering, Inc.
# For more information, contact Altair at www.altair.com.
#
# This file is part of the PBS Professional ("PBS Pro") software.
#
# Open Source License Information:
#
# PBS Pro is free software. You can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# PBS Pro is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Commercial License Information:
#
# For a copy of the commercial license terms and conditions,
# go to: (http://www.pbspro.com/UserArea/agreement.html)
# or contact the Altair Legal Department.
#
# Altair’s dual-license business model allows companies, individuals, and
# organizations to create proprietary derivative works of PBS Pro and
# distribute them - whether embedded or bundled with other software -
# under a commercial license agreement.
#
# Use of Altair’s trademarks, including but not limited to "PBS™",
# "PBS Professional®", and "PBS Pro™" and Altair’s logos is subject to Altair's
# trademark licensing policies.

if [ $# -eq 1 -a "$1" = "--version" ]; then
   echo pbs_version = 19.1.3
   exit 0
fi

PBS_VERSION=19.1.3
PBS_AES_SWITCH_VER='14.0'

get_db_user() {
	[ -f "${dbuser_fl}" ] && dbuser_name=`cat "${dbuser_fl}" | tr -d '[:space:]'`
	[ -z "${dbuser_name}" ] && dbuser_name="${PBS_DATA_SERVICE_USER:-postgres}"
	if [ ! -f "${dbuser_fl}" ]; then
		printf "%s" "$dbuser_name" >"${dbuser_fl}"
		chmod 0600 "${dbuser_fl}"
	fi
	cat "${dbuser_fl}"
	return $?
}

chk_dataservice_user() {

	chk_usr="$1"

	id=`id ${chk_usr} 2>&1`
	if [ $? -ne 0 ]; then
		echo "PBS Dataservice user ${chk_usr} does not exist"
		return 1;
	fi

	id=`echo ${id} | cut -c5- | cut -d "(" -f1`
	if [ "$id" = "0" ]; then
		echo "PBS Data Service user should not have root priviledges"
		return 1;
	fi

	# login as ${chk_usr} and try to cd to user home dir
	su - ${chk_usr} -c "cd" > /dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Unable to login as user ${chk_usr}. Is the user enabled/home directory accessible?"
		return 1
	fi
	return 0
}

is_cray_xt() {
	if [ -f /proc/cray_xt/cname ] ; then
		return 0
	fi
	return 1
}

set_db_trust_login() {
	datastore_dir=$1
	err=`cp -p ${datastore_dir}/pg_hba.conf ${datastore_dir}/pg_hba.conf.orig 2>&1`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`chown ${PBS_DATA_SERVICE_USER} ${datastore_dir}/pg_hba.conf.orig`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`sed 's/md5/trust/g' ${datastore_dir}/pg_hba.conf > ${datastore_dir}/pg_hba.conf.new 2>&1`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`chown ${PBS_DATA_SERVICE_USER} ${datastore_dir}/pg_hba.conf.new`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`mv ${datastore_dir}/pg_hba.conf.new ${datastore_dir}/pg_hba.conf 2>&1`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
}

revoke_db_trust_login() {
	datastore_dir=$1
	err=`cp -p ${datastore_dir}/pg_hba.conf.orig ${datastore_dir}/pg_hba.conf 2>&1`
	if [ $? -eq 0 ]; then
		rm -f ${datastore_dir}/pg_hba.conf.orig
	else
		echo "${err}"
		return 1
	fi
}

utf8_to_sql_ascii() {
	#
	# If the existing database has UTF-8 character encoding,
	# change the encoding to SQL_ASCII for database upgrade.
	#

	${PBS_EXEC}/sbin/pbs_dataservice status > /dev/null
	if [ $? -eq 0 ]; then
		# stop the dataservice
		${PBS_EXEC}/sbin/pbs_dataservice stop > /dev/null
		if [ $? -ne 0 ]; then
			echo "Error stopping PBS Data Service"
			kill -s SIGTERM `ps -ef | grep "${inst_dir}/bin/postgres" | grep -v grep | awk '{if ($3 == 1) print $2}'`
			return 1
		fi
	fi

	set_db_trust_login "${PBS_HOME}/datastore"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	# restart with new credentials
	# redirect the output,
	# Use old pgsql binaries since we
	# are working with old database.
	#

	PGSQL_INST_DIR="${PBS_HOME}/pgsql.old"
	export PGSQL_INST_DIR

	${PBS_EXEC}/sbin/pbs_dataservice start > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error starting PBS Data Service"
		revoke_db_trust_login "${PBS_HOME}/datastore"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi
		return 1
	fi

	enc_type=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "select pg_encoding_to_char(encoding) from pg_database where datname = 'pbs_datastore'"`
	if [ ! -z "${enc_type}" -a "${enc_type}" = "UTF8" ]; then
		db_list=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "select datname from pg_database"`
		if [ $? -eq 0 ]; then
			for db_entry in ${db_list}
			do
				update=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "update pg_database set encoding = pg_char_to_encoding('SQL_ASCII') where datname = '$db_entry'"`
				ret=$?
				if [ $ret -ne 0 ]; then
					return $ret
				fi
			done
		fi
	fi

	locale=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "SHOW LC_COLLATE"`
	if [ ! -z ${locale} -a ${locale} != "C" ]; then
		change_locale=1
	fi

	# stop the dataservice
	${PBS_EXEC}/sbin/pbs_dataservice stop > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error stopping PBS Data Service"
		kill -s SIGTERM `ps -ef | grep "${PGSQL_INST_DIR}/bin/postgres" | grep -v grep | awk '{if ($3 == 1) print $2}'`
		return 1
	fi

	unset PGSQL_INST_DIR

	revoke_db_trust_login "${PBS_HOME}/datastore"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	return 0
}

upgrade_pbs_database() {

	user="${PBS_DATA_SERVICE_USER}"
	inst_dir="${PGSQL_DIR}"
	data_dir="${PBS_HOME}/datastore"
	old_inst_dir="${PBS_HOME}/pgsql.old"
	old_data_dir="${PBS_HOME}/datastore.old"

	if [ ! -f "${data_dir}/PG_VERSION" ]; then
		echo "Database version file: ${data_dir}/PG_VERSION not found, cannot continue"
		return 1
	fi
	
	sys_pgsql_ver=$(echo `${PGSQL_BIN}/postgres -V` | awk 'NR==1 {print $NF}' | cut -d '.' -f 1,2)
	old_pgsql_ver=`cat ${data_dir}/PG_VERSION`
	# strip the minor version from sys_pgsql_ver if old_pgsql_ver does not have minor version (for comparison).
	[[ ! $old_pgsql_ver =~ "." ]] && sys_pgsql_ver=$(echo $sys_pgsql_ver | cut -d '.' -f 1)

	[ ${sys_pgsql_ver%.*} -eq ${old_pgsql_ver%.*} ] && [ ${sys_pgsql_ver#*.} \> ${old_pgsql_ver#*.} ] || [ ${sys_pgsql_ver%.*} -gt ${old_pgsql_ver%.*} ];
	result=$?
	if [ ${result} -eq 0 ]; then
		if [ -d "$PBS_EXEC/pgsql" ]; then
			#Start upgrade process of datastore
			upgrade_db
			return $?
		else
			return 2
		fi
	elif [ "${old_pgsql_ver}" = "${sys_pgsql_ver}" ]; then
		return 0
	fi

	echo "Upgrade from version ${old_pgsql_ver} unsupported"
	return 1
}

upgrade_db() {
	#
	# This routine will insatll a 9.3 database cluster,
	# will perform the pre-upgrade checks for datastore
	# with appropriate authentication management.
	#

	change_locale=0
	server_ctl="${PBS_EXEC}/sbin/pbs_dataservice"
	if [ ! -x "${server_ctl}" ]; then
		echo "${server_ctl} not found"
		return 1
	fi

	if [ ! -x "${PBS_EXEC}/sbin/pbs_ds_password" ]; then
		echo "${PBS_EXEC}/sbin/pbs_ds_password not found"
		return 1
	fi

	if [ ! -x "${inst_dir}/bin/pg_upgrade" ]; then
		echo "${inst_dir}/bin/pg_upgrade not found"
		return 1
	fi

	# Check for existence of old data service directory.
	if [ ! -d "${old_inst_dir}" ]; then
		echo "Data service directory from previous PBS installation not found,"
		echo "Datastore upgrade cannot continue"
		return 1
	fi

	# Check and convert UTF-8 character encoded database to SQL_ASCII
	utf8_to_sql_ascii
	if [ $? -ne 0 ]; then
		return 1
	fi

	# Backup datastore directory, if backup directory already
	# present then exit.
	if [ -d "${old_data_dir}" ]; then
		echo "Files from previous datastore upgrade found,"
		echo "Datastore upgrade cannot continue"
		return 1
	else
		err=`mv ${data_dir} ${old_data_dir} 2>&1`
		if [ $? -ne 0 ]; then
			echo "${err}"
			return 1
		fi
	fi

	# Invoke the dataservice creation script for pbs
	if [ ! -x "${PBS_EXEC}/libexec/install_db" ]; then
		echo "${PBS_EXEC}/libexec/install_db not found"
		return 1
	fi

	resp=`${PBS_EXEC}/libexec/install_db "upgrade" ${change_locale} 2>&1`
	ret=$?

	if [ ${ret} -ne 0 ]; then
		echo "*** Error initializing the PBS dataservice"
		echo "Error details:"
		echo "$resp"
		return ${ret}
	fi

	# Copy the pg_hba.conf from old cluster.
	err=`cp -p ${old_data_dir}/pg_hba.conf ${data_dir}/pg_hba.conf`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi

	set_db_trust_login "${data_dir}"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi
	set_db_trust_login "${old_data_dir}"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	CWD=`pwd`
	cd "${data_dir}"
	#Perform pg_upgrade -c to check if we can upgrade the cluster or not
	err=`su ${user} -c "/bin/sh -c '${PGSQL_LIBSTR} ${inst_dir}/bin/pg_upgrade -b ${old_inst_dir}/bin -B ${inst_dir}/bin -d ${old_data_dir} -D ${data_dir} -c'" 2>&1`
	if [ $? -ne 0 ]; then
		echo "Refer pg_upgrade log files at $PBS_HOME/datastore/pg_upgrade_internal.log,"
		echo "$PBS_HOME/datastore/pg_upgrade_server.log and"
		echo "$PBS_HOME/datastore/pg_upgrade_utility.log for more information"
		revoke_db_trust_login "${data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		revoke_db_trust_login "${old_data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		return 1
	fi

	#Perform pg_upgrade for database upgrade
	err=`su ${user} -c "/bin/sh -c '${PGSQL_LIBSTR} ${inst_dir}/bin/pg_upgrade -b ${old_inst_dir}/bin -B ${inst_dir}/bin -d ${old_data_dir} -D ${data_dir}'" 2>&1`
	if [ $? -ne 0 ]; then
		echo "Refer pg_upgrade log files at $PBS_HOME/datastore/pg_upgrade_internal.log,"
		echo "$PBS_HOME/datastore/pg_upgrade_server.log and"
		echo "$PBS_HOME/datastore/pg_upgrade_utility.log for more information"
		revoke_db_trust_login "${data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		revoke_db_trust_login "${old_data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		return 1
	fi

	# start the dataservice
	${server_ctl} start > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error starting PBS Data Service"
		return 1
	fi

	# Optimizer statistics are not transferred by pg_upgrade, so do it manually.
	ENVSTR="PGPORT=${PBS_DATA_SERVICE_PORT}; export PGPORT; PGHOST=${PBS_SERVER}; export PGHOST; PGUSER=${user}; export PGUSER; "
	err=`su ${user} -c "/bin/sh -c '${PGSQL_LIBSTR} ${ENVSTR} ${data_dir}/analyze_new_cluster.sh'"`

	# Update locale of pbs database to C
	if [ ${change_locale} -eq 1 ]; then
		${inst_dir}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${user} -c "update pg_database set datcollate='C', datctype='C'" > /dev/null
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi
	fi

	# stop the dataservice
	${server_ctl} stop > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error stopping PBS Data Service"
		kill -s SIGTERM `ps -ef | grep "${inst_dir}/bin/postgres" | grep -v grep | awk '{if ($3 == 1) print $2}'`
		return 1
	fi
	revoke_db_trust_login "${data_dir}"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	cd "${CWD}"
	# Delete old cluster
	err=`${data_dir}/delete_old_cluster.sh`
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi
}

chkenv() {
	line=`grep -s "^${1}=" $envfile`
	if [ -z "$line" ]; then
		echo "*** setting ${1}=$2"
		echo "${1}=$2" >> $envfile
	else
		echo "*** leave existing $line"
	fi
}

createdir() {
	if [ -n "$1" -a ! -d "$1" ]; then
		if ! mkdir -p "$1"; then
			echo "*** Could not create $1"
			exit 1
		fi
	fi
	if [ -n "$1" -a -n "$2" ]; then
		chmod "$2" "$1"
	fi
}

createpath() {
	while read mode dir ;do
		createdir "${PBS_HOME}/${dir}" $mode
	done
}

# Return the name of the PBS server host
get_server_hostname() {
	shn=""
	if [ -z "${PBS_PRIMARY}" -o -z "${PBS_SECONDARY}" ] ; then
		if [ -z "${PBS_SERVER_HOST_NAME}" ]; then
			shn="${PBS_SERVER}"
		else
			shn="${PBS_SERVER_HOST_NAME}"
		fi
	else
		shn="${PBS_PRIMARY}"
	fi
	echo ${shn} | awk '{print tolower($0)}'
}

# Ensure the supplied hostname is valid
check_hostname() {
	# Check the hosts file
	getent hosts "${1}" >/dev/null 2>&1 && return 0
	# Check DNS
	host "${1}" >/dev/null 2>&1 && return 0
	return 1
}

#
# Start of the pbs_habitat script
#
conf=${PBS_CONF_FILE:-/etc/pbs.conf}
ostype=`uname 2>/dev/null`
umask 022

echo "***"

# Source pbs.conf to get paths: PBS_EXEC and PBS_HOME and PBS_START_*
. $conf

# Ensure certain variables are set
if [ -z "$PBS_EXEC" ]; then
	echo "*** PBS_EXEC is not set"
	exit 1
fi
if [ ! -d "$PBS_EXEC" ]; then
	echo "*** $PBS_EXEC directory does not exist"
	exit 1
fi
if [ -z "$PBS_HOME" ]; then
	echo "*** PBS_HOME is not set"
	exit 1
fi
if [ ! -d "$PBS_HOME" ]; then
	echo "*** WARNING: PBS_HOME not found in $PBS_HOME"
	if [ -x "$PBS_EXEC/sbin/pbs_server" ]; then
		component="server"
	elif [ -x "$PBS_EXEC/sbin/pbs_mom" ]; then
		component="execution"
	else
		echo "*** No need to execute pbs_habitat in client installation."
		exit 0
	fi
	${PBS_EXEC}/libexec/pbs_postinstall $component $PBS_VERSION $PBS_EXEC $PBS_HOME "" "sameconf"
fi

# Store the old PBS VERSION for later use
if [ -f "$PBS_HOME/pbs_version" ]; then
	old_pbs_version=`cat $PBS_HOME/pbs_version`
fi

# Get the current PBS Pro version from qstat
if [ -x "$PBS_EXEC/bin/qstat" ]; then
	pbs_version=`"${PBS_EXEC}/bin/qstat" --version | sed -e 's/^.* = //'`
	if [ -z "$pbs_version" ]; then
		echo "*** Could not obtain PBS Pro version from qstat"
		exit 1
	fi
	if [ "$pbs_version" != "$PBS_VERSION" ]; then
		echo "*** Version mismatch."
		echo "*** Build version is $PBS_VERSION"
		echo "*** qstat version is $pbs_version"
		exit 1
	fi
else
	echo "*** File not found: $PBS_EXEC/bin/qstat"
	exit 1
fi

# Perform sanity check on server name in pbs.conf
server_hostname=`get_server_hostname`
[ "$server_hostname" = 'change_this_to_pbs_pro_server_hostname' ] && server_hostname=''
if [ -z "${server_hostname}" ] ; then
	echo "***" >&2
	echo "*** The hostname of the PBS Pro server in ${conf} is invalid." >&2
	echo "*** Update the configuration file before starting PBS Pro." >&2
	echo "***" >&2
	exit 1
fi
check_hostname "${server_hostname}"
if [ $? -ne 0 ]; then
	echo "***" >&2
	echo "*** The PBS Pro server could not be found: $server_hostname" >&2
	echo "*** This value must resolve to a valid IP address." >&2
	echo "***" >&2
	exit 1
fi
server=`echo ${server_hostname} | awk -F. '{print $1}'`

# Check the clienthost entries if present
if [ -f "$PBS_HOME/mom_priv/config" ]; then
	cmd='cat "$PBS_HOME/mom_priv/config"'
	cmd="$cmd | sed -e 's/\t/ /g' -e 's/ \\+/ /g' -e 's/^ //' -e 's/ $//'"
	cmd="$cmd | grep '\\\$clienthost' | cut -d' ' -f2"
	for host in `eval $cmd`; do
		if [ -z "$host" -o "$host" = 'CHANGE_THIS_TO_PBS_PRO_SERVER_HOSTNAME' ]; then
			echo "***" >&2
			echo "*** Invalid entry in $PBS_HOME/mom_priv/config" >&2
			echo "*** for clienthost: $host" >&2
			echo "***" >&2
			exit 1
		fi
		if ! check_hostname "${host}"; then
			echo "***" >&2
			echo "*** Invalid entry in $PBS_HOME/mom_priv/config" >&2
			echo "*** for clienthost: $host" >&2
			echo "*** This value must resolve to a valid IP address." >&2
			echo "***" >&2
			exit 1
		fi
	done
fi

if [ "${PBS_START_SERVER:-0}" != 0 ] ; then
	# Source the file that sets PGSQL_LIBSTR
	. "$PBS_EXEC"/libexec/pbs_pgsql_env.sh

	PBS_licensing_loc_file=PBS_licensing_loc

	# On some platforms LD_LIBRARY_PATH etc is not passed on after su
	# so we set it again after the su. PGSQL_LIBSTR contains the line to
	# set the variable again
	export PGSQL_LIBSTR

	dbuser_fl="${PBS_HOME}/server_priv/db_user"
	PBS_DATA_SERVICE_USER=`get_db_user`
	if [ $? -ne 0 ]; then
		echo "Could not retrieve PBS Data Service User"
		exit 1
	fi

	# Do not export the PBS_DATA_SERVICE_USER as a env var
	# since that would cause a false warning message
	# that "deprecated" variable PBS_DATA_SERVICE_USER is
	# being ignored.

	chk_dataservice_user ${PBS_DATA_SERVICE_USER}
	if [ $? -ne 0 ]; then
		exit 1
	fi

	# create the database user file if it does not exist
	if [ ! -f "${dbuser_fl}" ]; then
		printf "${PBS_DATA_SERVICE_USER}" >"${dbuser_fl}"
		chmod 0600 "${dbuser_fl}"
	fi

	server_started=0
	PBS_DATA_SERVICE_PORT=${PBS_DATA_SERVICE_PORT:-"15007"}
	export PBS_DATA_SERVICE_PORT

	create_new_svr_data=1

	# invoke the dataservice creation script for pbs
	if [ ! -x "${PBS_EXEC}/libexec/install_db" ]; then
		echo "${PBS_EXEC}/libexec/install_db not found"
		exit 1
	fi
	resp=`${PBS_EXEC}/libexec/install_db 2>&1`
	ret=$?
	if [ $ret -eq 2 ]; then
		create_new_svr_data=0
	elif [ $ret -ne 0 ]; then
		echo "*** Error initializing the PBS dataservice"
		echo "Error details:"
		echo "$resp"
		exit $ret
	fi

	export PBS_HOME
	export PBS_EXEC
	export PBS_SERVER
	export PBS_ENVIRONMENT

	if [ $create_new_svr_data -eq 0 ]; then
		# datastore directory already exists
		# do database upgrade activities
		upgrade_pbs_database
		ret=$?
		if [ $ret -ne 0 ]; then
			if [ $ret -eq 2 ]; then
				echo "It appears that PostgreSQL has been upgraded independently of PBS Pro."
				echo "The PBS Pro database must be manually upgraded. Please refer to the"
				echo "instructions in the release notes for version 19.1.3 located here:"
				echo "https://github.com/PBSPro/pbspro/releases"
			else 
				echo "Failed to upgrade PBS Datastore"
			fi
			exit $ret
		else
			#Remove old dataservice directory
			if [ `echo "${old_inst_dir}"| grep "pgsql.old"` ]; then
				err=`rm -rf "${old_inst_dir}"`
				if [ $? -ne 0 ]; then
					echo "$err"
				fi
			fi
		fi
		# do schema upgrade activities
		set_db_trust_login "${PBS_HOME}/datastore"
		${PBS_EXEC}/libexec/pbs_schema_upgrade ${PBS_DATA_SERVICE_PORT} ${PBS_DATA_SERVICE_USER}
		ret=$?
		if [ $ret -ne 0 ]; then
			revoke_db_trust_login "${PBS_HOME}/datastore"
			echo "Failed to upgrade PBS Datastore"
			exit $ret
		fi
		# We need to regenerate the db_password file since we have changed encryption/decryption
		# library from DES to AES in PBS Version PBS_AES_SWITCH_VER
		if [ "$old_pbs_version" \< "${PBS_AES_SWITCH_VER}" ] ;then
			rm -f  "${PBS_HOME}/server_priv/db_password"
			err=`${PBS_EXEC}/sbin/pbs_ds_password -r`
			if [ $? -ne 0 ]; then
				echo $err
				echo "Error setting password for PBS Data Service"
				${server_ctl} stop > /dev/null 2>&1
				revoke_db_trust_login "${PBS_HOME}/datastore"
				exit 1
			fi
		fi
		revoke_db_trust_login "${PBS_HOME}/datastore"
	else
		# Upgrade from FS version to DB
		if [ -f ${PBS_HOME}/server_priv/serverdb ] ; then
			# migrate filesystem data into db version
			${PBS_EXEC}/sbin/pbs_server -t updatedb > /dev/null
			ret=$?
			if [ $ret -ne 0 -o -f ${PBS_HOME}/server_priv/serverdb ] ; then
				echo "*** Error during overlay upgrade"
				exit $ret
			fi
			create_new_svr_data=0
		fi
	fi

	if [ $create_new_svr_data -eq 1 ] ; then
		echo "*** Setting default queue and resource limits."
		echo "***"

		${PBS_EXEC}/sbin/pbs_server -t create -a 0 > /dev/null
		ret=$?
		if [ $ret -ne 0 ]; then
			echo "*** Error starting pbs server"
			exit $ret
		fi
		server_started=1
		if is_cray_xt; then
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				set server restrict_res_to_release_on_suspend = ncpus
			EOF
		fi
		${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
			create queue workq
			set queue workq queue_type = Execution
			set queue workq enabled = True
			set queue workq started = True
			set server default_queue = workq
			set server log_events = 511
			set server mail_from = adm
			set server query_other_jobs = True
			set server scheduler_iteration = 600
			set server resources_default.ncpus = 1
			set server scheduling = True
		EOF
		if [ -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file ]; then
			read ans < ${PBS_HOME}/server_priv/$PBS_licensing_loc_file
			echo "*** Setting license file location(s)."
			echo "***"
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				set server pbs_license_info = $ans
			EOF
			if ! is_cray_xt; then
				rm -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file	# clean up after INSTALL
			fi
		fi
	else
		# the upgrade case:  serverdb already exists, but license file
		# information is new
		if [ -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file ]; then
			read ans < ${PBS_HOME}/server_priv/$PBS_licensing_loc_file
			echo "*** Setting license file location(s)."
			echo "***"
			${PBS_EXEC}/sbin/pbs_server > /dev/null
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				unset server pbs_license_file_location
				set server pbs_license_info = $ans
			EOF
			if ! is_cray_xt; then
				rm -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file	# clean up after INSTALL
			else
				${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
					set server restrict_res_to_release_on_suspend += ncpus
				EOF
			fi
			server_started=1
		fi
	fi

	if [ $PBS_START_MOM != 0 ]; then
		if [ $server_started -eq 0 ]; then
			${PBS_EXEC}/sbin/pbs_server > /dev/null
			server_started=1
		fi

		if ${PBS_EXEC}/bin/pbsnodes localhost > /dev/null 2>&1 ||
		   ${PBS_EXEC}/bin/pbsnodes $server > /dev/null 2>&1; then
			:
		else
			# node $server is not already available, create
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				create node $server
			EOF
		fi
	fi

	if [ $server_started -eq 1 ]; then
		${PBS_EXEC}/bin/qterm
	fi
fi

#
# For overlay upgrades PBS_START_MOM will be disabled per the install
# instructions. There may still be job and task files present that
# require updating.
#
if [ -d ${PBS_HOME}/mom_priv/jobs ]; then
	upgrade_cmd="${PBS_EXEC}/sbin/pbs_upgrade_job"
	if [ -x ${upgrade_cmd} ]; then
		total=0
		upgraded=0
		for file in ${PBS_HOME}/mom_priv/jobs/*.JB; do
			if [ -f ${file} ]; then
				${upgrade_cmd} -f ${file}
				if [ $? -ne 0 ]; then
					echo "Failed to upgrade ${file}"
				else
					upgraded=`expr ${upgraded} + 1`
				fi
				total=`expr ${total} + 1`
			fi
		done
		if [ ${total} -gt 0 ]; then
			echo "Upgraded ${upgraded} of ${total} job files."
		fi
	else
		echo "WARNING: $upgrade_cmd not found!"
	fi
fi

# Update the version file at the very end, after everything else succeeds.
# This allows it to be re-run, in case a previous update attempt failed.
# Clobber the existing pbs_version file and populate it with current version.
# This will prevent updating PBS_HOME if this same version is re-installed
#
echo "${pbs_version}" >"$PBS_HOME/pbs_version"

echo "*** End of ${0}"
exit 0
