/*
 * src/bin/pg_autoctl/pgctl.c
 *   API for controling PostgreSQL, using its binary tooling (pg_ctl,
 *   pg_controldata, pg_basebackup and such).
 *
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the PostgreSQL License.
 *
 */

#include <dirent.h>
#include <inttypes.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#include "postgres_fe.h"
#include "pqexpbuffer.h"

#include "defaults.h"
#include "env_utils.h"
#include "file_utils.h"
#include "log.h"
#include "parsing.h"
#include "pgctl.h"
#include "pgsql.h"
#include "pgsetup.h"
#include "pgtuning.h"
#include "signals.h"
#include "string_utils.h"

#define RUN_PROGRAM_IMPLEMENTATION
#include "runprogram.h"

#define AUTOCTL_CONF_INCLUDE_COMMENT \
	" # Auto-generated by pg_auto_failover, do not remove\n"

#define AUTOCTL_CONF_INCLUDE_LINE "include '" AUTOCTL_DEFAULTS_CONF_FILENAME "'"
#define AUTOCTL_SB_CONF_INCLUDE_LINE "include '" AUTOCTL_STANDBY_CONF_FILENAME "'"

static bool pg_include_config(const char *configFilePath,
							  const char *configIncludeLine,
							  const char *configIncludeComment);
static bool ensure_default_settings_file_exists(const char *configFilePath,
												GUC *settings,
												PostgresSetup *pgSetup,
												const char *hostname,
												bool includeTuning);
static bool prepare_guc_settings_from_pgsetup(const char *configFilePath,
											  PQExpBuffer config,
											  GUC *settings,
											  PostgresSetup *pgSetup,
											  const char *hostname,
											  bool includeTuning);
static void log_program_output(Program prog, int outLogLevel, int errorLogLevel);


static bool prepare_recovery_settings(const char *pgdata,
									  ReplicationSource *replicationSource,
									  char *primaryConnInfo,
									  char *primarySlotName,
									  char *targetLSN,
									  char *targetAction,
									  char *targetTimeline);

static bool escape_recovery_conf_string(char *destination,
										int destinationSize,
										const char *recoveryConfString);
static bool prepare_primary_conninfo(char *primaryConnInfo,
									 int primaryConnInfoSize,
									 const char *primaryHost, int primaryPort,
									 const char *replicationUsername,
									 const char *dbname,
									 const char *replicationPassword,
									 const char *applicationName,
									 SSLOptions sslOptions,
									 bool escape);
static bool prepare_conninfo_sslmode(PQExpBuffer buffer, SSLOptions sslOptions);

static bool pg_write_recovery_conf(const char *pgdata,
								   ReplicationSource *replicationSource);
static bool pg_write_standby_signal(const char *pgdata,
									ReplicationSource *replicationSource);
static bool ensure_empty_tablespace_dirs(const char *pgdata);

/*
 * Get pg_ctl --version output in pgSetup->pg_version.
 */
bool
pg_ctl_version(PostgresSetup *pgSetup)
{
	Program prog = run_program(pgSetup->pg_ctl, "--version", NULL);
	char pg_version_string[PG_VERSION_STRING_MAX] = { 0 };
	int pg_version = 0;

	if (prog.returnCode != 0)
	{
		errno = prog.error;
		log_error("Failed to run \"pg_ctl --version\" using program \"%s\": %m",
				  pgSetup->pg_ctl);
		free_program(&prog);
		return false;
	}

	if (!parse_version_number(prog.stdOut,
							  pg_version_string,
							  PG_VERSION_STRING_MAX,
							  &pg_version))
	{
		/* errors have already been logged */
		free_program(&prog);
		return false;
	}
	free_program(&prog);

	strlcpy(pgSetup->pg_version, pg_version_string, PG_VERSION_STRING_MAX);

	return true;
}


/*
 * set_pg_ctl_from_PG_CONFIG sets given pgSetup->pg_ctl to the pg_ctl binary
 * installed in the bindir of the target Postgres installation:
 *
 *  $(${PG_CONFIG} --bindir)/pg_ctl
 */
bool
set_pg_ctl_from_config_bindir(PostgresSetup *pgSetup, const char *pg_config)
{
	char pg_ctl[MAXPGPATH] = { 0 };

	if (!file_exists(pg_config))
	{
		log_debug("set_pg_ctl_from_config_bindir: file not found: \"%s\"",
				  pg_config);
		return false;
	}

	Program prog = run_program(pg_config, "--bindir", NULL);

	char *lines[1];

	if (prog.returnCode != 0)
	{
		errno = prog.error;
		log_error("Failed to run \"pg_config --bindir\" using program \"%s\": %m",
				  pg_config);
		free_program(&prog);
		return false;
	}

	if (splitLines(prog.stdOut, lines, 1) != 1)
	{
		log_error("Unable to parse output from pg_config --bindir");
		free_program(&prog);
		return false;
	}

	char *bindir = lines[0];
	join_path_components(pg_ctl, bindir, "pg_ctl");

	/* we're now done with the Program and its output */
	free_program(&prog);

	if (!file_exists(pg_ctl))
	{
		log_error("Failed to find pg_ctl at \"%s\" from PG_CONFIG at \"%s\"",
				  pgSetup->pg_ctl,
				  pg_config);
		return false;
	}

	strlcpy(pgSetup->pg_ctl, pg_ctl, sizeof(pgSetup->pg_ctl));

	return true;
}


/*
 * Read some of the information from pg_controldata output.
 */
bool
pg_controldata(PostgresSetup *pgSetup, bool missing_ok)
{
	char globalControlPath[MAXPGPATH] = { 0 };
	char pg_controldata_path[MAXPGPATH] = { 0 };

	if (pgSetup->pgdata[0] == '\0' || pgSetup->pg_ctl[0] == '\0')
	{
		log_error("BUG: pg_controldata: missing pgSetup pgdata or pg_ctl");
		return false;
	}

	/* globalControlFilePath = $PGDATA/global/pg_control */
	join_path_components(globalControlPath, pgSetup->pgdata, "global/pg_control");

	/*
	 * Refrain from doing too many pg_controldata checks, only proceed when the
	 * PGDATA/global/pg_control file exists on-disk: that's the first check
	 * that pg_controldata does anyway.
	 */
	if (!file_exists(globalControlPath))
	{
		return false;
	}

	/* now find the pg_controldata binary */
	path_in_same_directory(pgSetup->pg_ctl, "pg_controldata", pg_controldata_path);
	log_debug("%s %s", pg_controldata_path, pgSetup->pgdata);

	/* We parse the output of pg_controldata, make sure it's as expected */
	setenv("LANG", "C", 1);
	Program prog = run_program(pg_controldata_path, pgSetup->pgdata, NULL);

	if (prog.returnCode == 0)
	{
		if (prog.stdOut == NULL)
		{
			/* happens sometimes, and I don't know why */
			log_warn("Got empty output from `%s %s`, trying again in 1s",
					 pg_controldata_path, pgSetup->pgdata);
			sleep(1);

			return pg_controldata(pgSetup, missing_ok);
		}

		if (!parse_controldata(&pgSetup->control, prog.stdOut))
		{
			log_error("%s %s", pg_controldata_path, pgSetup->pgdata);
			log_warn("Failed to parse pg_controldata output:\n%s", prog.stdOut);
			free_program(&prog);
			return false;
		}

		free_program(&prog);
		return true;
	}
	else
	{
		int errorLogLevel = missing_ok ? LOG_DEBUG : LOG_ERROR;

		(void) log_program_output(prog, LOG_INFO, errorLogLevel);

		log_level(errorLogLevel,
				  "Failed to run \"%s\" on \"%s\", see above for details",
				  pg_controldata_path, pgSetup->pgdata);

		free_program(&prog);

		return missing_ok;
	}
}


/*
 * set_pg_ctl_from_PG_CONFIG sets the path to pg_ctl following the exported
 * environment variable PG_CONFIG, when it is found in the environment.
 *
 * Postgres developer environments often define PG_CONFIG in the environment to
 * build extensions for a specific version of Postgres. Let's use the hint here
 * too.
 */
bool
set_pg_ctl_from_PG_CONFIG(PostgresSetup *pgSetup)
{
	char PG_CONFIG[MAXPGPATH] = { 0 };

	if (!env_exists("PG_CONFIG"))
	{
		/* then we don't use PG_CONFIG to find pg_ctl */
		return false;
	}

	if (!get_env_copy("PG_CONFIG", PG_CONFIG, sizeof(PG_CONFIG)))
	{
		/* errors have already been logged */
		return false;
	}

	if (!file_exists(PG_CONFIG))
	{
		log_error("Failed to find a file for PG_CONFIG environment value \"%s\"",
				  PG_CONFIG);
		return false;
	}

	if (!set_pg_ctl_from_config_bindir(pgSetup, PG_CONFIG))
	{
		/* errors have already been logged */
		return false;
	}

	if (!pg_ctl_version(pgSetup))
	{
		log_fatal("Failed to get version info from %s --version",
				  pgSetup->pg_ctl);
		return false;
	}

	log_debug("Found pg_ctl for PostgreSQL %s at %s following PG_CONFIG",
			  pgSetup->pg_version, pgSetup->pg_ctl);

	return true;
}


/*
 * set_pg_ctl_from_pg_config sets the path to pg_ctl by using pg_config
 * --bindir when there is a single pg_config found in the PATH.
 *
 * When using debian/ubuntu packaging then pg_config is installed as part as
 * the postgresql-common package in /usr/bin, whereas pg_ctl is installed in a
 * major version dependent location such as /usr/lib/postgresql/12/bin, and
 * those locations are not included in the PATH.
 *
 * So when we can't find pg_ctl anywhere in the PATH, we look for pg_config
 * instead, and then use pg_config --bindir to discover the pg_ctl we can use.
 */
bool
set_pg_ctl_from_pg_config(PostgresSetup *pgSetup)
{
	SearchPath all_pg_configs = { 0 };
	SearchPath pg_configs = { 0 };

	if (!search_path("pg_config", &all_pg_configs))
	{
		return false;
	}

	if (!search_path_deduplicate_symlinks(&all_pg_configs, &pg_configs))
	{
		log_error("Failed to resolve symlinks found in PATH entries, "
				  "see above for details");
		return false;
	}

	switch (pg_configs.found)
	{
		case 0:
		{
			log_warn("Failed to find either pg_ctl or pg_config in PATH");
			return false;
		}

		case 1:
		{
			if (!set_pg_ctl_from_config_bindir(pgSetup, pg_configs.matches[0]))
			{
				/* errors have already been logged */
				return false;
			}

			if (!pg_ctl_version(pgSetup))
			{
				log_fatal("Failed to get version info from %s --version",
						  pgSetup->pg_ctl);
				return false;
			}

			log_debug("Found pg_ctl for PostgreSQL %s at %s from pg_config "
					  "found in PATH at \"%s\"",
					  pgSetup->pg_version,
					  pgSetup->pg_ctl,
					  pg_configs.matches[0]);

			return true;
		}

		default:
		{
			log_info("Found more than one pg_config entry in current PATH:");

			for (int i = 0; i < pg_configs.found; i++)
			{
				PostgresSetup currentPgSetup = { 0 };

				strlcpy(currentPgSetup.pg_ctl,
						pg_configs.matches[i],
						sizeof(currentPgSetup.pg_ctl));

				if (!pg_ctl_version(&currentPgSetup))
				{
					/*
					 * Because of this it's possible that there's now only a
					 * single working version of pg_ctl found in PATH. If
					 * that's the case we will still not use that by default,
					 * since the users intention is unclear. They might have
					 * wanted to use the version of pg_ctl that we could not
					 * parse the version string for. So we warn and continue,
					 * the user should make their intention clear by using the
					 * --pg_ctl option (or changing PATH).
					 */
					log_warn("Failed to get version info from %s --version",
							 currentPgSetup.pg_ctl);
					continue;
				}

				log_info("Found \"%s\" for pg version %s",
						 currentPgSetup.pg_ctl,
						 currentPgSetup.pg_version);
			}

			log_info("HINT: export PG_CONFIG to a specific pg_config entry");

			return false;
		}
	}

	return false;
}


/*
 * Find "pg_ctl" programs in the PATH. If a single one exists, set its absolute
 * location in pg_ctl, and the PostgreSQL version number in pg_version.
 *
 * Returns how many "pg_ctl" programs have been found in the PATH.
 */
bool
config_find_pg_ctl(PostgresSetup *pgSetup)
{
	SearchPath all_pg_ctls = { 0 };
	SearchPath pg_ctls = { 0 };

	pgSetup->pg_ctl[0] = '\0';
	pgSetup->pg_version[0] = '\0';

	/*
	 * Postgres developer environments often define PG_CONFIG in the
	 * environment to build extensions for a specific version of Postgres.
	 * Let's use the hint here too.
	 */
	if (set_pg_ctl_from_PG_CONFIG(pgSetup))
	{
		return true;
	}

	/* no PG_CONFIG. let's use the more classic approach with PATH instead */
	if (!search_path("pg_ctl", &all_pg_ctls))
	{
		return false;
	}

	if (!search_path_deduplicate_symlinks(&all_pg_ctls, &pg_ctls))
	{
		log_error("Failed to resolve symlinks found in PATH entries, "
				  "see above for details");
		return false;
	}

	if (pg_ctls.found == 1)
	{
		char *program = pg_ctls.matches[0];

		strlcpy(pgSetup->pg_ctl, program, MAXPGPATH);

		if (!pg_ctl_version(pgSetup))
		{
			log_fatal("Failed to get version info from \"%s\" --version",
					  pgSetup->pg_ctl);
			return false;
		}

		log_debug("Found pg_ctl for PostgreSQL %s at \"%s\"",
				  pgSetup->pg_version,
				  pgSetup->pg_ctl);

		return true;
	}
	else
	{
		/*
		 * Then, first look for pg_config --bindir with pg_config in PATH,
		 * we might have a single entry there, as is the case on a typical
		 * debian/ubuntu packaging, in /usr/bin/pg_config installed from
		 * the postgresql-common package.
		 */
		PostgresSetup pgSetupFromPgConfig = { 0 };

		if (pg_ctls.found == 0)
		{
			log_debug("Failed to find pg_ctl in PATH, looking for pg_config");
		}
		else
		{
			log_debug("Found %d entries for pg_ctl in PATH, "
					  "looking for pg_config",
					  pg_ctls.found);
		}

		if (set_pg_ctl_from_pg_config(&pgSetupFromPgConfig))
		{
			strlcpy(pgSetup->pg_ctl,
					pgSetupFromPgConfig.pg_ctl,
					sizeof(pgSetup->pg_ctl));

			strlcpy(pgSetup->pg_version,
					pgSetupFromPgConfig.pg_version,
					sizeof(pgSetup->pg_version));
			return true;
		}

		/*
		 * We failed to find a single pg_config in $PATH, error out and
		 * complain about the situation with enough details that the user
		 * can understand our struggle in picking a Postgres major version
		 * for them.
		 */
		log_info("Found more than one pg_ctl entry in current PATH, "
				 "and failed to find a single pg_config entry in current PATH");

		for (int i = 0; i < pg_ctls.found; i++)
		{
			PostgresSetup currentPgSetup = { 0 };

			strlcpy(currentPgSetup.pg_ctl, pg_ctls.matches[i], MAXPGPATH);

			if (!pg_ctl_version(&currentPgSetup))
			{
				/*
				 * Because of this it's possible that there's now only a
				 * single working version of pg_ctl found in PATH. If
				 * that's the case we will still not use that by default,
				 * since the users intention is unclear. They might have
				 * wanted to use the version of pg_ctl that we could not
				 * parse the version string for. So we warn and continue,
				 * the user should make their intention clear by using the
				 * --pg_ctl option (or setting PG_CONFIG, or PATH).
				 */
				log_warn("Failed to get version info from \"%s\" --version",
						 currentPgSetup.pg_ctl);
				continue;
			}

			log_info("Found \"%s\" for pg version %s",
					 currentPgSetup.pg_ctl,
					 currentPgSetup.pg_version);
		}

		log_error("Found several pg_ctl in PATH, please provide --pgctl");

		return false;
	}
}


/*
 * find_pg_config_from_pg_ctl finds the path to pg_config from the known path
 * to pg_ctl. If that exists, we first use the pg_config binary found in the
 * same directory as the pg_ctl binary itself.
 *
 * Otherwise, we have a look at the PG_CONFIG environment variable.
 *
 * Finally, we search in the PATH list for all the matches, and for each of
 * them we run pg_config --bindir, and if that's the directory where we have
 * our known pg_ctl, that's our pg_config.
 *
 * Rationale: when using debian, the postgresql-common package installs a
 * single entry for pg_config in /usr/bin/pg_config, and that's the system
 * default.
 *
 * A version specific file path found in /usr/lib/postgresql/11/bin/pg_config
 * when installing Postgres 11 is installed from the package
 * postgresql-server-dev-11.
 *
 * There is no single default entry for pg_ctl, that said, so we are using the
 * specific path /usr/lib/postgresql/11/bin/pg_config here.
 *
 * So depending on what packages have been deployed on this specific debian
 * instance, we might or might not find a pg_config binary in the same
 * directory as pg_ctl.
 *
 * Note that we could register the full path to whatever pg_config version we
 * use at pg_autoctl create time, but in most cases that is going to be
 * /usr/bin/pg_config, and it will point to a new pg_ctl (version 13 for
 * instance) when you apt-get upgrade your debian testing distribution and it
 * just migrated from Postgres 11 to Postgres 13 (bullseye cycle just did that
 * in december 2020).
 *
 * Either package libpq-dev or postgresql-server-dev-11 (or another version)
 * must be isntalled for this to work.
 */
bool
find_pg_config_from_pg_ctl(const char *pg_ctl, char *pg_config, size_t size)
{
	char pg_config_path[MAXPGPATH] = { 0 };

	/*
	 * 1. try pg_ctl directory
	 */
	path_in_same_directory(pg_ctl, "pg_config", pg_config_path);

	if (file_exists(pg_config_path))
	{
		log_debug("find_pg_config_from_pg_ctl: \"%s\" "
				  "in same directory as pg_ctl",
				  pg_config_path);
		strlcpy(pg_config, pg_config_path, size);
		return true;
	}

	/*
	 * 2. try PG_CONFIG from the environment, and check pg_config --bindir
	 */
	if (env_exists("PG_CONFIG"))
	{
		PostgresSetup pgSetup = { 0 };
		char PG_CONFIG[MAXPGPATH] = { 0 };

		/* check that the pg_config we found relates to the given pg_ctl */
		if (get_env_copy("PG_CONFIG", PG_CONFIG, sizeof(PG_CONFIG)) &&
			file_exists(PG_CONFIG) &&
			set_pg_ctl_from_config_bindir(&pgSetup, PG_CONFIG) &&
			strcmp(pgSetup.pg_ctl, pg_ctl) == 0)
		{
			log_debug("find_pg_config_from_pg_ctl: \"%s\" "
					  "from PG_CONFIG environment variable",
					  pg_config_path);
			strlcpy(pg_config, pg_config_path, size);
			return true;
		}
	}

	/*
	 * 3. search our PATH for pg_config entries and keep the first one that
	 *    relates to our known pg_ctl.
	 */
	SearchPath all_pg_configs = { 0 };
	SearchPath pg_configs = { 0 };

	if (!search_path("pg_config", &all_pg_configs))
	{
		return false;
	}

	if (!search_path_deduplicate_symlinks(&all_pg_configs, &pg_configs))
	{
		log_error("Failed to resolve symlinks found in PATH entries, "
				  "see above for details");
		return false;
	}

	for (int i = 0; i < pg_configs.found; i++)
	{
		PostgresSetup pgSetup = { 0 };

		if (set_pg_ctl_from_config_bindir(&pgSetup, pg_configs.matches[i]) &&
			strcmp(pgSetup.pg_ctl, pg_ctl) == 0)
		{
			log_debug("find_pg_config_from_pg_ctl: \"%s\" "
					  "from PATH search",
					  pg_configs.matches[i]);
			strlcpy(pg_config, pg_configs.matches[i], size);
			return true;
		}
	}

	return false;
}


/*
 * find_extension_control_file ensures that the extension is present in the
 * given Postgres installation. This does the equivalent of:
 * ls -l $(pg_config --sharedir)/extension/pg_stat_statements.control
 */
bool
find_extension_control_file(const char *pg_ctl, const char *extName)
{
	char pg_config_path[MAXPGPATH] = { 0 };
	char extension_path[MAXPGPATH] = { 0 };
	char *share_dir;
	char extension_control_file_name[MAXPGPATH] = { 0 };
	char *lines[1];

	log_debug("Checking if the %s extension is installed", extName);

	if (!find_pg_config_from_pg_ctl(pg_ctl, pg_config_path, MAXPGPATH))
	{
		log_warn("Failed to find pg_config from pg_ctl at \"%s\"", pg_ctl);
		return false;
	}

	Program prog = run_program(pg_config_path, "--sharedir", NULL);

	if (prog.returnCode == 0)
	{
		if (!prog.stdOut)
		{
			log_error("Got empty output from pg_config --sharedir");
			free_program(&prog);
			return false;
		}
		if (splitLines(prog.stdOut, lines, 1) != 1)
		{
			log_error("Unable to parse output from pg_config --sharedir");
			free_program(&prog);
			return false;
		}
		share_dir = lines[0];

		join_path_components(extension_path, share_dir, "extension");
		sformat(extension_control_file_name, MAXPGPATH, "%s.control", extName);
		join_path_components(extension_path, extension_path, extension_control_file_name);
		if (!file_exists(extension_path))
		{
			log_error("Failed to find extension control file \"%s\"",
					  extension_path);
			free_program(&prog);
			return false;
		}
	}
	else
	{
		(void) log_program_output(prog, LOG_INFO, LOG_ERROR);
		log_error("Failed to run \"%s\", see above for details",
				  pg_config_path);
		free_program(&prog);
		return false;
	}
	free_program(&prog);
	return true;
}


/*
 * pg_add_auto_failover_default_settings ensures the pg_auto_failover default
 * settings are included in postgresql.conf. For simplicity, this function
 * reads the whole contents of postgresql.conf into memory.
 */
bool
pg_add_auto_failover_default_settings(PostgresSetup *pgSetup,
									  const char *hostname,
									  const char *configFilePath,
									  GUC *settings)
{
	bool includeTuning = true;
	char pgAutoFailoverDefaultsConfigPath[MAXPGPATH];

	/*
	 * Write the default settings to postgresql-auto-failover.conf.
	 *
	 * postgresql-auto-failover.conf needs to be placed alongside
	 * postgresql.conf for the include to work. Determine the path by finding
	 * the parent directory of postgresql.conf.
	 */
	path_in_same_directory(configFilePath, AUTOCTL_DEFAULTS_CONF_FILENAME,
						   pgAutoFailoverDefaultsConfigPath);

	if (!ensure_default_settings_file_exists(pgAutoFailoverDefaultsConfigPath,
											 settings,
											 pgSetup,
											 hostname,
											 includeTuning))
	{
		return false;
	}

	return pg_include_config(configFilePath,
							 AUTOCTL_CONF_INCLUDE_LINE,
							 AUTOCTL_CONF_INCLUDE_COMMENT);
}


/*
 * pg_auto_failover_default_settings_file_exists returns true when our expected
 * postgresql-auto-failover.conf file exists in PGDATA.
 */
bool
pg_auto_failover_default_settings_file_exists(PostgresSetup *pgSetup)
{
	char pgAutoFailoverDefaultsConfigPath[MAXPGPATH] = { 0 };
	char *contents = NULL;
	long size = 0L;


	join_path_components(pgAutoFailoverDefaultsConfigPath,
						 pgSetup->pgdata,
						 AUTOCTL_DEFAULTS_CONF_FILENAME);


	/* make sure the file exists and is not empty (race conditions) */
	if (!read_file_if_exists(pgAutoFailoverDefaultsConfigPath, &contents, &size))
	{
		return false;
	}

	/* we don't actually need the contents here */
	free(contents);
	bool fileExistsWithContent = size > 0;

	return fileExistsWithContent;
}


/*
 * pg_include_config adds an include line to postgresql.conf to include the
 * given configuration file, with a comment refering pg_auto_failover.
 */
static bool
pg_include_config(const char *configFilePath,
				  const char *configIncludeLine,
				  const char *configIncludeComment)
{
	char *currentConfContents = NULL;
	long currentConfSize = 0L;

	/* read the current postgresql.conf contents */
	if (!read_file(configFilePath, &currentConfContents, &currentConfSize))
	{
		return false;
	}

	/* find the include 'postgresql-auto-failover.conf' line */
	char *includeLine = strstr(currentConfContents, configIncludeLine);

	if (includeLine != NULL && (includeLine == currentConfContents ||
								includeLine[-1] == '\n'))
	{
		log_debug("%s found in \"%s\"", configIncludeLine, configFilePath);

		/* defaults settings are already included */
		free(currentConfContents);
		return true;
	}

	log_debug("Adding %s to \"%s\"", configIncludeLine, configFilePath);

	/* build the new postgresql.conf contents */
	PQExpBuffer newConfContents = createPQExpBuffer();
	if (newConfContents == NULL)
	{
		log_error("Failed to allocate memory");
		free(currentConfContents);
		return false;
	}

	appendPQExpBufferStr(newConfContents, configIncludeLine);
	appendPQExpBufferStr(newConfContents, configIncludeComment);
	appendPQExpBufferStr(newConfContents, currentConfContents);

	/* done with the old postgresql.conf contents */
	free(currentConfContents);

	/* memory allocation could have failed while building string */
	if (PQExpBufferBroken(newConfContents))
	{
		log_error("Failed to allocate memory");
		destroyPQExpBuffer(newConfContents);
		return false;
	}

	/* write the new postgresql.conf */
	if (!write_file(newConfContents->data, newConfContents->len, configFilePath))
	{
		destroyPQExpBuffer(newConfContents);
		return false;
	}

	destroyPQExpBuffer(newConfContents);

	return true;
}


/*
 * ensure_default_settings_file_exists writes the postgresql-auto-failover.conf
 * file to the database directory.
 */
static bool
ensure_default_settings_file_exists(const char *configFilePath,
									GUC *settings,
									PostgresSetup *pgSetup,
									const char *hostname,
									bool includeTuning)
{
	PQExpBuffer defaultConfContents = createPQExpBuffer();

	if (defaultConfContents == NULL)
	{
		log_error("Failed to allocate memory");
		return false;
	}

	if (!prepare_guc_settings_from_pgsetup(configFilePath,
										   defaultConfContents,
										   settings,
										   pgSetup,
										   hostname,
										   includeTuning))
	{
		/* errors have already been logged */
		destroyPQExpBuffer(defaultConfContents);
		return false;
	}

	if (file_exists(configFilePath))
	{
		char *currentDefaultConfContents = NULL;
		long currentDefaultConfSize = 0L;

		if (!read_file(configFilePath, &currentDefaultConfContents,
					   &currentDefaultConfSize))
		{
			/* technically, we could still try writing, but this is pretty
			 * suspicious */
			destroyPQExpBuffer(defaultConfContents);
			return false;
		}

		if (strcmp(currentDefaultConfContents, defaultConfContents->data) == 0)
		{
			/* file is there and has the same contents, nothing to do */
			log_debug("Default settings file \"%s\" exists", configFilePath);
			free(currentDefaultConfContents);
			destroyPQExpBuffer(defaultConfContents);
			return true;
		}

		log_info("Contents of \"%s\" have changed, overwriting", configFilePath);
		free(currentDefaultConfContents);
	}
	else
	{
		log_debug("Configuration file \"%s\" doesn't exists yet, creating",
				  configFilePath);
	}

	if (!write_file(defaultConfContents->data,
					defaultConfContents->len,
					configFilePath))
	{
		destroyPQExpBuffer(defaultConfContents);
		return false;
	}

	log_debug("Wrote file \"%s\" with content:\n%s",
			  configFilePath, defaultConfContents->data);

	destroyPQExpBuffer(defaultConfContents);

	return true;
}


/*
 * prepare_guc_settings_from_pgsetup replaces some of the given GUC settings
 * with dynamic values found in the pgSetup argument, and prepare them in the
 * expected format for a postgresql.conf file in the given PQExpBuffer.
 *
 * While most of our settings are handle in a static way and thus known at
 * compile time, some of them can be provided by our users, such as
 * listen_addresses, port, and SSL related configuration parameters.
 */

#define streq(x, y) ((x != NULL) && (y != NULL) && (strcmp(x, y) == 0))

static bool
prepare_guc_settings_from_pgsetup(const char *configFilePath,
								  PQExpBuffer config,
								  GUC *settings,
								  PostgresSetup *pgSetup,
								  const char *hostname,
								  bool includeTuning)
{
	char tuning[BUFSIZE] = { 0 };
	int settingIndex = 0;

	appendPQExpBufferStr(config, "# Settings by pg_auto_failover\n");

	/* replace placeholder values with actual pgSetup values */
	for (settingIndex = 0; settings[settingIndex].name != NULL; settingIndex++)
	{
		GUC *setting = &settings[settingIndex];

		/*
		 * Settings for "listen_addresses" and "port" are replaced with the
		 * respective values present in pgSetup allowing those to be dynamic.
		 *
		 * At the moment our "needs quote" heuristic is pretty simple.
		 * There's the one parameter within those that we hardcode from
		 * pg_auto_failover that needs quoting, and that's
		 * listen_addresses.
		 *
		 * The reason why POSTGRES_DEFAULT_LISTEN_ADDRESSES is not quoting
		 * the value directly in the constant is that we are using that
		 * value both in the configuration file and at the pg_ctl start
		 * --options "-h *" command line.
		 *
		 * At the command line, using --options "-h '*'" would give:
		 *    could not create listen socket for "'*'"
		 */
		if (streq(setting->name, "listen_addresses"))
		{
			appendPQExpBuffer(config, "%s = '%s'\n",
							  setting->name,
							  pgSetup->listen_addresses);
		}
		else if (streq(setting->name, "password_encryption"))
		{
			/*
			 * Set password_encryption if the authMethod is password based.
			 */
			if (streq(pgSetup->authMethod, "md5") ||
				streq(pgSetup->authMethod, "scram-sha-256"))
			{
				appendPQExpBuffer(config, "%s = '%s'\n",
								  setting->name,
								  pgSetup->authMethod);
			}
			else if (streq(pgSetup->authMethod, "password"))
			{
				/*
				 * The "password" auth method supports only the "md5" and
				 * "scram-sha-256" password encryption settings.
				 * Default the encryption setting to "scram-sha-256" in this
				 * case, as it is the more secure alternative.
				 */
				appendPQExpBuffer(config, "%s = '%s'\n",
								  setting->name,
								  "scram-sha-256");
			}
		}
		else if (streq(setting->name, "port"))
		{
			appendPQExpBuffer(config, "%s = %d\n",
							  setting->name,
							  pgSetup->pgport);
		}
		else if (streq(setting->name, "ssl"))
		{
			appendPQExpBuffer(config, "%s = %s\n",
							  setting->name,
							  pgSetup->ssl.active == 0 ? "off" : "on");
		}
		else if (streq(setting->name, "ssl_ca_file"))
		{
			if (!IS_EMPTY_STRING_BUFFER(pgSetup->ssl.caFile))
			{
				appendPQExpBuffer(config, "%s = '%s'\n",
								  setting->name, pgSetup->ssl.caFile);
			}
		}
		else if (streq(setting->name, "ssl_crl_file"))
		{
			if (!IS_EMPTY_STRING_BUFFER(pgSetup->ssl.crlFile))
			{
				appendPQExpBuffer(config, "%s = '%s'\n",
								  setting->name, pgSetup->ssl.crlFile);
			}
		}
		else if (streq(setting->name, "ssl_cert_file"))
		{
			if (!IS_EMPTY_STRING_BUFFER(pgSetup->ssl.serverCert))
			{
				appendPQExpBuffer(config, "%s = '%s'\n",
								  setting->name, pgSetup->ssl.serverCert);
			}
		}
		else if (streq(setting->name, "ssl_key_file"))
		{
			if (!IS_EMPTY_STRING_BUFFER(pgSetup->ssl.serverKey))
			{
				appendPQExpBuffer(config, "%s = '%s'\n",
								  setting->name, pgSetup->ssl.serverKey);
			}
		}
		else if (streq(setting->name, "recovery_target_lsn"))
		{
			if (streq(setting->value, "'immediate'"))
			{
				appendPQExpBuffer(config, "recovery_target = 'immediate'\n");
			}
			else
			{
				appendPQExpBuffer(config, "%s = %s\n",
								  setting->name, setting->value);
			}
		}
		else if (streq(setting->name, "citus.node_conninfo"))
		{
			appendPQExpBuffer(config, "%s = '", setting->name);

			/* add sslmode, sslrootcert, and sslcrl if needed */
			if (!prepare_conninfo_sslmode(config, pgSetup->ssl))
			{
				/* errors have already been logged */
				return false;
			}

			appendPQExpBufferStr(config, "'\n");
		}
		else if (streq(setting->name, "citus.use_secondary_nodes"))
		{
			if (!IS_EMPTY_STRING_BUFFER(pgSetup->citusClusterName))
			{
				appendPQExpBuffer(config, "%s = 'always'\n", setting->name);
			}
		}
		else if (streq(setting->name, "citus.cluster_name"))
		{
			if (!IS_EMPTY_STRING_BUFFER(pgSetup->citusClusterName))
			{
				appendPQExpBuffer(config, "%s = '%s'\n",
								  setting->name, pgSetup->citusClusterName);
			}
		}
		else if (streq(setting->name, "citus.local_hostname"))
		{
			if (hostname != NULL && !IS_EMPTY_STRING_BUFFER(hostname))
			{
				appendPQExpBuffer(config, "%s = '%s'\n",
								  setting->name, hostname);
			}
		}
		else if (setting->value != NULL &&
				 !IS_EMPTY_STRING_BUFFER(setting->value))
		{
			appendPQExpBuffer(config, "%s = %s\n",
							  setting->name,
							  setting->value);
		}
		else if (setting->value == NULL ||
				 IS_EMPTY_STRING_BUFFER(setting->value))
		{
			/*
			 * Our GUC entry has a NULL (or empty) value. Skip the setting.
			 *
			 * In cases that's expected, such as when removing primary_conninfo
			 * from the recovery.conf settings so that we disconnect from the
			 * primary node being demoted.
			 *
			 * Still log about it, in case it might happen when it's not
			 * expected.
			 */
			log_debug("GUC setting \"%s\" has a NULL value", setting->name);
		}
		else
		{
			/* the GUC setting in the array has not been processed */
			log_error("BUG: GUC settings \"%s\" has not been processed",
					  setting->name);
			return false;
		}
	}

	if (includeTuning)
	{
		if (!pgtuning_prepare_guc_settings(postgres_tuning,
										   tuning,
										   sizeof(tuning)))
		{
			log_warn("Failed to compute Postgres basic tuning for this system");
		}

		appendPQExpBuffer(config, "\n%s\n", tuning);
	}

	/* memory allocation could have failed while building string */
	if (PQExpBufferBroken(config))
	{
		log_error("Failed to allocate memory while preparing config file \"%s\"",
				  configFilePath);
		destroyPQExpBuffer(config);
		return false;
	}

	return true;
}


bool
ensure_empty_tablespace_dirs(const char *pgdata)
{
	char *dirName = "pg_tblspc";
	struct dirent *dirEntry = NULL;

	char pgTblspcFullPath[MAXPGPATH] = { 0 };
	sformat(pgTblspcFullPath, MAXPGPATH, "%s/%s", pgdata, dirName);

	if (!directory_exists(pgTblspcFullPath))
	{
		log_debug("Postgres dir pg_tblspc does not exist at \"%s\"",
				  pgTblspcFullPath);
		return true;
	}


	/* open and scan through the Postgres tablespace directory */
	DIR *tblspcDir = opendir(pgTblspcFullPath);

	if (tblspcDir == NULL)
	{
		log_error("Failed to open Postgres tablespace directory \"%s\" at \"%s\"",
				  dirName, pgdata);
		return false;
	}

	while ((dirEntry = readdir(tblspcDir)) != NULL)
	{
		char tblspcDataDirPath[MAXPGPATH] = { 0 };

		/* skip non-symlinks, as all tablespace dirs are symlinks */
		if (dirEntry->d_type == DT_UNKNOWN)
		{
			struct stat structStat;
			char dirEntryFullPath[MAXPGPATH] = { 0 };
			sformat(dirEntryFullPath, MAXPGPATH, "%s/%s", pgTblspcFullPath,
					dirEntry->d_name);
			if (lstat(dirEntryFullPath, &structStat) != 0)
			{
				log_error("Failed to get file information for \"%s/%s\": %m",
						  pgTblspcFullPath, dirEntry->d_name);
				return false;
			}
			if (!S_ISLNK(structStat.st_mode))
			{
				log_debug("Non-symlink file found in tablespace directory: \"%s/%s\"",
						  pgTblspcFullPath, dirEntry->d_name);
				continue;
			}
		}
		else if (dirEntry->d_type != DT_LNK)
		{
			log_debug("Non-symlink file found in tablespace directory: \"%s/%s\"",
					  pgTblspcFullPath, dirEntry->d_name);
			continue;
		}

		join_path_components(tblspcDataDirPath, pgTblspcFullPath, dirEntry->d_name);

		log_debug("Removing contents of tablespace data directory \"%s\"",
				  tblspcDataDirPath);

		/* remove contents of tablespace data directory */
		if (!rmtree(tblspcDataDirPath, false))
		{
			log_error(
				"Failed to remove contents of existing tablespace data directory \"%s\": %m",
				tblspcDataDirPath);
			return false;
		}
	}

	return true;
}


/*
 * Call pg_basebackup, using a temporary directory for the duration of the data
 * transfer.
 */
bool
pg_basebackup(const char *pgdata,
			  const char *pg_ctl,
			  ReplicationSource *replicationSource)
{
	int returnCode;
	char pg_basebackup[MAXPGPATH];

	NodeAddress *primaryNode = &(replicationSource->primaryNode);
	char primaryConnInfo[MAXCONNINFO] = { 0 };

	char *args[16];
	int argsIndex = 0;

	char command[BUFSIZE];
	char pgpassword[BUFSIZE] = { 0 };

	log_debug("mkdir -p \"%s\"", replicationSource->backupDir);
	if (!ensure_empty_dir(replicationSource->backupDir, 0700))
	{
		/* errors have already been logged. */
		return false;
	}

	if (!ensure_empty_tablespace_dirs(pgdata))
	{
		/* errors have already been logged. */
		return false;
	}

	/* call pg_basebackup */
	path_in_same_directory(pg_ctl, "pg_basebackup", pg_basebackup);

	setenv("PGCONNECT_TIMEOUT", POSTGRES_CONNECT_TIMEOUT, 1);

	if (!IS_EMPTY_STRING_BUFFER(replicationSource->password))
	{
		if (env_exists("PGPASSWORD"))
		{
			if (!get_env_copy("PGPASSWORD", pgpassword, sizeof(pgpassword)))
			{
				/* errors have already been logged. */
				return false;
			}
		}
		setenv("PGPASSWORD", replicationSource->password, 1);
	}
	setenv("PGAPPNAME", replicationSource->applicationName, 1);

	if (!prepare_primary_conninfo(primaryConnInfo,
								  MAXCONNINFO,
								  primaryNode->host,
								  primaryNode->port,
								  replicationSource->userName,
								  NULL, /* no database */
								  NULL, /* no password here */
								  replicationSource->applicationName,
								  replicationSource->sslOptions,
								  false)) /* do not escape this one */
	{
		/* errors have already been logged. */
		return false;
	}

	args[argsIndex++] = (char *) pg_basebackup;
	args[argsIndex++] = "-w";
	args[argsIndex++] = "-d";
	args[argsIndex++] = primaryConnInfo;
	args[argsIndex++] = "--pgdata";
	args[argsIndex++] = replicationSource->backupDir;
	args[argsIndex++] = "-U";
	args[argsIndex++] = replicationSource->userName;
	args[argsIndex++] = "--verbose";
	args[argsIndex++] = "--progress";
	args[argsIndex++] = "--max-rate";
	args[argsIndex++] = replicationSource->maximumBackupRate;
	args[argsIndex++] = "--wal-method=stream";

	/* we don't use a replication slot e.g. when upstream is a standby */
	if (!IS_EMPTY_STRING_BUFFER(replicationSource->slotName))
	{
		args[argsIndex++] = "--slot";
		args[argsIndex++] = replicationSource->slotName;
	}

	args[argsIndex] = NULL;

	/*
	 * We do not want to call setsid() when running this program, as the
	 * pg_basebackup subprogram is not intended to be its own session leader,
	 * but remain a sub-process in the same group as pg_autoctl.
	 */
	Program program = { 0 };

	(void) initialize_program(&program, args, false);
	program.processBuffer = &processBufferCallback;

	/* log the exact command line we're using */
	int commandSize = snprintf_program_command_line(&program, command, BUFSIZE);

	if (commandSize >= BUFSIZE)
	{
		/* we only display the first BUFSIZE bytes of the real command */
		log_info("%s...", command);
	}
	else
	{
		log_info("%s", command);
	}

	(void) execute_subprogram(&program);

	/* clean-up the environment again */
	if (!IS_EMPTY_STRING_BUFFER(replicationSource->password))
	{
		if (IS_EMPTY_STRING_BUFFER(pgpassword))
		{
			unsetenv("PGPASSWORD");
		}
		else
		{
			setenv("PGPASSWORD", pgpassword, 1);
		}
	}

	returnCode = program.returnCode;
	free_program(&program);

	if (returnCode != 0)
	{
		log_error("Failed to run pg_basebackup: exit code %d", returnCode);
		return false;
	}

	/* replace $pgdata with the backup directory */
	if (directory_exists(pgdata))
	{
		if (!rmtree(pgdata, true))
		{
			log_error("Failed to remove directory \"%s\": %m", pgdata);
			return false;
		}
	}

	log_debug("mv \"%s\" \"%s\"", replicationSource->backupDir, pgdata);

	if (rename(replicationSource->backupDir, pgdata) != 0)
	{
		log_error(
			"Failed to install pg_basebackup dir " " \"%s\" in \"%s\": %m",
			replicationSource->backupDir, pgdata);
		return false;
	}

	return true;
}


/*
 * pg_rewind runs the pg_rewind program to rewind the given database directory
 * to a state where it can follow the given primary. We need the ability to
 * connect to the node.
 */
bool
pg_rewind(const char *pgdata,
		  const char *pg_ctl,
		  ReplicationSource *replicationSource)
{
	int returnCode;
	char pg_rewind[MAXPGPATH] = { 0 };

	NodeAddress *primaryNode = &(replicationSource->primaryNode);
	char primaryConnInfo[MAXCONNINFO] = { 0 };

	char *args[7];
	int argsIndex = 0;

	char command[BUFSIZE];
	char pgpassword[BUFSIZE] = { 0 };

	/* call pg_rewind*/
	path_in_same_directory(pg_ctl, "pg_rewind", pg_rewind);

	setenv("PGCONNECT_TIMEOUT", POSTGRES_CONNECT_TIMEOUT, 1);

	if (!IS_EMPTY_STRING_BUFFER(replicationSource->password))
	{
		if (env_exists("PGPASSWORD"))
		{
			if (!get_env_copy("PGPASSWORD", pgpassword, sizeof(pgpassword)))
			{
				/* errors have already been logged. */
				return false;
			}
		}
		setenv("PGPASSWORD", replicationSource->password, 1);
	}

	if (!prepare_primary_conninfo(primaryConnInfo,
								  MAXCONNINFO,
								  primaryNode->host,
								  primaryNode->port,
								  replicationSource->userName,
								  "postgres", /* pg_rewind needs a database */
								  NULL,       /* no password here */
								  replicationSource->applicationName,
								  replicationSource->sslOptions,
								  false)) /* do not escape this one */
	{
		/* errors have already been logged. */
		return false;
	}

	args[argsIndex++] = (char *) pg_rewind;
	args[argsIndex++] = "--target-pgdata";
	args[argsIndex++] = (char *) pgdata;
	args[argsIndex++] = "--source-server";
	args[argsIndex++] = primaryConnInfo;
	args[argsIndex++] = "--progress";
	args[argsIndex] = NULL;

	/*
	 * We do not want to call setsid() when running this program, as the
	 * pg_rewind subprogram is not intended to be its own session leader, but
	 * remain a sub-process in the same group as pg_autoctl.
	 */
	Program program = { 0 };

	(void) initialize_program(&program, args, false);
	program.processBuffer = &processBufferCallback;

	/* log the exact command line we're using */
	int commandSize = snprintf_program_command_line(&program, command, BUFSIZE);

	if (commandSize >= BUFSIZE)
	{
		/* we only display the first BUFSIZE bytes of the real command */
		log_info("%s...", command);
	}
	else
	{
		log_info("%s", command);
	}

	(void) execute_subprogram(&program);

	/* clean-up the environment again */
	if (!IS_EMPTY_STRING_BUFFER(replicationSource->password))
	{
		if (IS_EMPTY_STRING_BUFFER(pgpassword))
		{
			unsetenv("PGPASSWORD");
		}
		else
		{
			setenv("PGPASSWORD", pgpassword, 1);
		}
	}

	returnCode = program.returnCode;
	free_program(&program);

	if (returnCode != 0)
	{
		log_error("Failed to run pg_rewind: exit code %d", returnCode);
		return false;
	}

	return true;
}


/* log_program_output logs the output of the given program. */
static void
log_program_output(Program prog, int outLogLevel, int errorLogLevel)
{
	if (prog.stdOut != NULL)
	{
		char *outLines[BUFSIZE];
		int lineCount = splitLines(prog.stdOut, outLines, BUFSIZE);
		int lineNumber = 0;

		for (lineNumber = 0; lineNumber < lineCount; lineNumber++)
		{
			log_level(outLogLevel, "%s", outLines[lineNumber]);
		}
	}

	if (prog.stdErr != NULL)
	{
		char *errorLines[BUFSIZE];
		int lineCount = splitLines(prog.stdErr, errorLines, BUFSIZE);
		int lineNumber = 0;

		for (lineNumber = 0; lineNumber < lineCount; lineNumber++)
		{
			log_level(errorLogLevel, "%s", errorLines[lineNumber]);
		}
	}
}


/*
 * pg_ctl_initdb initializes a PostgreSQL directory from scratch by calling
 * "pg_ctl initdb", and returns true when this was successful. Beware that it
 * will inherit from the environment, such as LC_COLLATE and LC_ALL etc.
 *
 * No provision is made to control (sanitize?) that environment.
 */
bool
pg_ctl_initdb(const char *pg_ctl, const char *pgdata)
{
	/* initdb takes time, so log about the operation BEFORE doing it */
	log_info("Initialising a PostgreSQL cluster at \"%s\"", pgdata);
	log_info("%s initdb -s -D %s --option '--auth=trust'", pg_ctl, pgdata);

	Program program = run_program(pg_ctl, "initdb",
								  "--silent",
								  "--pgdata", pgdata,

	                              /* avoid warning message */
								  "--option", "'--auth=trust'",
								  NULL);

	bool success = program.returnCode == 0;

	if (program.returnCode != 0)
	{
		(void) log_program_output(program, LOG_INFO, LOG_ERROR);
		log_fatal("Failed to initialize Postgres cluster at \"%s\", "
				  "see above for details",
				  pgdata);
	}
	else
	{
		/* we might still have important information to read there */
		(void) log_program_output(program, LOG_INFO, LOG_WARN);
	}
	free_program(&program);

	return success;
}


/*
 * pg_ctl_postgres runs the "postgres" command-line in the current process,
 * with the same options as we would use in pg_ctl_start. pg_ctl_postgres does
 * not fork a Postgres process in the background, we keep the control over the
 * postmaster process. Think exec() rather then fork().
 *
 * This function will take over the current standard output and standard error
 * file descriptor, closing them and then giving control to them to Postgres
 * itself. This function is meant to be called in the child process of a fork()
 * call done by the caller.
 */
bool
pg_ctl_postgres(const char *pg_ctl, const char *pgdata, int pgport,
				char *listen_addresses, bool listen)
{
	char postgres[MAXPGPATH];
	char logfile[MAXPGPATH];

	char *args[12];
	int argsIndex = 0;

	char env_pg_regress_sock_dir[MAXPGPATH];

	char command[BUFSIZE];

	/* call postgres directly */
	path_in_same_directory(pg_ctl, "postgres", postgres);

	/* prepare startup.log file in PGDATA */
	join_path_components(logfile, pgdata, "startup.log");

	args[argsIndex++] = (char *) postgres;
	args[argsIndex++] = "-D";
	args[argsIndex++] = (char *) pgdata;
	args[argsIndex++] = "-p";
	args[argsIndex++] = (char *) intToString(pgport).strValue;

	if (listen)
	{
		if (IS_EMPTY_STRING_BUFFER(listen_addresses))
		{
			log_error("BUG: pg_ctl_postgres is given an empty listen_addresses "
					  "with argument listen set to true");
			return false;
		}
		args[argsIndex++] = "-h";
		args[argsIndex++] = (char *) listen_addresses;
	}
	else
	{
		args[argsIndex++] = "-h";
		args[argsIndex++] = "";
	}

	if (env_exists("PG_REGRESS_SOCK_DIR"))
	{
		if (!get_env_copy("PG_REGRESS_SOCK_DIR", env_pg_regress_sock_dir,
						  MAXPGPATH))
		{
			/* errors have already been logged */
			return false;
		}

		args[argsIndex++] = "-k";
		args[argsIndex++] = (char *) env_pg_regress_sock_dir;
	}

	args[argsIndex] = NULL;

	/*
	 * We do not want to call setsid() when running this program, as the
	 * postgres subprogram is not intended to be its own session leader, but
	 * remain a sub-process in the same group as pg_autoctl.
	 */
	Program program = { 0 };

	(void) initialize_program(&program, args, false);

	/* we want to redirect the output to logfile */
	int logFileDescriptor = open(logfile, FOPEN_FLAGS_W, 0644);

	if (logFileDescriptor == -1)
	{
		log_error("Failed to open file \"%s\": %m", logfile);
	}

	program.capture = false;    /* redirect output, don't capture */
	program.stdOutFd = logFileDescriptor;
	program.stdErrFd = logFileDescriptor;

	/* log the exact command line we're using */
	int commandSize = snprintf_program_command_line(&program, command, BUFSIZE);

	if (commandSize >= BUFSIZE)
	{
		/* we only display the first BUFSIZE bytes of the real command */
		log_info("%s...", command);
	}
	else
	{
		log_info("%s", command);
	}

	(void) execute_program(&program);

	return program.returnCode == 0;
}


/*
 * pg_log_startup logs the PGDATA/startup.log file contents so that our users
 * have enough information about why Postgres failed to start when that
 * happens.
 */
bool
pg_log_startup(const char *pgdata, int logLevel)
{
	char pgLogDirPath[MAXPGPATH] = { 0 };

	char pgStartupPath[MAXPGPATH] = { 0 };
	char *fileContents;
	long fileSize;

	/* logLevel to use when introducing which file path logs come from */
	int pathLogLevel = logLevel <= LOG_DEBUG ? LOG_DEBUG : LOG_WARN;

	struct stat pgStartupStat;

	struct dirent *logFileDirEntry = NULL;

	/* prepare startup.log file in PGDATA */
	join_path_components(pgStartupPath, pgdata, "startup.log");

	if (read_file(pgStartupPath, &fileContents, &fileSize) && fileSize > 0)
	{
		char *lines[BUFSIZE];
		int lineCount = splitLines(fileContents, lines, BUFSIZE);
		int lineNumber = 0;

		log_level(pathLogLevel, "Postgres logs from \"%s\":", pgStartupPath);

		for (lineNumber = 0; lineNumber < lineCount; lineNumber++)
		{
			log_level(logLevel, "%s", lines[lineNumber]);
		}

		free(fileContents);
	}

	/*
	 * Add in the most recent Postgres log file if it's been created after the
	 * startup.log file, it might contain very useful information, such as a
	 * FATAL line(s).
	 *
	 * Given that we setup Postgres to use the logging_collector, we expect
	 * there to be a single Postgres log file in the "log" directory that was
	 * created later than the "startup.log" file, and we expect the file to be
	 * rather short.
	 *
	 * Also we setup log_directory to be "log" so that's where we are looking
	 * into.
	 */

	/* prepare PGDATA/log directory path */
	join_path_components(pgLogDirPath, pgdata, "log");

	if (!directory_exists(pgLogDirPath))
	{
		/* then there's no other log files to process here */
		return true;
	}

	/* get the time of last modification of the startup.log file */
	if (lstat(pgStartupPath, &pgStartupStat) != 0)
	{
		log_error("Failed to get file information for \"%s\": %m",
				  pgStartupPath);
		return false;
	}
	int64_t pgStartupMtime = ST_MTIME_S(pgStartupStat);

	/* open and scan through the Postgres log directory */
	DIR *logDir = opendir(pgLogDirPath);

	if (logDir == NULL)
	{
		log_error("Failed to open Postgres log directory \"%s\": %m",
				  pgLogDirPath);
		return false;
	}

	while ((logFileDirEntry = readdir(logDir)) != NULL)
	{
		char pgLogFilePath[MAXPGPATH] = { 0 };
		struct stat pgLogFileStat;


		/* build the absolute file path for the logfile */
		join_path_components(pgLogFilePath,
							 pgLogDirPath,
							 logFileDirEntry->d_name);

		/* get the file information for the current logFile */
		if (lstat(pgLogFilePath, &pgLogFileStat) != 0)
		{
			log_error("Failed to get file information for \"%s\": %m",
					  pgLogFilePath);
			return false;
		}

		/*
		 * our logFiles are regular files, skip . and .. and others
		 * first, check for systems that do not handle d_type, and skip non-regular types
		 */
		if (logFileDirEntry->d_type == DT_UNKNOWN)
		{
			if (!S_ISREG(pgLogFileStat.st_mode))
			{
				continue;
			}
		}

		/*
		 * next, ignore all other non-regular types
		 * (if this check were first, we would skip all with DT_UNKNOWN)
		 */
		else if (logFileDirEntry->d_type != DT_REG)
		{
			continue;
		}

		int64_t pgLogFileMtime = ST_MTIME_S(pgLogFileStat);

		/*
		 * Compare modification times and only add to our logs the content
		 * from the Postgres log file that was created after the
		 * startup.log file.
		 */
		if (pgLogFileMtime >= pgStartupMtime)
		{
			log_level(pathLogLevel,
					  "Postgres logs from \"%s\":", pgLogFilePath);

			if (read_file(pgLogFilePath, &fileContents, &fileSize) &&
				fileSize > 0)
			{
				char *lines[BUFSIZE];
				int lineCount = splitLines(fileContents, lines, BUFSIZE);
				int lineNumber = 0;

				for (lineNumber = 0; lineNumber < lineCount; lineNumber++)
				{
					if (strstr(lines[lineNumber], "FATAL") != NULL)
					{
						log_fatal("%s", lines[lineNumber]);
					}
					else if (strstr(lines[lineNumber], "ERROR") != NULL)
					{
						log_error("%s", lines[lineNumber]);
					}
					else
					{
						log_level(logLevel, "%s", lines[lineNumber]);
					}
				}

				free(fileContents);
			}
		}
	}

	closedir(logDir);

	/* now add the contents of the recovery configuration */
	(void) pg_log_recovery_setup(pgdata, logLevel);

	return true;
}


/*
 * pg_log_recovery_setup logs the current Postgres recovery settings from
 * either the recovery.conf file or the standby setup. In case things go wrong
 * in the Postgres version detection mechanism, or upgrades, or clean-up, this
 * logs all the configuration files found rather than only those we expect we
 * should find.
 */
bool
pg_log_recovery_setup(const char *pgdata, int logLevel)
{
	char *filenames[] = {
		"recovery.conf",
		"standby.signal",
		AUTOCTL_STANDBY_CONF_FILENAME,
		NULL
	};

	for (int i = 0; filenames[i] != NULL; i++)
	{
		char recoveryConfPath[MAXPGPATH] = { 0 };
		char *fileContents;
		long fileSize;

		join_path_components(recoveryConfPath, pgdata, filenames[i]);

		if (file_exists(recoveryConfPath))
		{
			if (!read_file(recoveryConfPath, &fileContents, &fileSize))
			{
				/* errors have already been logged */
				continue;
			}

			if (fileSize > 0)
			{
				log_debug("Configuration file \"%s\":\n%s",
						  recoveryConfPath, fileContents);
			}
			else
			{
				log_debug("Configuration file \"%s\" is empty",
						  recoveryConfPath);
			}

			free(fileContents);
		}
	}

	return true;
}


/*
 * pg_ctl_stop tries to stop a PostgreSQL server by running a "pg_ctl stop"
 * command. If the server was stopped successfully, or if the server is not
 * running at all, it returns true.
 */
bool
pg_ctl_stop(const char *pg_ctl, const char *pgdata)
{
	const bool log_output = true;

	log_info("%s --pgdata %s --wait stop --mode fast", pg_ctl, pgdata);

	Program program = run_program(pg_ctl,
								  "--pgdata", pgdata,
								  "--wait",
								  "stop",
								  "--mode", "fast",
								  NULL);

	/*
	 * Case 1. "pg_ctl stop" was successful, so we could stop the PostgreSQL
	 * server successfully.
	 */
	if (program.returnCode == 0)
	{
		free_program(&program);
		return true;
	}

	/*
	 * Case 2. The data directory doesn't exist. So we assume PostgreSQL is
	 * not running, so stopping the PostgreSQL server was successful.
	 */
	bool pgdata_exists = directory_exists(pgdata);
	if (!pgdata_exists)
	{
		log_info("pgdata \"%s\" does not exist, consider this as PostgreSQL "
				 "not running", pgdata);
		free_program(&program);
		return true;
	}

	/*
	 * Case 3. "pg_ctl stop" returns non-zero return code when PostgreSQL is not
	 * running at all. So we double-check with "pg_ctl status", and return
	 * success if the PostgreSQL server is not running. Otherwise, we return
	 * failure.
	 *
	 * See https://www.postgresql.org/docs/current/static/app-pg-ctl.html
	 */

	int status = pg_ctl_status(pg_ctl, pgdata, log_output);
	if (status == PG_CTL_STATUS_NOT_RUNNING)
	{
		log_info("pg_ctl stop failed, but PostgreSQL is not running anyway");
		free_program(&program);
		return true;
	}

	log_info("Stopping PostgreSQL server failed. pg_ctl status returned: %d",
			 status);

	if (log_output)
	{
		(void) log_program_output(program, LOG_INFO, LOG_ERROR);
	}

	free_program(&program);
	return false;
}


/*
 * pg_ctl_status gets the status of the PostgreSQL server by running
 * "pg_ctl status". Output of this command is logged if log_output is true.
 * Return code of this command is returned.
 */
int
pg_ctl_status(const char *pg_ctl, const char *pgdata, bool log_output)
{
	Program program = run_program(pg_ctl, "status", "-D", pgdata, NULL);
	int returnCode = program.returnCode;

	log_level(log_output ? LOG_INFO : LOG_DEBUG,
			  "%s status -D %s [%d]", pg_ctl, pgdata, returnCode);

	if (log_output)
	{
		(void) log_program_output(program, LOG_INFO, LOG_ERROR);
	}

	free_program(&program);
	return returnCode;
}


/*
 * pg_ctl_promote promotes a standby by running "pg_ctl promote"
 */
bool
pg_ctl_promote(const char *pg_ctl, const char *pgdata)
{
	Program program =
		run_program(pg_ctl, "promote", "-D", pgdata, "--no-wait", NULL);
	int returnCode = program.returnCode;

	log_debug("%s promote -D %s --no-wait", pg_ctl, pgdata);

	if (program.stdErr != NULL)
	{
		log_error("%s", program.stdErr);
	}

	if (returnCode != 0)
	{
		/* pg_ctl promote will have logged errors */
		free_program(&program);
		return false;
	}

	free_program(&program);
	return true;
}


/*
 * pg_setup_standby_mode sets up standby mode by either writing a recovery.conf
 * file or adding the configuration items to postgresql.conf and then creating
 * a standby.signal file in PGDATA.
 */
bool
pg_setup_standby_mode(uint32_t pg_control_version,
					  const char *pgdata,
					  const char *pg_ctl,
					  ReplicationSource *replicationSource)
{
	if (pg_control_version < 1000)
	{
		log_fatal("pg_auto_failover does not support PostgreSQL before "
				  "Postgres 10, we have pg_control version number %d from "
				  "pg_controldata \"%s\"",
				  pg_control_version, pgdata);
		return false;
	}

	/*
	 * Check our primary_conninfo connection string by attempting to connect in
	 * replication mode and issuing a IDENTIFY_SYSTEM command.
	 */
	if (!IS_EMPTY_STRING_BUFFER(replicationSource->primaryNode.host) &&
		!pgctl_identify_system(replicationSource))
	{
		log_error("Failed to setup standby mode: can't connect to the primary. "
				  "See above for details");
		return false;
	}

	if (pg_control_version < 1200)
	{
		/*
		 * Before Postgres 12 we used to place recovery configuration in a
		 * specific file recovery.conf, located alongside postgresql.conf.
		 * Controling whether the server would start in PITR or standby mode
		 * was controlled by a setting in the recovery.conf file.
		 */
		return pg_write_recovery_conf(pgdata, replicationSource);
	}
	else
	{
		/*
		 * Starting in Postgres 12 we need to add our recovery configuration to
		 * the main postgresql.conf file and create an empty standby.signal
		 * file to trigger starting the server in standby mode.
		 */
		return pg_write_standby_signal(pgdata, replicationSource);
	}
}


/*
 * pg_write_recovery_conf writes a recovery.conf file to a postgres data
 * directory with the given primary connection info and replication slot name.
 */
static bool
pg_write_recovery_conf(const char *pgdata, ReplicationSource *replicationSource)
{
	char recoveryConfPath[MAXPGPATH];

	/* prepare storage areas for parameters */
	char primaryConnInfo[MAXCONNINFO] = { 0 };
	char primarySlotName[MAXCONNINFO] = { 0 };
	char targetLSN[PG_LSN_MAXLENGTH] = { 0 };
	char targetAction[NAMEDATALEN] = { 0 };
	char targetTimeline[NAMEDATALEN] = { 0 };

	GUC recoverySettingsStandby[] = {
		{ "standby_mode", "'on'" },
		{ "primary_conninfo", (char *) primaryConnInfo },
		{ "primary_slot_name", (char *) primarySlotName },
		{ "recovery_target_timeline", (char *) targetTimeline },
		{ NULL, NULL }
	};

	GUC recoverySettingsTargetLSN[] = {
		{ "standby_mode", "'on'" },
		{ "primary_conninfo", (char *) primaryConnInfo },
		{ "primary_slot_name", (char *) primarySlotName },
		{ "recovery_target_timeline", (char *) targetTimeline },
		{ "recovery_target_lsn", (char *) targetLSN },
		{ "recovery_target_inclusive", "'true'" },
		{ "recovery_target_action", (char *) targetAction },
		{ NULL, NULL }
	};

	GUC *recoverySettings =
		IS_EMPTY_STRING_BUFFER(replicationSource->targetLSN)
		? recoverySettingsStandby
		: recoverySettingsTargetLSN;

	bool includeTuning = false;

	join_path_components(recoveryConfPath, pgdata, "recovery.conf");

	log_info("Writing recovery configuration to \"%s\"", recoveryConfPath);

	if (!prepare_recovery_settings(pgdata,
								   replicationSource,
								   primaryConnInfo,
								   primarySlotName,
								   targetLSN,
								   targetAction,
								   targetTimeline))
	{
		/* errors have already been logged */
		return false;
	}

	return ensure_default_settings_file_exists(recoveryConfPath,
											   recoverySettings,
											   NULL,
											   NULL,
											   includeTuning);
}


/*
 * pg_write_standby_signal writes the ${PGDATA}/standby.signal file that is in
 * use starting with Postgres 12 for starting a standby server. The file only
 * needs to exists, and the setup is to be found in the main Postgres
 * configuration file.
 */
static bool
pg_write_standby_signal(const char *pgdata,
						ReplicationSource *replicationSource)
{
	char standbyConfigFilePath[MAXPGPATH] = { 0 };
	char signalFilePath[MAXPGPATH] = { 0 };
	char configFilePath[MAXPGPATH] = { 0 };

	/* prepare storage areas for parameters */
	char primaryConnInfo[MAXCONNINFO] = { 0 };
	char primarySlotName[MAXCONNINFO] = { 0 };
	char targetLSN[PG_LSN_MAXLENGTH] = { 0 };
	char targetAction[NAMEDATALEN] = { 0 };
	char targetTimeline[NAMEDATALEN] = { 0 };

	GUC recoverySettingsStandby[] = {
		{ "primary_conninfo", (char *) primaryConnInfo },
		{ "primary_slot_name", (char *) primarySlotName },
		{ "recovery_target_timeline", (char *) targetTimeline },
		{ NULL, NULL }
	};

	GUC recoverySettingsTargetLSN[] = {
		{ "primary_conninfo", (char *) primaryConnInfo },
		{ "primary_slot_name", (char *) primarySlotName },
		{ "recovery_target_timeline", (char *) targetTimeline },
		{ "recovery_target_lsn", (char *) targetLSN },
		{ "recovery_target_inclusive", "'true'" },
		{ "recovery_target_action", targetAction },
		{ NULL, NULL }
	};

	GUC *recoverySettings =
		IS_EMPTY_STRING_BUFFER(replicationSource->targetLSN)
		? recoverySettingsStandby
		: recoverySettingsTargetLSN;

	bool includeTuning = false;

	log_trace("pg_write_standby_signal");

	if (!prepare_recovery_settings(pgdata,
								   replicationSource,
								   primaryConnInfo,
								   primarySlotName,
								   targetLSN,
								   targetAction,
								   targetTimeline))
	{
		/* errors have already been logged */
		return false;
	}

	/* set our configuration file paths, all found in PGDATA */
	join_path_components(signalFilePath, pgdata, "standby.signal");
	join_path_components(configFilePath, pgdata, "postgresql.conf");
	join_path_components(standbyConfigFilePath,
						 pgdata,
						 AUTOCTL_STANDBY_CONF_FILENAME);

	/*
	 * First install the standby.signal file, so that if there's a problem
	 * later and Postgres is started, it is started as a standby, with missing
	 * configuration.
	 */

	/* only logs about this the first time */
	if (!file_exists(signalFilePath))
	{
		log_info("Creating the standby signal file at \"%s\", "
				 "and replication setup at \"%s\"",
				 signalFilePath, standbyConfigFilePath);
	}

	if (!write_file("", 0, signalFilePath))
	{
		/* write_file logs I/O error */
		return false;
	}

	/*
	 * Now write the standby settings to postgresql-auto-failover-standby.conf
	 * and include that file from postgresql.conf.
	 *
	 * we pass NULL as pgSetup because we know it won't be used...
	 */
	if (!ensure_default_settings_file_exists(standbyConfigFilePath,
											 recoverySettings,
											 NULL,
											 NULL,
											 includeTuning))
	{
		return false;
	}

	/*
	 * We successfully created the standby.signal file, so Postgres will start
	 * as a standby. If we fail to install the standby settings, then we return
	 * false here and let the main loop try again. At least Postgres won't
	 * start as a cloned single accepting writes.
	 */
	if (!pg_include_config(configFilePath,
						   AUTOCTL_SB_CONF_INCLUDE_LINE,
						   AUTOCTL_CONF_INCLUDE_COMMENT))
	{
		log_error("Failed to prepare \"%s\" with standby settings",
				  standbyConfigFilePath);
		return false;
	}

	return true;
}


/*
 * prepare_recovery_settings prepares the settings that we need to install in
 * either recovery.conf or our own postgresql-auto-failover-standby.conf
 * depending on the Postgres major version.
 */
static bool
prepare_recovery_settings(const char *pgdata,
						  ReplicationSource *replicationSource,
						  char *primaryConnInfo,
						  char *primarySlotName,
						  char *targetLSN,
						  char *targetAction,
						  char *targetTimeline)
{
	bool escape = true;
	NodeAddress *primaryNode = &(replicationSource->primaryNode);

	/* when reaching REPORT_LSN we set recovery with no primary conninfo */
	if (!IS_EMPTY_STRING_BUFFER(primaryNode->host))
	{
		log_debug("prepare_recovery_settings: "
				  "primary node %" PRId64 " \"%s\" (%s:%d)",
				  primaryNode->nodeId,
				  primaryNode->name,
				  primaryNode->host,
				  primaryNode->port);

		if (!prepare_primary_conninfo(primaryConnInfo,
									  MAXCONNINFO,
									  primaryNode->host,
									  primaryNode->port,
									  replicationSource->userName,
									  NULL, /* no database */
									  replicationSource->password,
									  replicationSource->applicationName,
									  replicationSource->sslOptions,
									  escape))
		{
			/* errors have already been logged. */
			return false;
		}
	}
	else
	{
		log_debug("prepare_recovery_settings: no primary node!");
	}

	/*
	 * We don't always have a replication slot name to use when connecting to a
	 * standby node.
	 */
	if (!IS_EMPTY_STRING_BUFFER(replicationSource->slotName))
	{
		sformat(primarySlotName, MAXCONNINFO, "'%s'",
				replicationSource->slotName);
	}

	/* The default target timeline is 'latest' */
	if (IS_EMPTY_STRING_BUFFER(replicationSource->targetTimeline))
	{
		sformat(targetTimeline, NAMEDATALEN, "'latest'");
	}
	else
	{
		sformat(targetTimeline, NAMEDATALEN, "'%s'",
				replicationSource->targetTimeline);
	}

	/* We use the targetLSN only when doing a WAL fast_forward operation */
	if (!IS_EMPTY_STRING_BUFFER(replicationSource->targetLSN))
	{
		sformat(targetLSN, PG_LSN_MAXLENGTH, "'%s'",
				replicationSource->targetLSN);
	}

	/* The default target Action is 'pause' */
	if (IS_EMPTY_STRING_BUFFER(replicationSource->targetAction))
	{
		sformat(targetAction, NAMEDATALEN, "'pause'");
	}
	else
	{
		sformat(targetAction, NAMEDATALEN, "'%s'",
				replicationSource->targetAction);
	}

	return true;
}


/*
 * pg_cleanup_standby_mode cleans-up the replication settings for the local
 * instance of Postgres found at pgdata.
 *
 *  - remove either recovery.conf or standby.signal
 *
 *  - when using Postgres 12 also make postgresql-auto-failover-standby.conf an
 *    empty file, so that we can still include it, but it has no effect.
 */
bool
pg_cleanup_standby_mode(uint32_t pg_control_version,
						const char *pg_ctl,
						const char *pgdata,
						PGSQL *pgsql)
{
	if (pg_control_version < 1200)
	{
		char recoveryConfPath[MAXPGPATH];

		join_path_components(recoveryConfPath, pgdata, "recovery.conf");

		log_debug("pg_cleanup_standby_mode: rm \"%s\"", recoveryConfPath);

		if (!unlink_file(recoveryConfPath))
		{
			/* errors have already been logged */
			return false;
		}
	}
	else
	{
		char standbyConfigFilePath[MAXPGPATH];
		char signalFilePath[MAXPGPATH];

		join_path_components(signalFilePath, pgdata, "standby.signal");
		join_path_components(standbyConfigFilePath,
							 pgdata,
							 AUTOCTL_STANDBY_CONF_FILENAME);

		log_debug("pg_cleanup_standby_mode: rm \"%s\"", signalFilePath);

		if (!unlink_file(signalFilePath))
		{
			/* errors have already been logged */
			return false;
		}

		/* empty out the standby configuration file */
		log_debug("pg_cleanup_standby_mode: > \"%s\"", standbyConfigFilePath);

		if (!write_file("", 0, standbyConfigFilePath))
		{
			/* write_file logs I/O error */
			return false;
		}
	}

	return true;
}


/*
 * escape_recovery_conf_string escapes a string that is used in a recovery.conf
 * file by converting single quotes into two single quotes.
 *
 * The result is written to destination and the length of the result.
 */
static bool
escape_recovery_conf_string(char *destination, int destinationSize,
							const char *recoveryConfString)
{
	int charIndex = 0;
	int length = strlen(recoveryConfString);
	int escapedStringLength = 0;

	/* we are going to add at least 3 chars: two quotes and a NUL character */
	if (destinationSize < (length + 3))
	{
		log_error("BUG: failed to escape recovery parameter value \"%s\" "
				  "in a buffer of %d bytes",
				  recoveryConfString, destinationSize);
		return false;
	}

	destination[escapedStringLength++] = '\'';

	for (charIndex = 0; charIndex < length; charIndex++)
	{
		char currentChar = recoveryConfString[charIndex];

		if (currentChar == '\'')
		{
			destination[escapedStringLength++] = '\'';
			if (destinationSize < escapedStringLength)
			{
				log_error(
					"BUG: failed to escape recovery parameter value \"%s\" "
					"in a buffer of %d bytes, stopped at index %d",
					recoveryConfString, destinationSize, charIndex);
				return false;
			}
		}

		destination[escapedStringLength++] = currentChar;
		if (destinationSize < escapedStringLength)
		{
			log_error("BUG: failed to escape recovery parameter value \"%s\" "
					  "in a buffer of %d bytes, stopped at index %d",
					  recoveryConfString, destinationSize, charIndex);
			return false;
		}
	}

	destination[escapedStringLength++] = '\'';
	destination[escapedStringLength] = '\0';

	return true;
}


/*
 * prepare_primary_conninfo prepares a connection string to the primary server.
 * The connection string may be used unquoted in a command line calling either
 * pg_basebackup ro pg_rewind, or may be used quoted in the primary_conninfo
 * setting for PostgreSQL.
 *
 * Also, pg_rewind needs a database to connect to.
 */
static bool
prepare_primary_conninfo(char *primaryConnInfo,
						 int primaryConnInfoSize,
						 const char *primaryHost,
						 int primaryPort,
						 const char *replicationUsername,
						 const char *dbname,
						 const char *replicationPassword,
						 const char *applicationName,
						 SSLOptions sslOptions,
						 bool escape)
{
	int size = 0;
	char escaped[BUFSIZE];

	if (IS_EMPTY_STRING_BUFFER(primaryHost))
	{
		log_debug("prepare_primary_conninfo: missing primary hostname");

		bzero((void *) primaryConnInfo, primaryConnInfoSize);

		return true;
	}

	PQExpBuffer buffer = createPQExpBuffer();

	if (buffer == NULL)
	{
		log_error("Failed to allocate memory");
		return false;
	}

	/* application_name shows up in pg_stat_replication on the primary */
	appendPQExpBuffer(buffer, "application_name=%s", applicationName);
	appendPQExpBuffer(buffer, " host=%s", primaryHost);
	appendPQExpBuffer(buffer, " port=%d", primaryPort);
	appendPQExpBuffer(buffer, " user=%s", replicationUsername);

	if (dbname != NULL)
	{
		appendPQExpBuffer(buffer, " dbname=%s", dbname);
	}

	if (replicationPassword != NULL && !IS_EMPTY_STRING_BUFFER(replicationPassword))
	{
		appendPQExpBuffer(buffer, " password=%s", replicationPassword);
	}

	appendPQExpBufferStr(buffer, " ");
	if (!prepare_conninfo_sslmode(buffer, sslOptions))
	{
		/* errors have already been logged */
		destroyPQExpBuffer(buffer);
		return false;
	}

	/* memory allocation could have failed while building string */
	if (PQExpBufferBroken(buffer))
	{
		log_error("Failed to allocate memory");
		destroyPQExpBuffer(buffer);
		return false;
	}

	if (escape)
	{
		if (!escape_recovery_conf_string(escaped, BUFSIZE, buffer->data))
		{
			/* errors have already been logged. */
			destroyPQExpBuffer(buffer);
			return false;
		}

		/* now copy the buffer into primaryConnInfo for the caller */
		size = sformat(primaryConnInfo, primaryConnInfoSize, "%s", escaped);

		if (size == -1 || size > primaryConnInfoSize)
		{
			log_error("BUG: the escaped primary_conninfo requires %d bytes and "
					  "pg_auto_failover only support up to %d bytes",
					  size, primaryConnInfoSize);
			destroyPQExpBuffer(buffer);
			return false;
		}
	}
	else
	{
		strlcpy(primaryConnInfo, buffer->data, primaryConnInfoSize);
	}

	destroyPQExpBuffer(buffer);

	return true;
}


/*
 * prepare_conninfo_sslmode adds the sslmode setting to the buffer, which is
 * used as a connection string.
 */
static bool
prepare_conninfo_sslmode(PQExpBuffer buffer, SSLOptions sslOptions)
{
	if (sslOptions.sslMode == SSL_MODE_UNKNOWN)
	{
		if (sslOptions.active)
		{
			/* that's a bug really */
			log_error("SSL is active in the configuration, "
					  "but sslmode is unknown");
			return false;
		}

		return true;
	}

	appendPQExpBuffer(buffer, "sslmode=%s",
					  pgsetup_sslmode_to_string(sslOptions.sslMode));

	if (sslOptions.sslMode >= SSL_MODE_VERIFY_CA)
	{
		/* ssl revocation list might not be provided, it's ok */
		if (!IS_EMPTY_STRING_BUFFER(sslOptions.crlFile))
		{
			appendPQExpBuffer(buffer, " sslrootcert=%s sslcrl=%s",
							  sslOptions.caFile, sslOptions.crlFile);
		}
		else
		{
			appendPQExpBuffer(buffer, " sslrootcert=%s", sslOptions.caFile);
		}
	}

	return true;
}


/*
 * pgctl_identify_system connects with replication=1 to our target node and run
 * the IDENTIFY_SYSTEM command to check that HBA is ready.
 */
bool
pgctl_identify_system(ReplicationSource *replicationSource)
{
	NodeAddress *primaryNode = &(replicationSource->primaryNode);

	char primaryConnInfo[MAXCONNINFO] = { 0 };
	char primaryConnInfoReplication[MAXCONNINFO] = { 0 };
	PGSQL replicationClient = { 0 };

	if (!prepare_primary_conninfo(primaryConnInfo,
								  MAXCONNINFO,
								  primaryNode->host,
								  primaryNode->port,
								  replicationSource->userName,
								  NULL, /* no database */
								  replicationSource->password,
								  replicationSource->applicationName,
								  replicationSource->sslOptions,
								  false)) /* no need for escaping */
	{
		/* errors have already been logged. */
		return false;
	}

	/*
	 * Per https://www.postgresql.org/docs/12/protocol-replication.html:
	 *
	 * To initiate streaming replication, the frontend sends the replication
	 * parameter in the startup message. A Boolean value of true (or on, yes,
	 * 1) tells the backend to go into physical replication walsender mode,
	 * wherein a small set of replication commands, shown below, can be issued
	 * instead of SQL statements.
	 */
	int len = sformat(primaryConnInfoReplication, MAXCONNINFO,
					  "%s replication=1",
					  primaryConnInfo);

	if (len >= MAXCONNINFO)
	{
		log_warn("Failed to call IDENTIFY_SYSTEM: primary_conninfo too large");
		return false;
	}

	if (!pgsql_init(&replicationClient,
					primaryConnInfoReplication,
					PGSQL_CONN_UPSTREAM))
	{
		/* errors have already been logged */
		return false;
	}

	if (!pgsql_identify_system(&replicationClient,
							   &(replicationSource->system)))
	{
		/* errors have already been logged */
		return false;
	}

	return true;
}


/*
 * pg_is_running returns true if PostgreSQL is running.
 */
bool
pg_is_running(const char *pg_ctl, const char *pgdata)
{
	return pg_ctl_status(pg_ctl, pgdata, false) == 0;
}


/*
 * pg_create_self_signed_cert creates self-signed certificates for the local
 * Postgres server and places the private key in $PGDATA/server.key and the
 * public certificate in $PGDATA/server.cert
 *
 * We simply follow Postgres documentation at:
 * https://www.postgresql.org/docs/current/ssl-tcp.html#SSL-CERTIFICATE-CREATION
 *
 * openssl req -new -x509 -days 365 -nodes -text -out server.crt \
 *             -keyout server.key -subj "/CN=dbhost.yourdomain.com"
 */
bool
pg_create_self_signed_cert(PostgresSetup *pgSetup, const char *hostname)
{
	char subject[BUFSIZE] = { 0 };
	char openssl[MAXPGPATH] = { 0 };

	if (!search_path_first("openssl", openssl, LOG_ERROR))
	{
		/* errors have already been logged */
		return false;
	}

	/* ensure PGDATA has been normalized */
	if (!normalize_filename(pgSetup->pgdata, pgSetup->pgdata, MAXPGPATH))
	{
		return false;
	}

	int size = sformat(pgSetup->ssl.serverKey, MAXPGPATH,
					   "%s/server.key", pgSetup->pgdata);

	if (size == -1 || size > MAXPGPATH)
	{
		log_error("BUG: the ssl server key file path requires %d bytes and "
				  "pg_auto_failover only support up to %d bytes",
				  size, MAXPGPATH);
		return false;
	}

	size = sformat(pgSetup->ssl.serverCert, MAXPGPATH,
				   "%s/server.crt", pgSetup->pgdata);

	if (size == -1 || size > MAXPGPATH)
	{
		log_error("BUG: the ssl server key file path requires %d bytes and "
				  "pg_auto_failover only support up to %d bytes",
				  size, MAXPGPATH);
		return false;
	}

	size = sformat(subject, BUFSIZE, "/CN=%s", hostname);

	if (size == -1 || size > BUFSIZE)
	{
		log_error("BUG: the ssl subject \"/CN=%s\" requires %d bytes and"
				  "pg_auto_failover only support up to %d bytes",
				  hostname, size, BUFSIZE);
		return false;
	}

	log_info(" %s req -new -x509 -days 365 -nodes -text "
			 "-out %s -keyout %s -subj \"%s\"",
			 openssl,
			 pgSetup->ssl.serverCert,
			 pgSetup->ssl.serverKey,
			 subject);

	Program program = run_program(openssl,
								  "req", "-new", "-x509", "-days", "365",
								  "-nodes", "-text",
								  "-out", pgSetup->ssl.serverCert,
								  "-keyout", pgSetup->ssl.serverKey,
								  "-subj", subject,
								  NULL);

	if (program.returnCode != 0)
	{
		(void) log_program_output(program, LOG_INFO, LOG_ERROR);
		log_error("openssl failed with return code: %d", program.returnCode);
		free_program(&program);
		return false;
	}

	(void) log_program_output(program, LOG_DEBUG, LOG_DEBUG);
	free_program(&program);

	/*
	 * Then do: chmod og-rwx server.key
	 */
	if (chmod(pgSetup->ssl.serverKey, S_IRUSR | S_IWUSR) != 0)
	{
		log_error("Failed to chmod og-rwx \"%s\": %m", pgSetup->ssl.serverKey);
		return false;
	}

	return true;
}
