/*
 * Copyright 2010-2011 Red Hat, Inc.
 *
 * This copyrighted material is made available to anyone wishing to use,
 * modify, copy, or redistribute it subject to the terms and conditions
 * of the GNU General Public License v2 or (at your option) any later version.
 */

#include <inttypes.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <stddef.h>
#include <fcntl.h>
#include <string.h>
#include <errno.h>
#include <limits.h>
#include <pthread.h>
#include <time.h>
#include <syslog.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/mman.h>

#include "sanlock_internal.h"
#include "sanlock.h"
#include "sanlock_admin.h"
#include "diskio.h"
#include "ondisk.h"
#include "direct.h"
#include "log.h"
#include "paxos_lease.h"
#include "delta_lease.h"
#include "timeouts.h"

/* Based on "Light-Weight Leases for Storage-Centric Coordination"
   by Gregory Chockler and Dahlia Malkhi */

/* delta_leases are a series max_hosts leader_records, one leader per sector,
   host N's delta_lease is the leader_record in sectors N-1 */

/*
 * variable names:
 * rv: success is 0, failure is < 0
 * error: success is 1 (SANLK_OK), failure is < 0
 */

static void log_leader_error(int result,
			     char *space_name,
			     uint64_t host_id,
			     struct sync_disk *disk,
			     struct leader_record *lr,
			     const char *caller)
{
	log_error("leader1 %s error %d lockspace %.48s host_id %llu",
		  caller ? caller : "unknown",
		  result,
		  space_name,
		  (unsigned long long)host_id);

	log_error("leader2 path %s offset %llu",
		  disk->path,
		  (unsigned long long)disk->offset);

	log_error("leader3 m %x v %x ss %u nh %llu mh %llu oi %llu og %llu lv %llu",
		  lr->magic,
		  lr->version,
		  lr->sector_size,
		  (unsigned long long)lr->num_hosts,
		  (unsigned long long)lr->max_hosts,
		  (unsigned long long)lr->owner_id,
		  (unsigned long long)lr->owner_generation,
		  (unsigned long long)lr->lver);

	log_error("leader4 sn %.48s rn %.48s ts %llu cs %x",
		  lr->space_name,
		  lr->resource_name,
		  (unsigned long long)lr->timestamp,
		  lr->checksum);
}

static int verify_leader(struct sync_disk *disk,
			 char *space_name,
			 uint64_t host_id,
			 struct leader_record *lr,
			 uint32_t checksum,
			 const char *caller)
{
	int result;

	if (lr->magic != DELTA_DISK_MAGIC) {
		log_error("verify_leader %llu wrong magic %x %s",
			  (unsigned long long)host_id,
			  lr->magic, disk->path);
		result = SANLK_LEADER_MAGIC;
		goto fail;
	}

	if ((lr->version & 0xFFFF0000) > DELTA_DISK_VERSION_MAJOR) {
		log_error("verify_leader %llu newer major version leader 0x%x local 0x%x %s",
			  (unsigned long long)host_id,
			  lr->version,
			  DELTA_DISK_VERSION_MAJOR|DELTA_DISK_VERSION_MINOR,
			  disk->path);
		result = SANLK_LEADER_VERSION;
		goto fail;
	}

	if (strncmp(lr->space_name, space_name, NAME_ID_SIZE)) {
		log_error("verify_leader %llu wrong space name %.48s %.48s %s",
			  (unsigned long long)host_id,
			  lr->space_name, space_name, disk->path);
		result = SANLK_LEADER_LOCKSPACE;
		goto fail;
	}

	if (lr->checksum != checksum) {
		log_error("verify_leader %llu wrong checksum %x %x %s",
			  (unsigned long long)host_id,
			  lr->checksum, checksum, disk->path);
		result = SANLK_LEADER_CHECKSUM;
		goto fail;
	}

	return SANLK_OK;

 fail:
	log_leader_error(result, space_name, host_id, disk, lr, caller);
	return result;
}

int delta_read_lockspace(struct task *task,
			 struct sync_disk *disk,
			 int sector_size,
			 uint64_t host_id,
			 struct sanlk_lockspace *ls,
			 int io_timeout,
			 struct sanlk_host *hs)
{
	struct leader_record leader_end;
	struct leader_record leader;
	uint32_t checksum;
	char *space_name;
	int align_size;
	int rv, error;

	/* host_id N is block offset N-1 */

	memset(&leader_end, 0, sizeof(struct leader_record));

	rv = read_sector(disk, sector_size, host_id - 1, (char *)&leader_end, sizeof(struct leader_record),
			 task, task->use_aio, io_timeout, "read_lockspace");
	if (rv < 0)
		return rv;

	/* N.B. compute checksum before byte swapping */
	checksum = leader_checksum(&leader_end);

	leader_record_in(&leader_end, &leader);

	if (!ls->name[0])
		space_name = leader.space_name;
	else
		space_name = ls->name;

	error = verify_leader(disk, space_name, host_id, &leader, checksum, "read_lockspace");

	if (error == SANLK_OK) {
		memcpy(ls->name, leader.space_name, SANLK_NAME_LEN);
		ls->host_id = host_id;

		align_size = leader_align_size_from_flag(leader.flags);
		if (!align_size)
			align_size = sector_size_to_align_size_old(leader.sector_size);

		/* The flags set by the user may not have been correct. */
		sanlk_lsf_sector_flags_clear(&ls->flags);
		sanlk_lsf_align_flags_clear(&ls->flags);

		ls->flags |= sanlk_lsf_sector_size_to_flag(leader.sector_size);
		ls->flags |= sanlk_lsf_align_size_to_flag(align_size);

		if (leader.flags & LFL_NO_TIMEOUT)
			ls->flags |= SANLK_LSF_NO_TIMEOUT;

		if (leader.flags & LFL_CAW)
			ls->flags |= SANLK_LSF_USING_CAW;

		if (hs) {
			hs->host_id = leader.owner_id;
			hs->generation = leader.owner_generation;
			hs->timestamp = leader.timestamp;
			hs->io_timeout = leader.io_timeout;
			if (leader.flags & LFL_DEAD_EXT)
				hs->flags |= SANLK_HOST_DEAD_EXT;
		}
	}

	return error;
}

int delta_read_lockspace_sizes(struct task *task,
			 struct sync_disk *disk,
			 uint64_t host_id,
			 int use_aio,
			 int io_timeout,
			 int *sector_size,
			 int *align_size,
			 int *is_caw)
{
	struct leader_record leader_end;
	struct leader_record leader;
	int rv;

	memset(&leader_end, 0, sizeof(struct leader_record));

	/* With no host_id, get sizes from any leader record. */
	if (!host_id)
		host_id = 1;

	rv = read_sector(disk, 512, host_id - 1, (char *)&leader_end, sizeof(struct leader_record),
			 task, use_aio, io_timeout, "read_lockspace_sector_size");
	if (rv < 0)
		goto next;

	leader_record_in(&leader_end, &leader);

	if (leader.magic != DELTA_DISK_MAGIC)
		goto next;

	if ((leader.version & 0xFFFF0000) > DELTA_DISK_VERSION_MAJOR)
		goto next;

	if (leader.owner_id && (leader.owner_id != host_id))
		goto next;

	if (leader.sector_size != 512)
		return SANLK_LEADER_SECTORSIZE;

	*sector_size = leader.sector_size;

	*align_size = leader_align_size_from_flag(leader.flags);
	if (!*align_size)
		*align_size = sector_size_to_align_size_old(leader.sector_size);

	if (leader.flags & LFL_CAW)
		*is_caw = 1;

	return SANLK_OK;

 next:
	rv = read_sector(disk, 4096, host_id - 1, (char *)&leader_end, sizeof(struct leader_record),
			 task, use_aio, io_timeout, "read_lockspace_sector_size");
	if (rv < 0)
		return rv;

	leader_record_in(&leader_end, &leader);

	if (leader.magic != DELTA_DISK_MAGIC)
		return SANLK_LEADER_MAGIC;

	if ((leader.version & 0xFFFF0000) > DELTA_DISK_VERSION_MAJOR)
		return SANLK_LEADER_VERSION;

	if (leader.owner_id && (leader.owner_id != host_id))
		return SANLK_LEADER_DIFF;

	if (leader.sector_size != 4096)
		return SANLK_LEADER_SECTORSIZE;

	*sector_size = leader.sector_size;

	*align_size = leader_align_size_from_flag(leader.flags);
	if (!*align_size)
		*align_size = sector_size_to_align_size_old(leader.sector_size);

	if (leader.flags & LFL_CAW)
		*is_caw = 1;

	return SANLK_OK;
}

int delta_lease_leader_read(struct task *task, int sector_size, int io_timeout,
			    struct sync_disk *disk,
			    char *space_name,
			    uint64_t host_id,
			    struct leader_record *leader_ret,
			    const char *caller)
{
	struct leader_record leader_end;
	struct leader_record leader;
	uint32_t checksum;
	int rv, error;

	if (!sector_size) {
		log_error("delta_lease_leader_read with zero sector_size %s", space_name);
		return -EINVAL;
	}

	/* host_id N is block offset N-1 */

	memset(&leader_end, 0, sizeof(struct leader_record));
	memset(leader_ret, 0, sizeof(struct leader_record));

	rv = read_sector(disk, sector_size, host_id - 1, (char *)&leader_end, sizeof(struct leader_record),
			 task, task->use_aio, io_timeout, "delta_leader");
	if (rv < 0)
		return rv;

	/* N.B. compute checksum before byte swapping */
	checksum = leader_checksum(&leader_end);

	leader_record_in(&leader_end, &leader);

	error = verify_leader(disk, space_name, host_id, &leader, checksum, caller);

	memcpy(leader_ret, &leader, sizeof(struct leader_record));
	return error;
}

int delta_lease_leader_read_buf(struct task *task, int sector_size, int use_aio, int io_timeout,
			    struct sync_disk *disk, char *space_name, uint64_t host_id,
			    struct leader_record *leader_ret, char *buf, const char *caller)
{
	struct leader_record leader_end = { 0 };
	struct leader_record leader = { 0 };
	uint32_t checksum;
	int rv;

	rv = read_iobuf(disk->fd, disk->offset + ((host_id - 1) * sector_size), buf, sector_size,
			task, use_aio, io_timeout, NULL);
	if (rv < 0)
		return rv;

	memcpy(&leader_end, buf, sizeof(struct leader_record));
	checksum = leader_checksum(&leader_end);
	leader_record_in(&leader_end, &leader);

	rv = verify_leader(disk, space_name, host_id, &leader, checksum, caller);
	if (rv < 0)
		return rv;

	memcpy(leader_ret, &leader, sizeof(struct leader_record));
	return SANLK_OK;
}

int delta_caw_lease_leader_clobber(struct task *task, int io_timeout,
			       struct sync_disk *disk,
			       uint64_t host_id,
			       struct leader_record *leader,
			       int sector_size,
			       uint32_t caw_major,
			       uint32_t caw_minor,
			       uint64_t caw_offset,
			       char *caw_iobuf,
			       const char *caller)
{
	char caw_path[PATH_MAX] = { 0 };
	uint64_t offset_bytes;
	struct leader_record leader_end = { 0 };
	uint32_t checksum;
	int fd;
	int rv;

	if (devno_to_path(caw_major, caw_minor, caw_path) < 0) {
		log_error("delta_caw_lease_leader_clobber devno_to_path %u:%u error",
			  caw_major, caw_minor);
		return -1;
	}

	if ((fd = open(caw_path, O_RDWR | O_DIRECT | O_SYNC, 0)) < 0) {
		log_error("delta_caw_lease_leader_clobber %u:%u %s open errno %d",
			  caw_major, caw_minor, caw_path, errno);
		return -1;
	}

	leader_record_out(leader, &leader_end);
	checksum = leader_checksum(&leader_end);
	leader_end.checksum = cpu_to_le32(checksum);

	memmove(caw_iobuf + sector_size, caw_iobuf, sector_size);
	memcpy(caw_iobuf + sector_size, &leader_end, sizeof(struct leader_record));

	offset_bytes = caw_offset + ((host_id - 1) * sector_size);

	rv = compare_and_write(task, fd, io_timeout, caw_major, caw_minor,
			       sector_size, offset_bytes, caw_iobuf, NULL, NULL);
	close(fd);
	return rv;
}

/*
 * NB. this should not be used to write the leader record, it is meant only
 * for manually clobbering the disk to corrupt it for testing, or to manually
 * repair it after it's corrupted.
 */

int delta_lease_leader_clobber(struct task *task, int io_timeout,
			       struct sync_disk *disk,
			       uint64_t host_id,
			       struct leader_record *leader,
			       const char *caller)
{
	struct leader_record leader_end = { 0 };
	uint32_t checksum;
	int rv;

	leader_record_out(leader, &leader_end);

	checksum = leader_checksum(&leader_end);

	leader_end.checksum = cpu_to_le32(checksum);

	rv = write_sector(disk, leader->sector_size, host_id - 1, (char *)&leader_end, sizeof(struct leader_record),
			  task, task->use_aio, io_timeout, caller);
	if (rv < 0)
		return rv;
	return SANLK_OK;
}

static void make_local_copy_path(char *path, int len, struct space *sp)
{
	snprintf(path, len, "/var/lib/sanlock/delta_lease_%s_%llu",
		 sp->space_name, (unsigned long long)sp->host_id);
}

static void delta_local_copy_read(struct space *sp, uint64_t *prev_gen)
{
	char path[PATH_MAX];
	FILE *file;
	int rv;

	*prev_gen = 0;

	make_local_copy_path(path, sizeof(path), sp);

	file = fopen(path, "r");
	if (!file) {
		log_space(sp, "delta_local_copy_read open error %d %s", errno, path);
		return;
	}

	rv = fscanf(file, "%llu", (unsigned long long *)prev_gen);
	if (rv != 1)
		log_space(sp, "delta_local_copy_read read error %d %s", errno, path);

	fclose(file);
}

static void delta_local_copy_write(struct space *sp, uint64_t next_gen)
{
	char path[PATH_MAX];
	FILE *file;
	int rv;

	make_local_copy_path(path, sizeof(path), sp);

	file = fopen(path, "w");
	if (!file) {
		log_space(sp, "delta_local_copy_write open error %d", errno);
		return;
	}

	rv = fprintf(file, "%llu\n", (unsigned long long)next_gen);
	if (rv < 0)
		log_erros(sp, "delta_local_copy_write write error %d %s", errno, path);
	if (fflush(file))
		log_erros(sp, "delta_local_copy_write flush error %d %s", errno, path);
	if (fclose(file))
		log_erros(sp, "delta_local_copy_write close error %d %s", errno, path);
}

int delta_compare_and_write(struct task *task, struct space *sp, uint64_t host_id, char *caw_data, int *wr_ms)
{
	uint64_t offset_bytes;
	struct timespec begin, end, diff;
	int rv;

	if (wr_ms)
		clock_gettime(CLOCK_MONOTONIC_RAW, &begin);

	if (sp->caw_dev_use) {
		offset_bytes = sp->caw_dev_offset + ((host_id - 1) * sp->sector_size);

		rv = compare_and_write(task, sp->caw_dev_fd, sp->io_timeout,
				       sp->caw_dev_major, sp->caw_dev_minor,
				       sp->sector_size, offset_bytes, caw_data, NULL, NULL);
	} else {
		offset_bytes = sp->host_id_disk.offset + ((host_id - 1) * sp->sector_size);

		rv = compare_and_write(task, sp->host_id_disk.fd, sp->io_timeout,
				       sp->disk_major, sp->disk_minor,
				       sp->sector_size, offset_bytes, caw_data, NULL, NULL);
	}

	if (wr_ms) {
		clock_gettime(CLOCK_MONOTONIC_RAW, &end);
		ts_diff(&begin, &end, &diff);
		*wr_ms = (diff.tv_sec * 1000) + (diff.tv_nsec / 1000000);
	}

	return rv;
}

int delta_caw_lease_acquire(struct task *task,
			struct space *sp,
			char *our_host_name,
			struct leader_record *leader_ret)
{
	struct leader_record prev_leader;
	struct leader_record next_leader;
	struct leader_record leader_end;
	uint32_t checksum;
	int retries = 0;
	int error;

retry:
	/*
	 * On host_id_disk, read host_id sector into caw_data buf,
	 * copy leader out of that, and verify delta lease fields are valid.
	 * (delta_read_lockspace_sizes has just read this, so reading again
	 * here could be avoided with code reorg, but this is not frequent.)
	 */
	error = delta_lease_leader_read_buf(task, sp->sector_size, task->use_aio, sp->io_timeout,
					    &sp->host_id_disk, sp->space_name, sp->host_id,
					    &prev_leader, sp->caw_data, "caw_lease_acquire");
	if (error == SANLK_AIO_TIMEOUT) {
		log_space(sp, "delta_caw_acquire leader_read error %d", error);
		sp->caw_data = NULL;
		return error;
	}
	if (error < 0) {
		log_space(sp, "delta_caw_acquire leader_read error %d", error);
		return error;
	}

	log_space(sp, "delta_caw_acquire read magic 0x%x version 0x%x flags 0x%x owner %llu %llu %llu %.48s %s",
		  prev_leader.magic,
		  prev_leader.version,
		  prev_leader.flags,
		  (unsigned long long)prev_leader.owner_id,
		  (unsigned long long)prev_leader.owner_generation,
		  (unsigned long long)prev_leader.timestamp,
		  prev_leader.resource_name,
		  (prev_leader.flags & LFL_DEAD_EXT) ? "DEAD_EXT" : "");

	if (!(prev_leader.flags & LFL_CAW)) {
		/* shouldn't happen */
		log_erros(sp, "delta_caw_acquire leader not caw");
		return SANLK_WRONG_CAW;
	}

	if (retries && !(prev_leader.flags & LFL_DEAD_EXT)) {
		/* the only reason for a miscompare retry is if another host is
		   setting our DEAD_EXT flag. */
		log_erros(sp, "delta_caw_acquire retry did not find dead_ext flags 0x%x owner %llu %llu %llu %.48s",
			  prev_leader.flags,
			  (unsigned long long)prev_leader.owner_id,
			  (unsigned long long)prev_leader.owner_generation,
			  (unsigned long long)prev_leader.timestamp,
			  prev_leader.resource_name);
	}

	memcpy(&next_leader, &prev_leader, sizeof(struct leader_record));
	next_leader.flags &= ~LFL_DEAD_EXT;
	next_leader.timestamp = monotime();
	next_leader.io_timeout = (sp->io_timeout & 0x00FF);
	next_leader.owner_id = sp->host_id;
	next_leader.owner_generation += 1;
	memcpy(next_leader.resource_name, our_host_name, NAME_ID_SIZE);
	next_leader.checksum = 0; /* set below */

	log_space(sp, "delta_caw_acquire write owner %llu %llu %llu %.48s",
		  (unsigned long long)next_leader.owner_id,
		  (unsigned long long)next_leader.owner_generation,
		  (unsigned long long)next_leader.timestamp,
		  next_leader.resource_name);

	leader_record_out(&next_leader, &leader_end);
	checksum = leader_checksum(&leader_end); /* compute checksum on byte swapped data */
	next_leader.checksum = checksum;
	leader_end.checksum = cpu_to_le32(checksum);

	/*
	 * prev_data is the data currently on the disk,
	 * and exists in the first half of caw_data.
	 * next_data is a copy of prev_data updated with
	 * the new leader record, and goes in second half
	 * of caw_data.
	 */
	memmove(sp->caw_data + sp->sector_size, sp->caw_data, sp->sector_size);
	memcpy(sp->caw_data + sp->sector_size, &leader_end, sizeof(struct leader_record));

	error = delta_compare_and_write(task, sp, sp->host_id, sp->caw_data, NULL);
	if (error == SANLK_CAW_MISCOMPARE) {
		/* can get miscompare if another host is setting dead_ext */
		log_space(sp, "delta_caw_acquire compare_and_write error %d", error);
		if (!retries) {
			retries++;
			goto retry;
		}
		return error;
	} else if (error < 0) {
		log_erros(sp, "delta_caw_acquire compare_and_write error %d", error);
		return error;
	}

	memcpy(leader_ret, &next_leader, sizeof(struct leader_record));

	return SANLK_OK;
}

/*
 * delta_lease_acquire:
 * set the owner of host_id to our_host_name.
 *
 * paxos_lease_acquire:
 * set the owner of resource_name to host_id.
 *
 * our_host_name is a unique host identifier used to detect when two different
 * hosts are trying to acquire the same host_id (since both will be using the
 * same host_id, that host_id won't work to distinguish between them.) We copy
 * our_host_name into leader.resource_name, so in a sense the owner_id and
 * resource_name fields of the leader_record switch functions: the common
 * resource is the ower_id, and the distinguishing id is the resource_name.
 */

int delta_lease_acquire(struct task *task,
			struct space *sp,
			struct sync_disk *disk,
			char *space_name,
			char *our_host_name,
			uint64_t host_id,
			int nodelay,
			struct leader_record *leader_ret)
{
	struct leader_record leader;
	struct leader_record leader1;
	struct leader_record leader_end;
	uint64_t new_ts;
	uint64_t prev_generation;
	uint64_t next_generation;
	uint32_t checksum;
	uint32_t version;
	int other_io_timeout, other_host_dead_seconds, other_id_renewal_seconds;
	int i, error, rv, delay, delta_large_delay;
	int is_free, is_same;
	int no_delay = 0;

	log_space(sp, "delta_acquire begin %.48s:%llu",
		  sp->space_name, (unsigned long long)host_id);

	/*
	 * Keep a local copy of the last started generation number
	 * in case the delta lease on disk is reverted by another
	 * host writing the DEAD_EXT flag.  The local copy of the
	 * generation number ensures we don't reuse the same
	 * generation number that we started last.
	 */
	delta_local_copy_read(sp, &prev_generation);

	error = delta_lease_leader_read(task, sp->sector_size, sp->io_timeout, disk, space_name, host_id, &leader,
					"delta_acquire_begin");
	if (error < 0) {
		log_space(sp, "delta_acquire leader_read1 error %d", error);
		return error;
	}

	version = leader.version;

	other_io_timeout = leader.io_timeout;
	if (!other_io_timeout)
		other_io_timeout = sp->io_timeout;

	/*
	 * If the delta lease is free, and the prev owner matches our host
	 * name, then reacquire with no delay.
	 *
	 * If the delta lease is not free, and the prev owner matches our host
	 * name, and our host name is from product_uuid, then reacquire with no
	 * delay.  Assumption here is that the delta lease is not free because
	 * this host did not release it cleanly last time.  The non-free lease
	 * generally indicates it's being used by someone, and we should
	 * monitor it for a renewal period for updates.  But, we skip this
	 * monitoring given the certainty that comes from the owner being
	 * the product_uuid, and the assumption that our product_uuid will not
	 * be used by another host.
	 *
	 * If the prev owner was not our name, we delay for a rewnewal period
	 * to monitor for current use from another host.
	 *
	 * If the lease is not free, we delay and monitor for a renewal period
	 * to monitor (except when owner is our product_uuid per above.)
	 *
	 * If the lease is not free, and the owner does not match our host
	 * name, then use a long delay to monitor for other hosts using it,
	 * or to ensure a prev host using this lease is dead.
	 */

	is_free = (leader.timestamp == LEASE_FREE);
	is_same = !memcmp(leader.resource_name, our_host_name, NAME_ID_SIZE);

	if (!is_same)
		log_debug("delta_acquire new owner %.48s old owner %.48s", our_host_name, leader.resource_name);

	if (is_same || is_free) {
		if (is_same && is_free)
			no_delay = 1;
		if (is_same && our_host_name_matches_product_uuid)
			no_delay = 1;
		if (nodelay)
			no_delay = 1;

		log_space(sp, "delta_acquire %s owner, %s free, %s our_product_uuid, %s delay, other_io_timeout %d, flag_nodelay %d",
			  is_same ? "same" : "new",
			  is_free ? "is" : "not",
			  our_host_name_matches_product_uuid ? "is" : "not",
			  no_delay ? "no" : "short",
			  other_io_timeout,
			  nodelay);

		goto write_new;
	}

	/*
	 * we need to ensure that a host_id cannot be acquired and released
	 * sooner than host_dead_seconds because the change in host_id
	 * ownership affects the host_id "liveness" determination used by paxos
	 * leases, and the ownership of paxos leases cannot change until after
	 * host_dead_seconds to ensure that the watchdog has fired.  So, I
	 * think we want the delay here to be the max of host_dead_seconds and
	 * the D+6d delay.
	 *
	 * Per the algorithm in the paper, a delta lease can change ownership
	 * in the while loop below after the delta_delay of D+6d.  However,
	 * because we use the change of delta lease ownership to directly
	 * determine the change in paxos lease ownership, we need the delta
	 * delay to also meet the delay requirements of the paxos leases.  The
	 * paxos leases cannot change ownership until a min of
	 * host_dead_seconds to ensure the watchdog has fired.  So, the timeout
	 * we use here must be the max of the delta delay (D+6d) and
	 * host_dead_seconds.
	 */

	other_host_dead_seconds = calc_host_dead_seconds(other_io_timeout);
	other_id_renewal_seconds = calc_id_renewal_seconds(other_io_timeout);

	log_space(sp, "delta_acquire other_host %.48s, timestamp %llu, other_io_timeout %d",
		  leader.resource_name, (unsigned long long)leader.timestamp, other_io_timeout);

	delay = other_host_dead_seconds;
	delta_large_delay = other_id_renewal_seconds + (6 * other_io_timeout);
	if (delta_large_delay > delay)
		delay = delta_large_delay;

	while (1) {
		memcpy(&leader1, &leader, sizeof(struct leader_record));

		log_space(sp, "delta_acquire delta_large_delay %d delay %d",
			  delta_large_delay, delay);

		/* TODO: we could reread every several seconds to see if
		   it has changed, so we can abort more quickly if so */

		for (i = 0; i < delay; i++) {
			if (sp->external_remove || external_shutdown) {
				log_space(sp, "delta_acquire abort1 remove %d shutdown %d",
					  sp->external_remove, external_shutdown);
				return SANLK_ERROR;
			}
			sleep(1);
		}

		error = delta_lease_leader_read(task, sp->sector_size, sp->io_timeout, disk, space_name, host_id,
						&leader, "delta_acquire_wait");
		if (error < 0) {
			log_space(sp, "delta_acquire leader_read2 error %d", error);
			return error;
		}

		if (!memcmp(&leader1, &leader, sizeof(struct leader_record)))
			break;

		if (leader.timestamp == LEASE_FREE)
			break;

		log_erros(sp, "delta_acquire host_id %llu busy1 %llu %llu %llu %.48s",
			  (unsigned long long)host_id,
			  (unsigned long long)leader.owner_id,
			  (unsigned long long)leader.owner_generation,
			  (unsigned long long)leader.timestamp,
			  leader.resource_name);
		return SANLK_HOSTID_BUSY;
	}

 write_new:

	if (leader.owner_generation && (prev_generation != leader.owner_generation)) {
		log_erros(sp, "delta_acquire %llu wrong prev_generation %llu leader %llu %llu %llu %s",
			  (unsigned long long)host_id,
			  (unsigned long long)prev_generation,
			  (unsigned long long)leader.owner_id,
			  (unsigned long long)leader.owner_generation,
			  (unsigned long long)leader.timestamp,
			  (leader.flags & LFL_DEAD_EXT) ? "dead_ext" : "");
	}

	if (leader.flags & LFL_DEAD_EXT) {
		leader.flags &= ~LFL_DEAD_EXT;

		if (!prev_generation) {
			/*
			 * This shouldn't happen. If it does happen,
			 * some extra caution could be to do
			 * next = leader.gen + 2, but skipping a gen
			 * unnecessarily could also complicate external
			 * assumptions about the normal gen sequence.
			 */
			next_generation = leader.owner_generation + 1;

			log_space(sp, "delta_acquire %llu leader dead_ext gen %llu no prev_generation next %llu",
				  (unsigned long long)host_id,
				  (unsigned long long)leader.owner_generation,
				  (unsigned long long)next_generation);
		} else if (leader.owner_generation + 1 == prev_generation) {
			/*
			 * our last generation was clobbered by a DEAD_EXT
			 * write from another host, and then we failed before
			 * renewing and replacing that late DEAD_EXT write.
			 * We must +2 to avoid reusing the previous gen.
			 */
			next_generation = leader.owner_generation + 2;

			log_space(sp, "delta_acquire %llu leader dead_ext gen %llu clobbered prev_generation %llu next %llu",
				  (unsigned long long)host_id,
				  (unsigned long long)leader.owner_generation,
				  (unsigned long long)prev_generation,
				  (unsigned long long)next_generation);
		} else {
			/*
			 * normal case when we restart after DEAD_EXT
			 * was set in our lease.
			 */
			next_generation = leader.owner_generation + 1;

			log_space(sp, "delta_acquire %llu leader dead_ext gen %llu normal prev_generation %llu next %llu",
				  (unsigned long long)host_id,
				  (unsigned long long)leader.owner_generation,
				  (unsigned long long)prev_generation,
				  (unsigned long long)next_generation);
		}
	} else {
		/* common case */
		next_generation = leader.owner_generation + 1;
	}

	new_ts = monotime();
	leader.version = version;
	leader.timestamp = new_ts;
	leader.io_timeout = (sp->io_timeout & 0x00FF);
	leader.owner_id = host_id;
	leader.owner_generation = next_generation;
	memcpy(leader.resource_name, our_host_name, NAME_ID_SIZE);
	leader.checksum = 0; /* set below */

	log_space(sp, "delta_acquire write %llu %llu %llu %.48s",
		  (unsigned long long)leader.owner_id,
		  (unsigned long long)leader.owner_generation,
		  (unsigned long long)leader.timestamp,
		  leader.resource_name);

	leader_record_out(&leader, &leader_end);

	/*
	 * N.B. must compute checksum after the data has been byte swapped.
	 */
	checksum = leader_checksum(&leader_end);
	leader.checksum = checksum;
	leader_end.checksum = cpu_to_le32(checksum);

	rv = write_sector(disk, sp->sector_size, host_id - 1, (char *)&leader_end, sizeof(struct leader_record),
			  task, task->use_aio, sp->io_timeout, "delta_leader");
	if (rv < 0) {
		log_space(sp, "delta_acquire write error %d", rv);
		return rv;
	}

	delta_local_copy_write(sp, next_generation);

	memcpy(&leader1, &leader, sizeof(struct leader_record));

	if (no_delay) {
		usleep(10000);
		goto reread;
	}

	delay = 2 * other_io_timeout;
	log_space(sp, "delta_acquire delta_short_delay %d", delay);

	for (i = 0; i < delay; i++) {
		if (sp->external_remove || external_shutdown) {
			log_space(sp, "delta_acquire abort2 remove %d shutdown %d",
				  sp->external_remove, external_shutdown);
			return SANLK_ERROR;
		}
		sleep(1);
	}

reread:
	error = delta_lease_leader_read(task, sp->sector_size, sp->io_timeout, disk, space_name, host_id, &leader,
					"delta_acquire_check");
	if (error < 0) {
		log_space(sp, "delta_acquire leader_read3 error %d", error);
		return error;
	}

	if (memcmp(&leader1, &leader, sizeof(struct leader_record))) {
		log_erros(sp, "delta_acquire host_id %llu busy2 %llu %llu %llu %.48s",
			  (unsigned long long)host_id,
			  (unsigned long long)leader.owner_id,
			  (unsigned long long)leader.owner_generation,
			  (unsigned long long)leader.timestamp,
			  leader.resource_name);
		return SANLK_HOSTID_BUSY;
	}

	log_space(sp, "delta_acquire done %llu %llu %llu",
		  (unsigned long long)leader.owner_id,
		  (unsigned long long)leader.owner_generation,
		  (unsigned long long)leader.timestamp);

	memcpy(leader_ret, &leader, sizeof(struct leader_record));
	return SANLK_OK;
}

int delta_caw_lease_renew(struct task *task,
		      struct space *sp,
		      char *bitmap,
		      struct delta_extra *extra,
		      int prev_result,
		      int *read_result,
		      int log_renewal_level,
		      struct leader_record *leader_last,
		      struct leader_record *leader_ret,
		      int *rd_ms, int *wr_ms)
{
	struct leader_record leader;
	struct leader_record leader_end;
	struct sync_disk *disk = &sp->host_id_disk;
	uint64_t id_offset;
	uint64_t new_ts;
	uint64_t now;
	uint64_t host_id = sp->host_id;
	int sector_size = sp->sector_size;
	int align_size = sp->align_size;
	uint32_t checksum;
	int rv;

	if (!leader_last) {
		log_erros(sp, "delta_caw_renew no leader_last");
		return -EINVAL;
	}

	*rd_ms = -1;
	*wr_ms = -1;

	*read_result = SANLK_ERROR;

	/* offset in bytes of our leader_record in the lease area */
	id_offset = (host_id - 1) * sector_size;
	if (id_offset > align_size) {
		log_erros(sp, "delta_caw_renew bad offset %llu align_size %d",
			  (unsigned long long)id_offset, align_size);
		return -EINVAL;
	}

	/*
	 * The first call to renew will allocate this special task->iobuf
	 * into which the full lease area is read by each renew.
	 *
	 * NB. this task->iobuf is also copied by the lockspace thread
	 * into renewal_read_buf, which is then copied in the main loop
	 * by check_our_lease and passed to check_other_leases.
	 */

	if (!task->iobuf) {
		char **p_iobuf;
		int read_mem_align = getpagesize();
		int is_huge = 0;
		int iobuf_len = sp->align_size;

		p_iobuf = &task->iobuf;

		/*
		 * Minimum hugepage is 2MB, so for 1MB lease area we increase
		 * the iobuf size and alignment to 2MB, wasting a MB of space
		 * to more likely avoid splitting the i/o (read len remains
		 * 1MB even though iobuf_len is extended to 2MB.)
		 * iobuf_len begins being set to align_size (1MB, 2MB, 4MB, 8MB).
		 */
		if (com.use_hugepages && ((iobuf_len == ONE_MB_IN_BYTES) || !(iobuf_len % TWO_MB_IN_BYTES))) {
			read_mem_align = TWO_MB_IN_BYTES;
			if (iobuf_len < TWO_MB_IN_BYTES)
				iobuf_len = TWO_MB_IN_BYTES;
			is_huge = 1;
		}

		rv = posix_memalign((void *)p_iobuf, read_mem_align, iobuf_len);
		if (rv) {
			log_erros(sp, "dela_caw_renew memalign rv %d", rv);
			return -ENOMEM;
		}

		if (is_huge) {
			char *iobuf = task->iobuf;
			char *bp;

			madvise(iobuf, iobuf_len, MADV_HUGEPAGE);
			/* allocate the pages */
			for (bp = iobuf; bp < iobuf+iobuf_len; bp += TWO_MB_IN_BYTES)
				memset(bp, 0, 1);
		}
	}

	if (log_renewal_level != -1)
		log_level(sp->space_id, 0, NULL, log_renewal_level, "delta_caw_renew begin read");

	/*
	 * Read the full lease area into task->iobuf.
	 *
	 * This is done with a sync io read on the host_id_disk
	 * (not the underlying caw dev used by CAW SGIO.)
	 *
	 * Technically, renew would not need to read the lease
	 * area, or the local host_id lease, and could just
	 * issue a compare-and-write to update the leader
	 * using the previous renewal buffer as the prev data.
	 * But, reading the full lease acts as a monitor for
	 * the liveness of all the other host_id leases.
	 *
	 * In the future, the delta caw lease renewal might
	 * just use a single caw each time, and reading the
	 * entire lease area (to monitor other leases) could
	 * be done separately, e.g. by a different thread on
	 * a different schedule, and not by the renewal thread.
	 * This would let the renewal thread focus on the
	 * critical updates and not be delayed by the less
	 * important monitor reads.
	 *
	 * read len remains align_size even if iobuf_len
	 * is the larger hugepage size.
	 */
	rv = read_iobuf(disk->fd, disk->offset, task->iobuf, align_size, task, NO_AIO, 0, rd_ms);
	if (rv) {
		/* This is NO_AIO so no need to check AIO_TIMEOUT and set task->iobuf = NULL */
		log_erros(sp, "delta_caw_renew read rv %d offset %llu %s",
			  rv, (unsigned long long)disk->offset, disk->path);
		return rv;
	}

	*read_result = SANLK_OK;
	memcpy(&leader_end, task->iobuf+id_offset, sizeof(struct leader_record));
	checksum = leader_checksum(&leader_end); /* compute checksum before byte swapping */
	leader_record_in(&leader_end, &leader);

	rv = verify_leader(disk, sp->space_name, host_id, &leader, checksum, "delta_caw_renew");
	if (rv < 0) {
		log_erros(sp, "delta_caw_renew verify_leader error %d", rv);
		return rv;
	}

	/*
	 * This can happen because another host can declare we have failed
	 * (e.g. our renewals are delayed), but we are still trying to run
	 * and renew our lease.  The other host removes our persistent
	 * reservation, and sets DEAD_EXT in our lease.  We are able to
	 * read our lease (still possible without a PR), and see here that
	 * another host has said we're dead.  Our PR has been removed, so
	 * there's no hope of us continuing normally, so treat this ls as
	 * if it is corrupt.
	 *
	 * I don't think there will be a scenario where leader.owner_generation
	 * is less than leader_last->owner_generation, as is possible in the
	 * non-CAW case, because CAW should resolve any race between us starting
	 * the lockspace and another host setting DEAD_EXT.
	 */
	if (leader.flags & LFL_DEAD_EXT) {
		log_erros(sp, "delta_caw_renew read leader %llu %llu %llu dead_ext from 0x%llx last %llu %llu %llu",
			  (unsigned long long)leader.owner_id,
			  (unsigned long long)leader.owner_generation,
			  (unsigned long long)leader.timestamp,
			  (unsigned long long)leader.unused1,
			  (unsigned long long)leader_last->owner_id,
			  (unsigned long long)leader_last->owner_generation,
			  (unsigned long long)leader_last->timestamp);
		return SANLK_RENEW_DEAD_EXT;
	}

	/* We can't always memcmp(&leader, leader_last) because previous writes
	   may have failed and we don't know if they were actually written
	   or not.  We can verify that we're still the owner, though, which is
	   the main thing to verify. */

	if (leader.owner_id != leader_last->owner_id ||
	    leader.owner_generation != leader_last->owner_generation ||
	    memcmp(leader.resource_name, leader_last->resource_name, NAME_ID_SIZE)) {
		log_erros(sp, "delta_caw_renew not owner");
		log_leader_error(0, sp->space_name, host_id, &sp->host_id_disk, leader_last, "delta_caw_renew_last");
		log_leader_error(0, sp->space_name, host_id, &sp->host_id_disk, &leader, "delta_caw_renew_read");
		return SANLK_RENEW_OWNER;
	}

	if (prev_result == SANLK_OK &&
	    memcmp(&leader, leader_last, sizeof(struct leader_record))) {
		log_erros(sp, "delta_caw_renew reread mismatch");
		log_leader_error(0, sp->space_name, host_id, &sp->host_id_disk, leader_last, "delta_caw_renew_last");
		log_leader_error(0, sp->space_name, host_id, &sp->host_id_disk, &leader, "delta_caw_renew_read");
		return SANLK_RENEW_DIFF;
	}

	if (leader.io_timeout != sp->io_timeout) {
		log_erros(sp, "delta_caw_renew io_timeout changed disk %d sp %d",
			  leader.io_timeout, sp->io_timeout);
		leader.io_timeout = (sp->io_timeout & 0x00FF);
	}

	/*
	 * Renewal updates the leader timestamp field.
	 */

	/* Copy our sector from iobuf into sp->caw_data (both next and prev). */
	memcpy(sp->caw_data, task->iobuf+id_offset, sector_size);
	memcpy(sp->caw_data + sector_size, task->iobuf+id_offset, sector_size);

	new_ts = monotime();

	if (log_renewal_level != -1)
		log_level(sp->space_id, 0, NULL, log_renewal_level, "delta_caw_renew begin write for new ts %llu", (unsigned long long)new_ts);

	if (leader.timestamp >= new_ts)
		log_erros(sp, "delta_caw_renew timestamp too small");

	leader.timestamp = new_ts;

	leader.checksum = 0; /* set below */

	/* TODO: rename the leader fields */
	if (extra) {
		leader.write_id = extra->field1;
		leader.write_generation = extra->field2;
		leader.write_timestamp = extra->field3;
	}

	leader_record_out(&leader, &leader_end);

	/*
	 * N.B. must compute checksum after the data has been byte swapped.
	 */
	checksum = leader_checksum(&leader_end);
	leader.checksum = checksum;
	leader_end.checksum = cpu_to_le32(checksum);

	memcpy(sp->caw_data + sector_size, &leader_end, sizeof(struct leader_record));
	memcpy(sp->caw_data + sector_size + LEADER_RECORD_MAX, bitmap, HOSTID_BITMAP_SIZE);

	rv = delta_compare_and_write(task, sp, host_id, sp->caw_data, wr_ms);

	now = monotime();

	if (rv < 0) {
		log_erros(sp, "delta_caw_renew write time %llu error %d",
			  (unsigned long long)(now - new_ts), rv);
		return rv;
	}

	if (now - new_ts >= sp->io_timeout)
		log_erros(sp, "delta_caw_renew long write time %llu sec",
			  (unsigned long long)(now - new_ts));

	memcpy(leader_ret, &leader, sizeof(struct leader_record));
	return SANLK_OK;
}

int delta_lease_renew(struct task *task,
		      struct space *sp,
		      struct sync_disk *disk,
		      char *space_name,
		      char *bitmap,
		      struct delta_extra *extra,
		      int prev_result,
		      int *read_result,
		      int log_renewal_level,
		      struct leader_record *leader_last,
		      struct leader_record *leader_ret,
		      int *rd_ms, int *wr_ms)
{
	struct leader_record leader;
	struct leader_record leader_end;
	char **p_iobuf;
	char **p_wbuf;
	char *wbuf;
	struct timespec begin, end, diff;
	uint32_t checksum;
	uint32_t reap_timeout_msec;
	uint64_t host_id, id_offset, new_ts, now;
	int sector_size = sp->sector_size;
	int align_size = sp->align_size; /* 1,2,4,8 MB depending on sector_size */
	int iobuf_len = sp->align_size; /* may increase (from 1MB) for hugepage */
	int read_len = sp->align_size;  /* not increased (from 1MB) for hugepage */
	int rv;

	if (!leader_last) {
		log_erros(sp, "delta_renew no leader_last");
		return -EINVAL;
	}

	*rd_ms = -1;
	*wr_ms = -1;

	*read_result = SANLK_ERROR;

	host_id = leader_last->owner_id;

	/* offset of our leader_record */
	id_offset = (host_id - 1) * sector_size;
	if (id_offset > align_size) {
		log_erros(sp, "delta_renew bad offset %llu align_size %d",
			  (unsigned long long)id_offset, align_size);
		return -EINVAL;
	}

	/* if the previous renew timed out in this initial read, and that read
	   is now complete, we can use that result here instead of discarding
	   it and doing another. */

	if (prev_result == SANLK_AIO_TIMEOUT) {
		if (!task->read_iobuf_timeout_aicb) {
			/* shouldn't happen, when do_linux_aio returned AIO_TIMEOUT
			   it should have set read_iobuf_timeout_aicb */
			log_erros(sp, "delta_renew reap no aicb");
			goto skip_reap;
		}

		if (!task->iobuf) {
			/* shouldn't happen */
			log_erros(sp, "delta_renew reap no iobuf");
			goto skip_reap;
		}

		log_space(sp, "delta_renew begin reap");

		if (!sp->renewal_read_extend_sec) {
			/* only wait .5 sec when trying to reap a prev io to clear it */
			reap_timeout_msec = 500;
		} else {
			/* effectively continue/extend the read phase from the previous renewal */
			reap_timeout_msec = sp->renewal_read_extend_sec * 1000;
		}

		clock_gettime(CLOCK_MONOTONIC_RAW, &begin);

		rv = read_iobuf_reap(disk->fd, disk->offset,
				     task->iobuf, read_len, task, reap_timeout_msec);

		log_space(sp, "delta_renew reap %d", rv);

		if (!rv) {
			/* read time for this renewal is the io_timeout length
			   for the previous read plus the time spent in reap. */
			clock_gettime(CLOCK_MONOTONIC_RAW, &end);
			ts_diff(&begin, &end, &diff);
			*rd_ms = (diff.tv_sec * 1000) + (diff.tv_nsec / 1000000) + (sp->io_timeout * 1000);
			task->read_iobuf_timeout_aicb = NULL;
			goto read_done;
		}
 skip_reap:
		/* abandon the previous timed out read and try a new
		   one from scratch.  the current task->iobuf mem will
		   freed when timeout_aicb completes sometime */

		task->read_iobuf_timeout_aicb = NULL;
		task->iobuf = NULL;
	}

	if (task->read_iobuf_timeout_aicb) {
		/* this could happen get here if there was another read between
		   renewal reads, which timed out and caused
		   read_iobuf_timeout_aicb to be set; I don't think there are
		   any cases where that would happen, though.  we could avoid
		   this confusion by passing back the timed out aicb along with
		   SANLK_AIO_TIMEOUT, and only save the timed out aicb when we
		   want to try to reap it later. */

		log_space(sp, "delta_renew timeout_aicb is unexpectedly %p iobuf %p",
			  task->read_iobuf_timeout_aicb, task->iobuf);
		task->read_iobuf_timeout_aicb = NULL;
		task->iobuf = NULL;
	}

	/*
	 * NB. this task->iobuf is also copied by the lockspace thread
	 * into renewal_read_buf, which is then copied in the main loop
	 * by check_our_lease and passed to check_other_leases.
	 */

	if (!task->iobuf) {
		int read_mem_align = getpagesize();
		int is_huge = 0;

		/* this will happen the first time renew is called, and after
		   a timed out renewal read fails to be reaped (see
		   task->iobuf = NULL above) */

		p_iobuf = &task->iobuf;

		/*
		 * Minimum hugepage is 2MB, so for 1MB lease area we increase
		 * the iobuf size and alignment to 2MB, wasting a MB of space
		 * to more likely avoid splitting the i/o (read_len remains
		 * 1MB even though iobuf_len is extended to 2MB.)
		 * iobuf_len begins being set to align_size (1MB, 2MB, 4MB, 8MB).
		 */
		if (com.use_hugepages && ((iobuf_len == ONE_MB_IN_BYTES) || !(iobuf_len % TWO_MB_IN_BYTES))) {
			read_mem_align = TWO_MB_IN_BYTES;
			if (iobuf_len < TWO_MB_IN_BYTES)
				iobuf_len = TWO_MB_IN_BYTES;
			is_huge = 1;
		}

		rv = posix_memalign((void *)p_iobuf, read_mem_align, iobuf_len);
		if (rv) {
			log_erros(sp, "dela_renew memalign rv %d", rv);
			rv = -ENOMEM;
		}

		if (is_huge) {
			char *iobuf = task->iobuf;
			char *bp;

			madvise(iobuf, iobuf_len, MADV_HUGEPAGE);
			/* allocate the pages */
			for (bp = iobuf; bp < iobuf+iobuf_len; bp += TWO_MB_IN_BYTES)
				memset(bp, 0, 1);
		}
	}

	if (log_renewal_level != -1)
		log_level(sp->space_id, 0, NULL, log_renewal_level, "delta_renew begin read");

	rv = read_iobuf(disk->fd, disk->offset, task->iobuf, read_len, task, task->use_aio, sp->io_timeout, rd_ms);
	if (rv) {
		/* the next time delta_lease_renew() is called, prev_result
		   will be this rv.  If this rv is SANLK_AIO_TIMEOUT, we'll
		   try to reap the event */

		if (rv == SANLK_AIO_TIMEOUT)
			log_erros(sp, "delta_renew read timeout %u sec offset %llu %s",
				  sp->io_timeout, (unsigned long long)disk->offset, disk->path);
		else
			log_erros(sp, "delta_renew read rv %d offset %llu %s",
				  rv, (unsigned long long)disk->offset, disk->path);
		return rv;
	}

 read_done:
	*read_result = SANLK_OK;
	memcpy(&leader_end, task->iobuf+id_offset, sizeof(struct leader_record));

	/* N.B. compute checksum before byte swapping */
	checksum = leader_checksum(&leader_end);

	leader_record_in(&leader_end, &leader);

	rv = verify_leader(disk, space_name, host_id, &leader, checksum, "delta_renew");
	if (rv < 0) {
		log_erros(sp, "delta_renew verify_leader error %d", rv);
		return rv;
	}

	/*
	 * Someone has set DEAD_EXT in our lease, likely along with
	 * our previous generation.  This would be a race between us
	 * joining (acquiring our lease), and another host declaring
	 * us dead (ext) for our prior failure.  In this case, we want
	 * to renew/write the values that we would have written if the
	 * other host had not clobbered our previous write.
	 *
	 * If we find our current generation has had DEAD_EXT set,
	 * it means that another host has seen us enter the fail state,
	 * has removed our persistent reservation, and set DEAD_EXT.
	 * This still allows us to read our lease, and see the DEAD_EXT
	 * set here.  There's no hope of continuing normally since we
	 * cannot write, so handle this like our lease is corrupted.
	 */
	if (leader.flags & LFL_DEAD_EXT) {
		if (leader.owner_generation < leader_last->owner_generation) {
			log_erros(sp, "delta_renew %llu late dead_ext gen %llu from %llu",
				  (unsigned long long)leader.owner_id,
				  (unsigned long long)leader.owner_generation,
				  (unsigned long long)leader.unused1);
			goto write_renew;
		} else {
			log_erros(sp, "delta_renew %llu current gen %llu is dead_ext from %llu",
				  (unsigned long long)leader.owner_id,
				  (unsigned long long)leader.owner_generation,
				  (unsigned long long)leader.unused1);
			return SANLK_RENEW_DEAD_EXT;
		}
	}

	/* We can't always memcmp(&leader, leader_last) because previous writes
	   may have timed out and we don't know if they were actually written
	   or not.  We can definitely verify that we're still the owner,
	   though, which is the main thing we need to know. */

	if (leader.owner_id != leader_last->owner_id ||
	    leader.owner_generation != leader_last->owner_generation ||
	    memcmp(leader.resource_name, leader_last->resource_name, NAME_ID_SIZE)) {
		log_erros(sp, "delta_renew not owner");
		log_leader_error(0, space_name, host_id, disk, leader_last, "delta_renew_last");
		log_leader_error(0, space_name, host_id, disk, &leader, "delta_renew_read");
		return SANLK_RENEW_OWNER;
	}

	if (prev_result == SANLK_OK &&
	    memcmp(&leader, leader_last, sizeof(struct leader_record))) {
		log_erros(sp, "delta_renew reread mismatch");
		log_leader_error(0, space_name, host_id, disk, leader_last, "delta_renew_last");
		log_leader_error(0, space_name, host_id, disk, &leader, "delta_renew_read");
		return SANLK_RENEW_DIFF;
	}

	if (leader.io_timeout != sp->io_timeout) {
		log_erros(sp, "delta_renew io_timeout changed disk %d sp %d",
			  leader.io_timeout, sp->io_timeout);
		leader.io_timeout = (sp->io_timeout & 0x00FF);
	}

write_renew:
	new_ts = monotime();

	if (log_renewal_level != -1)
		log_level(sp->space_id, 0, NULL, log_renewal_level, "delta_renew begin write for new ts %llu", (unsigned long long)new_ts);

	if (leader.timestamp >= new_ts)
		log_erros(sp, "delta_renew timestamp too small");

	leader.timestamp = new_ts;

	leader.flags &= ~LFL_DEAD_EXT;

	leader.checksum = 0; /* set below */

	/* TODO: rename the leader fields */
	if (extra) {
		leader.write_id = extra->field1;
		leader.write_generation = extra->field2;
		leader.write_timestamp = extra->field3;
	}

	p_wbuf = &wbuf;
	rv = posix_memalign((void *)p_wbuf, getpagesize(), sector_size);
	if (rv) {
		log_erros(sp, "dela_renew write memalign rv %d", rv);
		return -ENOMEM;
	}
	memset(wbuf, 0, sector_size);

	leader_record_out(&leader, &leader_end);

	/*
	 * N.B. must compute checksum after the data has been byte swapped.
	 */
	checksum = leader_checksum(&leader_end);
	leader.checksum = checksum;
	leader_end.checksum = cpu_to_le32(checksum);

	memcpy(wbuf, &leader_end, sizeof(struct leader_record));
	memcpy(wbuf+LEADER_RECORD_MAX, bitmap, HOSTID_BITMAP_SIZE);

	/* extend io timeout for this one write; we need to give this write
	   every chance to succeed, and there's no point in letting it time
	   out.  there's nothing we would do but retry it, and timing out and
	   retrying unnecessarily would probably be counter productive. */

	rv = write_iobuf(disk->fd, disk->offset+id_offset, wbuf, sector_size, task, task->use_aio,
			 calc_host_dead_seconds(sp->io_timeout), wr_ms);

	if (rv != SANLK_AIO_TIMEOUT)
		free(wbuf);

	now = monotime();

	if (rv < 0) {
		log_erros(sp, "delta_renew write time %llu error %d",
			  (unsigned long long)(now - new_ts), rv);
		return rv;
	}

	if (now - new_ts >= sp->io_timeout)
		log_erros(sp, "delta_renew long write time %llu sec",
			  (unsigned long long)(now - new_ts));

	/* the paper shows doing a delay and another read here, but it seems
	   unnecessary since we do the same at the beginning of the next renewal */

	memcpy(leader_ret, &leader, sizeof(struct leader_record));
	return SANLK_OK;
}

int delta_caw_lease_release(struct task *task,
			struct space *sp,
			struct sync_disk *disk,
			char *space_name GNUC_UNUSED,
			struct leader_record *leader_last,
			struct leader_record *leader_ret)
{
	struct leader_record leader;
	struct leader_record leader_end;
	struct leader_record prev_leader;
	char *rdbuf = NULL;
	char **p_rdbuf;
	uint64_t host_id;
	uint32_t checksum;
	int retries = 0;
	int error;
	int rv;

	if (!leader_last)
		return -EINVAL;

	host_id = leader_last->owner_id;

	log_space(sp, "delta_caw_release begin %.48s:%llu",
		  sp->space_name, (unsigned long long)host_id);

	memcpy(&leader, leader_last, sizeof(struct leader_record));
	leader.timestamp = LEASE_FREE;
	leader.checksum = 0; /* set below */

	leader_record_out(&leader, &leader_end);
	/* compute checksum on byte swapped data */
	checksum = leader_checksum(&leader_end);
	leader.checksum = checksum;
	leader_end.checksum = cpu_to_le32(checksum);

	/*
	 * If our last compare_and_write (from acquire or renew) was
	 * successful, then the second half of caw_data is what's currently on
	 * disk, so copy that to the first half.
	 *
	 * If our last compare_and_write failed, then we don't know
	 * what's currently on disk, so reread it (delta_lease_leader_read_buf),
	 * then retry the compare_and_write with the new leader.
	 * (We might avoid this by keeping a separate buffer of our
	 * last successful compare_and_write data, and use that here
	 * instead of the second half of sp->caw_data.)
	 */
	memmove(sp->caw_data, sp->caw_data + sp->sector_size, sp->sector_size);
	memcpy(sp->caw_data + sp->sector_size, &leader_end, sizeof(struct leader_record));

retry_caw:
	error = delta_compare_and_write(task, sp, host_id, sp->caw_data, NULL);

	if (error == SANLK_CAW_MISCOMPARE) {
		/* this is probably from initial acquire failure? other reasons? */
		p_rdbuf = &rdbuf;
		rv = posix_memalign((void *)p_rdbuf, getpagesize(), sp->sector_size);
		if (rv) {
			log_erros(sp, "delta_caw_release miscompare reread nomem");
			return SANLK_CAW_MISCOMPARE;
		}

		rv = delta_lease_leader_read_buf(task, sp->sector_size, NO_AIO, sp->io_timeout,
					&sp->host_id_disk, sp->space_name, sp->host_id,
					&prev_leader, rdbuf, "caw_lease_release");
		if (rv == SANLK_AIO_TIMEOUT) {
			log_erros(sp, "delta_caw_release miscompare reread failed %d", rv);
			return SANLK_CAW_MISCOMPARE;
		}
		if (rv < 0) {
			log_erros(sp, "delta_caw_release miscompare reread failed %d", rv);
			free(rdbuf);
			return SANLK_CAW_MISCOMPARE;
		}
		log_erros(sp, "delta_caw_release miscompare reread %llu %llu %llu %x last %llu %llu %llu %x",
			  (unsigned long long)prev_leader.owner_id,
			  (unsigned long long)prev_leader.owner_generation,
			  (unsigned long long)prev_leader.timestamp,
			  prev_leader.flags,
			  (unsigned long long)leader_last->owner_id,
			  (unsigned long long)leader_last->owner_generation,
			  (unsigned long long)leader_last->timestamp,
			  leader_last->flags);
		memcpy(&leader, &prev_leader, sizeof(struct leader_record));
		leader.timestamp = LEASE_FREE;
		leader.checksum = 0;
		leader_record_out(&leader, &leader_end);
		checksum = leader_checksum(&leader_end);
		leader.checksum = checksum;
		leader_end.checksum = cpu_to_le32(checksum);
		memcpy(sp->caw_data, rdbuf, sp->sector_size);
		memcpy(sp->caw_data + sp->sector_size, rdbuf, sp->sector_size);
		memcpy(sp->caw_data + sp->sector_size, &leader_end, sizeof(struct leader_record));
		free(rdbuf);
		if (!retries) {
			retries++;
			goto retry_caw;
		}
	}

	if (error < 0) {
		log_erros(sp, "delta_caw_release compare_and_write error %d", error);
		return error;
	}

	log_space(sp, "delta_caw_release done %llu %llu %llu",
		  (unsigned long long)leader.owner_id,
		  (unsigned long long)leader.owner_generation,
		  (unsigned long long)leader.timestamp);

	memcpy(leader_ret, &leader, sizeof(struct leader_record));
	return SANLK_OK;
}

int delta_lease_release(struct task *task,
			struct space *sp,
			struct sync_disk *disk,
			char *space_name GNUC_UNUSED,
			struct leader_record *leader_last,
			struct leader_record *leader_ret)
{
	struct leader_record leader;
	struct leader_record leader_end;
	uint64_t host_id;
	uint32_t checksum;
	int rv;

	if (!leader_last)
		return -EINVAL;

	host_id = leader_last->owner_id;

	log_space(sp, "delta_release begin %.48s:%llu",
		  sp->space_name, (unsigned long long)host_id);

	memcpy(&leader, leader_last, sizeof(struct leader_record));
	leader.timestamp = LEASE_FREE;
	leader.checksum = 0; /* set below */

	leader_record_out(&leader, &leader_end);

	/*
	 * N.B. must compute checksum after the data has been byte swapped.
	 */
	checksum = leader_checksum(&leader_end);
	leader.checksum = checksum;
	leader_end.checksum = cpu_to_le32(checksum);

	rv = write_sector(disk, sp->sector_size, host_id - 1, (char *)&leader_end, sizeof(struct leader_record),
			  task, task->use_aio, sp->io_timeout, "delta_leader");
	if (rv < 0) {
		log_space(sp, "delta_release write error %d", rv);
		return rv;
	}

	log_space(sp, "delta_release done %llu %llu %llu",
		  (unsigned long long)leader.owner_id,
		  (unsigned long long)leader.owner_generation,
		  (unsigned long long)leader.timestamp);

	memcpy(leader_ret, &leader, sizeof(struct leader_record));
	return SANLK_OK;
}

int delta_lease_init_host(struct task *task,
		     struct sanlk_lockspace *ls,
		     char *our_host_name,
		     uint64_t generation,
		     uint64_t timestamp,
		     int io_timeout,
		     int try_caw,
		     struct sync_disk *disk)
{
	struct leader_record leader_end;
	struct leader_record leader;
	char *iobuf, **p_iobuf;
	int iobuf_len;
	int sector_size = 0;
	int align_size = 0;
	int max_hosts = 0;
	int write_io_timeout;
	int no_timeout = (ls->flags & SANLK_LSF_NO_TIMEOUT) ? 1 : 0;
	int use_caw = try_caw;
	uint32_t checksum;
	uint64_t new_ts;
	uint64_t offset;
	int rv;

	if (no_timeout && com.create_delta_34)
		return -EINVAL;

	if (!io_timeout)
		io_timeout = com.io_timeout;

	if (timestamp == 0)
		new_ts = LEASE_FREE;
	else if (timestamp == 1)
		new_ts = monotime();
	else
		new_ts = timestamp;

	rv = sizes_from_flags(ls->flags, &sector_size, &align_size, &max_hosts, "LSF");
	if (rv)
		return rv;

	if (!sector_size) {
		/* sector/align flags were not set, use historical defaults */
		sector_size = disk->sector_size;
		align_size = sector_size_to_align_size_old(sector_size);
		max_hosts = DEFAULT_MAX_HOSTS;
	}

	iobuf_len = sector_size;

	p_iobuf = &iobuf;

	rv = posix_memalign((void *)p_iobuf, getpagesize(), iobuf_len);
	if (rv)
		return rv;

	memset(iobuf, 0, iobuf_len);

	memset(&leader, 0, sizeof(struct leader_record));
	leader.magic = DELTA_DISK_MAGIC;
	if (com.create_delta_34)
		leader.version = DELTA_DISK_VERSION_34;
	else if (com.create_delta_40)
		leader.version = DELTA_DISK_VERSION_40;
	else
		leader.version = DELTA_DISK_VERSION_MAJOR | DELTA_DISK_VERSION_MINOR;
	leader.flags = leader_align_flag_from_size(align_size);
	if (no_timeout)
		leader.flags |= LFL_NO_TIMEOUT;
	if (use_caw)
		leader.flags |= LFL_CAW;
	leader.sector_size = sector_size;
	leader.owner_id = ls->host_id;
	leader.owner_generation = generation;
	leader.max_hosts = 1;
	leader.timestamp = new_ts;
	leader.io_timeout = (io_timeout & 0x00FF);
	memcpy(leader.space_name, ls->name, NAME_ID_SIZE);
	if (our_host_name)
		memcpy(leader.resource_name, our_host_name, NAME_ID_SIZE);
	leader.checksum = 0; /* set below */

	leader_record_out(&leader, &leader_end);

	/*
	 * N.B. must compute checksum after the data has been byte swapped.
	 */
	checksum = leader_checksum(&leader_end);
	leader.checksum = checksum;
	leader_end.checksum = cpu_to_le32(checksum);

	memcpy(iobuf, &leader_end, sizeof(struct leader_record));
	
	/*
	 * The io_timeout arg is a part of the lockspace logic, and
	 * determines how the lockspace times out.  The process of
	 * initializing the lease on disk can to use a longer timeout
	 * than the algorithm uses.
	 */
	if (com.write_init_io_timeout)
		write_io_timeout = com.write_init_io_timeout;
	else
		write_io_timeout = io_timeout;

	offset = disk->offset + (sector_size * (ls->host_id - 1));

	rv = write_iobuf(disk->fd, offset, iobuf, iobuf_len, task, task->use_aio, write_io_timeout, NULL);
	if (rv < 0)
		goto out;
 out:
	if (rv != SANLK_AIO_TIMEOUT)
		free(iobuf);

	return rv;
}

/* the host_id lease area begins disk->offset bytes from the start of
   block device disk->path */

int delta_lease_init(struct task *task,
		     struct sanlk_lockspace *ls,
		     int io_timeout,
		     int try_caw,
		     struct sync_disk *disk)
{
	struct leader_record leader_first;
	struct leader_record leader_end;
	struct leader_record leader;
	char *iobuf, **p_iobuf;
	int iobuf_len;
	int sector_size = 0;
	int align_size = 0;
	int max_hosts = 0;
	int write_io_timeout;
	int no_timeout = (ls->flags & SANLK_LSF_NO_TIMEOUT) ? 1 : 0;
	int use_caw = try_caw;
	int i, rv, err;
	uint32_t checksum;

	if (no_timeout && com.create_delta_34)
		return -EINVAL;

	if (!io_timeout)
		io_timeout = com.io_timeout;

	rv = sizes_from_flags(ls->flags, &sector_size, &align_size, &max_hosts, "LSF");
	if (rv)
		return rv;

	if (!sector_size) {
		/* sector/align flags were not set, use historical defaults */
		sector_size = disk->sector_size;
		align_size = sector_size_to_align_size_old(sector_size);
		max_hosts = DEFAULT_MAX_HOSTS;
	}

	if (try_caw) {
		err = compare_and_write_test(task, disk->path, disk->offset, sector_size, com.delta_caw_verify);
		if (err < 0)
			use_caw = 0;

		/* This likely indicates the device supports CAW, but not the FUA/DPO flags */
		if ((err == SANLK_CAW_ILLEGAL) && (com.caw_dpo || com.caw_fua))
			log_warn("Check CAW FUA/DPO settings");
	}

	iobuf_len = align_size;

	p_iobuf = &iobuf;

	rv = posix_memalign((void *)p_iobuf, getpagesize(), iobuf_len);
	if (rv)
		return rv;

	memset(iobuf, 0, iobuf_len);

	/* host_id N is block offset N-1 */

	for (i = 0; i < max_hosts; i++) {
		memset(&leader, 0, sizeof(struct leader_record));
		leader.magic = DELTA_DISK_MAGIC;
		if (com.create_delta_34)
			leader.version = DELTA_DISK_VERSION_34;
		else if (com.create_delta_40)
			leader.version = DELTA_DISK_VERSION_40;
		else
			leader.version = DELTA_DISK_VERSION_MAJOR | DELTA_DISK_VERSION_MINOR;
		leader.flags = leader_align_flag_from_size(align_size);
		if (no_timeout)
			leader.flags |= LFL_NO_TIMEOUT;
		if (use_caw)
			leader.flags |= LFL_CAW;
		leader.sector_size = sector_size;
		leader.max_hosts = 1;
		leader.timestamp = LEASE_FREE;
		leader.io_timeout = (io_timeout & 0x00FF);
		memcpy(leader.space_name, ls->name, NAME_ID_SIZE);
		leader.checksum = 0; /* set below */

		/* make the first record invalid so we can do a single atomic
		   write below to commit the whole thing */
		if (!i) {
			leader.magic = 0;
			memcpy(&leader_first, &leader, sizeof(struct leader_record));
		}

		leader_record_out(&leader, &leader_end);

		/*
		 * N.B. must compute checksum after the data has been byte swapped.
		 */
		checksum = leader_checksum(&leader_end);
		leader.checksum = checksum;
		leader_end.checksum = cpu_to_le32(checksum);

		memcpy(iobuf + (i * sector_size), &leader_end, sizeof(struct leader_record));
	}
	
	/*
	 * The io_timeout arg is a part of the lockspace logic, and
	 * determines how the lockspace times out.  The process of
	 * initializing the lease on disk can to use a longer timeout
	 * than the algorithm uses.
	 */
	if (com.write_init_io_timeout)
		write_io_timeout = com.write_init_io_timeout;
	else
		write_io_timeout = io_timeout;

	rv = write_iobuf(disk->fd, disk->offset, iobuf, iobuf_len, task, task->use_aio, write_io_timeout, NULL);
	if (rv < 0)
		goto out;

	/* commit the whole lockspace by making the first record valid */

	leader_first.magic = DELTA_DISK_MAGIC;
	leader_first.checksum = 0; /* set below */

	leader_record_out(&leader_first, &leader_end);

	/*
	 * N.B. must compute checksum after the data has been byte swapped.
	 */
	checksum = leader_checksum(&leader_end);
	leader_first.checksum = checksum;
	leader_end.checksum = cpu_to_le32(checksum);

	memcpy(iobuf, &leader_end, sizeof(struct leader_record));

	rv = write_iobuf(disk->fd, disk->offset, iobuf, sector_size, task, task->use_aio, write_io_timeout, NULL);
 out:
	if (rv != SANLK_AIO_TIMEOUT)
		free(iobuf);

	return rv;
}

/* 
 * Rewrites all delta leases in the lockspace like delta_lease_init(),
 * but keeping existing leader values while modifying leader flags.
 *
 * TODO: not yet able to change the caw flag on lockspace delta leases.
 * (Also updating resource paxos leases in the lockspace.)
 */

int delta_lease_update_lockspace(struct task *task,
		     struct sanlk_lockspace *ls,
		     int io_timeout,
		     struct sync_disk *disk)
{
	struct leader_record leader_end;
	struct leader_record leader;
	char *iobuf, **p_iobuf;
	int iobuf_len;
	int sector_size = 0;
	int align_size = 0;
	int max_hosts = 0;
	int write_io_timeout;
	int no_timeout = (ls->flags & SANLK_LSF_NO_TIMEOUT) ? 1 : 0;
	int i, rv;
	uint32_t checksum;

	if (no_timeout && com.create_delta_34)
		return -EINVAL;

	if (!io_timeout)
		io_timeout = com.io_timeout;

	rv = sizes_from_flags(ls->flags, &sector_size, &align_size, &max_hosts, "LSF");
	if (rv)
		return rv;

	if (!sector_size || !align_size || !max_hosts) {
		log_error("delta_lease_update_lockspace requires LSF flags for size and alignment.");
		return -EINVAL;
	}

	iobuf_len = align_size;

	p_iobuf = &iobuf;

	rv = posix_memalign((void *)p_iobuf, getpagesize(), iobuf_len);
	if (rv)
		return rv;

	memset(iobuf, 0, iobuf_len);

	rv = read_iobuf(disk->fd, disk->offset, iobuf, iobuf_len, task, task->use_aio, io_timeout, NULL);
	if (rv < 0)
		goto out;

	/* host_id N is block offset N-1 */

	for (i = 0; i < max_hosts; i++) {
		memcpy(&leader_end, iobuf + (i * sector_size), sizeof(struct leader_record));

		leader_record_in(&leader_end, &leader);

		if (leader.magic != DELTA_DISK_MAGIC)
			continue;

		if (no_timeout)
			leader.flags |= LFL_NO_TIMEOUT;
		else
			leader.flags &= ~LFL_NO_TIMEOUT;

		leader.checksum = 0; /* set below */

		leader_record_out(&leader, &leader_end);

		/*
		 * N.B. must compute checksum after the data has been byte swapped.
		 */
		checksum = leader_checksum(&leader_end);
		leader.checksum = checksum;
		leader_end.checksum = cpu_to_le32(checksum);

		memcpy(iobuf + (i * sector_size), &leader_end, sizeof(struct leader_record));
	}
	
	if (com.write_init_io_timeout)
		write_io_timeout = com.write_init_io_timeout;
	else
		write_io_timeout = io_timeout;

	rv = write_iobuf(disk->fd, disk->offset, iobuf, iobuf_len, task, task->use_aio, write_io_timeout, NULL);
	if (rv < 0)
		goto out;

 out:
	if (rv != SANLK_AIO_TIMEOUT)
		free(iobuf);

	return rv;
}

