/*
 * Copyright 2010-2011 Red Hat, Inc.
 *
 * This copyrighted material is made available to anyone wishing to use,
 * modify, copy, or redistribute it subject to the terms and conditions
 * of the GNU General Public License v2 or (at your option) any later version.
 */

#include <inttypes.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <stddef.h>
#include <fcntl.h>
#include <string.h>
#include <errno.h>
#include <limits.h>
#include <time.h>
#include <syslog.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/mman.h>

#include "sanlock_internal.h"
#include "diskio.h"
#include "ondisk.h"
#include "direct.h"
#include "log.h"
#include "lockspace.h"
#include "delta_lease.h"
#include "paxos_lease.h"
#include "resource.h"
#include "timeouts.h"

uint32_t crc32c(uint32_t crc, uint8_t *data, size_t length);
int get_rand(int a, int b);

/*
 * BK_DEBUG_SIZE:  size of buffer to hold ballot debug info,
 *                 this can't be larger than LOG_STR_LEN 512
 * BK_STR_SIZE:    the max length of a dblock string for one host
 * BK_DEBUG_COUNT: the max number of hosts for which we'll copy
 *                 dblock info
 *
 * BK_DEBUG_COUNT * BK_STR_SIZE + extra debug text that comes before
 * the dblock info needs to be less than BK_DEBUG_SIZE.
 * Be very careful about increasing BK_DEBUG_COUNT because the use
 * of strncat depends on it.
 */
#define BK_DEBUG_SIZE 512
#define BK_DEBUG_COUNT 4
#define BK_STR_SIZE 80

static void bk_debug_append(char *bk_debug, char *bk_str)
{
	int off = 0;
	int i, j;

	/* bk_debug buffer is BK_DEBUG_SIZE, bk_str is BK_STR_SIZE */
	/* include as much debugging text in bk_debug as will fit */

	for (i = 0; i < BK_DEBUG_SIZE - 1; i++) {
		if (!bk_debug[i]) {
			off = i;
			break;
		}
	}

	if (off >= (BK_DEBUG_SIZE-1)) {
		bk_debug[BK_DEBUG_SIZE-1] = '\0';
		return;
	}

	for (j = 0; j < strlen(bk_str); j++) {
		bk_debug[off++] = bk_str[j];
		if (off >= BK_DEBUG_SIZE) {
			bk_debug[BK_DEBUG_SIZE-1] = '\0';
			return;
		}
	}
}

/*
static uint32_t roundup_power_of_two(uint32_t val)
{
	val--;
	val |= val >> 1;
	val |= val >> 2;
	val |= val >> 4;
	val |= val >> 8;
	val |= val >> 16;
	val++;
	return val;
}
*/

uint32_t leader_checksum(struct leader_record *lr)
{
	return crc32c((uint32_t)~1, (uint8_t *)lr, LEADER_CHECKSUM_LEN);
}

uint32_t dblock_checksum(struct paxos_dblock *pd)
{
	return crc32c((uint32_t)~1, (uint8_t *)pd, DBLOCK_CHECKSUM_LEN);
}

int paxos_lease_request_read(struct task *task, struct token *token,
			     struct request_record *rr)
{
	struct request_record rr_end;
	int rv;

	/* 1 = request record is second sector */

	rv = read_sector(&token->disks[0], token->sector_size, 1, (char *)&rr_end,
			 sizeof(struct request_record),
			 task, task->use_aio, token->io_timeout, "request");
	if (rv < 0)
		return rv;

	request_record_in(&rr_end, rr);

	return SANLK_OK;
}

int paxos_lease_request_write(struct task *task, struct token *token,
			      struct request_record *rr)
{
	struct request_record rr_end;
	int rv;

	request_record_out(rr, &rr_end);

	rv = write_sector(&token->disks[0], token->sector_size, 1, (char *)&rr_end,
			  sizeof(struct request_record),
			  task, task->use_aio, token->io_timeout, "request");
	if (rv < 0)
		return rv;

	return SANLK_OK;
}

static int write_dblock(struct task *task,
		        struct token *token,
			struct sync_disk *disk,
			uint64_t host_id,
			struct paxos_dblock *pd);

int paxos_erase_dblock(struct task *task,
		       struct token *token,
		       uint64_t host_id)
{
	struct paxos_dblock dblock_end;
	int num_disks = token->r.num_disks;
	int num_writes = 0;
	int d, rv, error = -1;

	memset(&dblock_end, 0, sizeof(struct paxos_dblock));

	for (d = 0; d < num_disks; d++) {
		rv = write_dblock(task, token, &token->disks[d], host_id, &dblock_end);
		if (rv < 0) {
			error = rv;
			continue;
		}
		num_writes++;
	}

	if (!majority_disks(num_disks, num_writes))
		return error;
	return SANLK_OK;
}

/*
 * Write a combined dblock and mblock.  This is an odd case that doesn't fit
 * well with the way the code has been written.  It's used when we want to
 * convert sh to ex, which requires acquiring the lease owner, but we don't
 * want to clobber our SHARED mblock by writing a plain dblock in the process
 * in case there's a problem with the acquiring, we don't want to lose our
 * shared mode lease.
 *
 * NB. this assumes the only mblock flag we want is MBLOCK_SHARED and that
 * the generation we want is token->host_generation.  This is currently
 * the case, but could change in the future.
 */

static int write_dblock_mblock_sh(struct task *task,
			          struct token *token,
			          struct sync_disk *disk,
			          uint64_t host_id,
			          struct paxos_dblock *pd)
{
	struct paxos_dblock pd_end;
	struct mode_block mb;
	struct mode_block mb_end;
	char *iobuf, **p_iobuf;
	uint64_t offset;
	uint32_t checksum;
	int iobuf_len, rv, sector_size;

	memset(&mb, 0, sizeof(mb));
	mb.flags = MBLOCK_SHARED;
	mb.generation = token->host_generation;

	sector_size = token->sector_size;

	iobuf_len = sector_size;
	if (!iobuf_len)
		return -EINVAL;

	p_iobuf = &iobuf;

	rv = posix_memalign((void *)p_iobuf, getpagesize(), iobuf_len);
	if (rv)
		return -ENOMEM;

	offset = disk->offset + ((2 + host_id - 1) * sector_size);

	paxos_dblock_out(pd, &pd_end);

	/*
	 * N.B. must compute checksum after the data has been byte swapped.
	 */
	checksum = dblock_checksum(&pd_end);
	pd->checksum = checksum;
	pd_end.checksum = cpu_to_le32(checksum);

	mode_block_out(&mb, &mb_end);

	memcpy(iobuf, (char *)&pd_end, sizeof(struct paxos_dblock));
	memcpy(iobuf + MBLOCK_OFFSET, (char *)&mb_end, sizeof(struct mode_block));

	rv = write_iobuf(disk->fd, offset, iobuf, iobuf_len, task, task->use_aio, token->io_timeout, NULL);

	if (rv < 0) {
		log_errot(token, "write_dblock_mblock_sh host_id %llu gen %llu rv %d",
			  (unsigned long long)host_id,
			  (unsigned long long)token->host_generation,
			  rv);
	}

	if (rv != SANLK_AIO_TIMEOUT)
		free(iobuf);
	return rv;
}

static int write_dblock(struct task *task,
		        struct token *token,
			struct sync_disk *disk,
			uint64_t host_id,
			struct paxos_dblock *pd)
{
	struct paxos_dblock pd_end;
	uint32_t checksum;
	int rv;

	if (token->flags & T_WRITE_DBLOCK_MBLOCK_SH) {
		/* special case to preserve our SH mode block within the dblock */
		return write_dblock_mblock_sh(task, token, disk, host_id, pd);
	}

	/* 1 leader block + 1 request block;
	   host_id N is block offset N-1 */

	paxos_dblock_out(pd, &pd_end);

	/*
	 * N.B. must compute checksum after the data has been byte swapped.
	 */
	checksum = dblock_checksum(&pd_end);
	pd->checksum = checksum;
	pd_end.checksum = cpu_to_le32(checksum);

	rv = write_sector(disk, token->sector_size, 2 + host_id - 1, (char *)&pd_end, sizeof(struct paxos_dblock),
			  task, task->use_aio, token->io_timeout, "dblock");
	return rv;
}

static int write_leader(struct task *task,
			struct token *token,
			struct sync_disk *disk,
			struct leader_record *lr)
{
	struct leader_record lr_end;
	uint32_t checksum;
	int rv;

	leader_record_out(lr, &lr_end);

	/*
	 * N.B. must compute checksum after the data has been byte swapped.
	 */
	checksum = leader_checksum(&lr_end);
	lr->checksum = checksum;
	lr_end.checksum = cpu_to_le32(checksum);

	rv = write_sector(disk, token->sector_size, 0, (char *)&lr_end, sizeof(struct leader_record),
			  task, task->use_aio, token->io_timeout, "leader");
	return rv;
}

/*
 * NB. this should not be used to write the leader record, it is meant only
 * for manually clobbering the disk to corrupt it for testing, or to manually
 * repair it after it's corrupted.
 */

int paxos_lease_leader_clobber(struct task *task,
			       struct token *token,
			       struct leader_record *leader,
			       const char *caller)
{
	struct leader_record lr_end;
	uint32_t checksum;
	int rv;

	leader_record_out(leader, &lr_end);

	/*
	 * N.B. must compute checksum after the data has been byte swapped.
	 */
	checksum = leader_checksum(&lr_end);
	leader->checksum = checksum;
	lr_end.checksum = cpu_to_le32(checksum);

	rv = write_sector(&token->disks[0], token->sector_size, 0, (char *)&lr_end, sizeof(struct leader_record),
			  task, task->use_aio, token->io_timeout, caller);
	return rv;
}

static int read_dblock(struct task *task,
		       struct token *token,
		       struct sync_disk *disk,
		       uint64_t host_id,
		       struct paxos_dblock *pd)
{
	struct paxos_dblock pd_end;
	int rv;

	/* 1 leader block + 1 request block; host_id N is block offset N-1 */

	rv = read_sector(disk, token->sector_size, 2 + host_id - 1, (char *)&pd_end, sizeof(struct paxos_dblock),
			 task, task->use_aio, token->io_timeout, "dblock");

	paxos_dblock_in(&pd_end, pd);

	return rv;
}

#if 0
static int read_dblocks(struct task *task,
			struct sync_disk *disk,
			struct paxos_dblock *pds,
			int pds_count)
{
	struct paxos_dblock pd_end;
	char *data;
	int data_len, rv, i;

	data_len = pds_count * sector_size;

	data = malloc(data_len);
	if (!data) {
		log_error("read_dblocks malloc %d %s", data_len, disk->path);
		rv = -ENOMEM;
		goto out;
	}

	/* 2 = 1 leader block + 1 request block */

	rv = read_sectors(disk, token->sector_size, 2, pds_count, data, data_len,
			  task, task->use_aio, token->io_timeout, "dblocks");
	if (rv < 0)
		goto out_free;

	/* copy the first N bytes from each sector, where N is size of
	   paxos_dblock */

	for (i = 0; i < pds_count; i++) {
		memcpy(&pd_end, data + (i * sector_size),
		       sizeof(struct paxos_dblock));

		paxos_dblock_in(&pd_end, &pd);

		memcpy(&pds[i], &pd, sizeof(struct paxos_dblock));
	}

	rv = 0;
 out_free:
	free(data);
 out:
	return rv;
}
#endif

static int read_leader(struct task *task,
		       int use_aio,
		       struct token *token,
		       struct sync_disk *disk,
		       struct leader_record *lr,
		       uint32_t *checksum)
{
	struct leader_record lr_end;
	int rv;

	if (!token->sector_size) {
		log_errot(token, "paxos_read_leader with zero sector_size");
		return -EINVAL;
	}

	/* 0 = leader record is first sector */

	rv = read_sector(disk, token->sector_size, 0, (char *)&lr_end, sizeof(struct leader_record),
			 task, use_aio, token->io_timeout, "leader");

	/* N.B. checksum is computed while the data is in ondisk format. */
	*checksum = leader_checksum(&lr_end);

	leader_record_in(&lr_end, lr);

	return rv;
}

static int verify_dblock(struct token *token, struct paxos_dblock *pd, uint32_t checksum)
{
	if (!pd->checksum && !pd->mbal && !pd->bal && !pd->inp && !pd->lver)
		return SANLK_OK;

	if (pd->checksum != checksum) {
		log_errot(token, "verify_dblock wrong checksum %x %x",
			  pd->checksum, checksum);
		return SANLK_DBLOCK_CHECKSUM;
	}

	return SANLK_OK;
}

/*
 * It's possible that we pick a bk_max from another host which has our own
 * inp values in it, and we can end up committing our own inp values, copied
 * from another host's dblock:
 *
 * host2 leader free
 * host2 phase1 mbal 14002
 * host2 writes dblock[1] mbal 14002
 * host2 reads  no higher mbal
 * host2 choose own inp 2,1
 * host2 phase2 mbal 14002 bal 14002 inp 2,1
 * host2 writes dblock[1] bal 14002 inp 2,1
 *                                           host1 leader free
 *                                           host1 phase1 mbal 20001
 *                                           host1 writes dblock[0] mbal 20001
 *                                           host1 reads  no higher mbal
 *                                           host1 choose dblock[1] bal 14002 inp 2,1
 *                                           host1 phase2 mbal 20001 bal 20001 inp 2,1
 *                                           host1 writes dblock[0] bal 20001 inp 2,1
 * host2 reads  dblock[0] mbal 20001 > 14002
 *              abort2, retry
 * host2 leader free
 * host2 phase1 mbal 16002
 * host2 writes dblock[1] mbal 16002
 * host2 reads  dblock[0] mbal 20001 > 16002
 *       abort1 retry
 * host2 leader free
 * host2 phase1 mbal 18002
 * host2 writes dblock[1] mbal 18002
 * host2 reads  dblock[0] mbal 20001 > 18002
 *       abort1 retry
 * host2 leader free
 * host2 phase1 mbal 20002
 * host2 writes dblock[1] mbal 20002
 * host2 reads  no higher mbal
 * host2 choose dblock[0] bal 20001 inp 2,1
 *                                           host1 reads  dblock[1] mbal 20002 > 20001
 *                                                 abort2 retry
 * host2 phase2 mbal 20002 bal 20002 inp 2,1
 * host2 writes dblock[1] bal 20002 inp 2,1
 * host2 reads  no higher mbal
 * host2 commit inp 2,1
 * host2 success
 *                                           host1 leader owner 2,1
 *                                           host1 fail
 */

static int run_ballot(struct task *task, struct token *token, uint32_t flags,
		      int num_hosts, uint64_t next_lver, uint64_t our_mbal,
		      struct paxos_dblock *dblock_out, uint64_t *other_max_mbal, uint64_t *larger_lver)
{
	char bk_debug[BK_DEBUG_SIZE];
	char bk_str[BK_STR_SIZE];
	int bk_debug_count;
	struct paxos_dblock dblock;
	struct paxos_dblock bk_in;
	struct paxos_dblock bk_max;
	struct paxos_dblock *bk_end;
	struct paxos_dblock *bk;
	struct sync_disk *disk;
	char **p_iobuf = NULL;
	char *iobuf = NULL;
	uint32_t checksum;
	int num_disks = token->r.num_disks;
	int num_writes, num_reads;
	int sector_size;
	int read_len;
	int iobuf_len;
	int use_huge = 0;
	int read_mem_align;
	int phase2 = 0;
	int d, q, rv = 0;
	int q_max = -1;
	int error;

	if (num_disks != 1)
		return -1;

	sector_size = token->sector_size;
	/* token->align_size: 1,2,4,8 MB depending on sector_size */
	read_len = token->align_size;   /* remains at 1MB with hugepage */
	iobuf_len = token->align_size;  /* increases to 2MB with hugepage, even with 1MB align_size */
	read_mem_align = getpagesize(); /* increases to 2MB with hugepage, even with 1MB align_size */

	/* Reuse a previously allocated buffer. */
	if (task->paxos_areabuf && (task->paxos_areabuf_size >= iobuf_len)) {
		memset(task->paxos_areabuf, 0, task->paxos_areabuf_size);
		task->paxos_areabuf_used++;
		iobuf = task->paxos_areabuf;
		goto mem_done;
	}

	/* Replace previously allocated buffer with a larger one. */
	if (task->paxos_areabuf && (task->paxos_areabuf_size < iobuf_len)) {
		log_taskw(task, "replacing areabuf with size %u smaller than iobuf_len %u", task->paxos_areabuf_size, iobuf_len);
		free(task->paxos_areabuf);
		task->paxos_areabuf = NULL;
		task->paxos_areabuf_size = 0;
	}

	/* Determine if hugepages is applicable. */
	if (com.use_hugepages && ((iobuf_len == ONE_MB_IN_BYTES) || !(iobuf_len % TWO_MB_IN_BYTES))) {
		read_mem_align = TWO_MB_IN_BYTES;
		if (iobuf_len < TWO_MB_IN_BYTES)
			iobuf_len = TWO_MB_IN_BYTES;
		use_huge = 1;
	}

	/* Allocate a new buffer. */
	p_iobuf = &iobuf;
	rv = posix_memalign((void *)p_iobuf, read_mem_align, iobuf_len);
	if (rv)
		return rv;

	/* Make the new buffer use hugepages. */
	if (use_huge) {
		char *bp;
		madvise(iobuf, iobuf_len, MADV_HUGEPAGE);
		/* allocate the pages */
		for (bp = iobuf; bp < iobuf+iobuf_len; bp += TWO_MB_IN_BYTES)
			memset(bp, 0, 1);
	}

	/* Save the new buffer to reuse if the task doesn't have one. */
	if (!task->paxos_areabuf) {
		task->paxos_areabuf = iobuf;
		task->paxos_areabuf_size = iobuf_len;
		task->paxos_areabuf_created++;
		log_taskd(task, "created areabuf %p size %u %u", iobuf, iobuf_len, task->paxos_areabuf_created);
	}

 mem_done:

	/*
	 * phase 1
	 *
	 * "For each disk d, it tries first to write dblock[p] to disk[d][p]
	 * and then to read disk[d][q] for all other processors q.  It aborts
	 * the ballot if, for any d and q, it finds disk[d][q].mbal >
	 * dblock[p].mbal. The phase completes when p has written and read a
	 * majority of the disks, without reading any block whose mbal
	 * component is greater than dblock[p].mbal."
	 */

	log_token(token, "ballot %llu phase1 write mbal %llu",
		  (unsigned long long)next_lver,
		  (unsigned long long)our_mbal);

	memset(&dblock, 0, sizeof(struct paxos_dblock));
	dblock.mbal = our_mbal;
	dblock.lver = next_lver;
	dblock.checksum = 0; /* set after paxos_dblock_out */

	memset(&bk_max, 0, sizeof(struct paxos_dblock));

	num_writes = 0;

	for (d = 0; d < num_disks; d++) {
		/* acquire io: write 1 (sector) */
		rv = write_dblock(task, token, &token->disks[d], token->host_id, &dblock);
		if (rv < 0)
			continue;
		num_writes++;
	}

	if (!majority_disks(num_disks, num_writes)) {
		log_errot(token, "ballot %llu dblock write error %d",
			  (unsigned long long)next_lver, rv);
		error = SANLK_DBLOCK_WRITE;
		goto out;
	}

	memset(bk_debug, 0, sizeof(bk_debug));
	bk_debug_count = 0;

	num_reads = 0;

	for (d = 0; d < num_disks; d++) {
		disk = &token->disks[d];

		memset(iobuf, 0, read_len);

		/* acquire io: read 2 (lease area) */
		rv = read_iobuf(disk->fd, disk->offset, iobuf, read_len, task, task->use_aio, token->io_timeout, NULL);
		if (rv == SANLK_AIO_TIMEOUT) {
			iobuf = NULL;
			/* areabuf will be freed when aio completes */
			task->paxos_areabuf = NULL;
			task->paxos_areabuf_size = 0;
		}
		if (rv < 0)
			continue;
		num_reads++;

		for (q = 0; q < num_hosts; q++) {
			bk_end = (struct paxos_dblock *)(iobuf + ((2 + q)*sector_size));

			checksum = dblock_checksum(bk_end);

			paxos_dblock_in(bk_end, &bk_in);
			bk = &bk_in;

			if (bk_in.mbal && ((flags & PAXOS_ACQUIRE_DEBUG_ALL) || (bk_in.lver >= dblock.lver))) {
				if (bk_debug_count >= BK_DEBUG_COUNT) {
					log_token(token, "ballot %llu phase1 read %s",
						  (unsigned long long)next_lver, bk_debug);
					memset(bk_debug, 0, sizeof(bk_debug));
					bk_debug_count = 0;
				}

				memset(bk_str, 0, sizeof(bk_str));
				snprintf(bk_str, BK_STR_SIZE, "%d:%llu:%llu:%llu:%llu:%llu:%llu:%x,", q,
					 (unsigned long long)bk_in.mbal,
					 (unsigned long long)bk_in.bal,
					 (unsigned long long)bk_in.inp,
					 (unsigned long long)bk_in.inp2,
					 (unsigned long long)bk_in.inp3,
					 (unsigned long long)bk_in.lver,
					 bk_in.flags);
				bk_str[BK_STR_SIZE-1] = '\0';
				bk_debug_append(bk_debug, bk_str);
				bk_debug_count++;
			}

			rv = verify_dblock(token, bk, checksum);
			if (rv < 0)
				continue;

			check_mode_block(token, next_lver, q, (char *)bk_end);

			if (bk->lver < dblock.lver)
				continue;

			if (bk->lver > dblock.lver) {
				log_warnt(token, "ballot %llu abort1 larger lver in bk[%d] %llu:%llu:%llu:%llu:%llu:%llu "
					  "our dblock %llu:%llu:%llu:%llu:%llu:%llu",
					  (unsigned long long)next_lver, q,
					  (unsigned long long)bk->mbal,
					  (unsigned long long)bk->bal,
					  (unsigned long long)bk->inp,
					  (unsigned long long)bk->inp2,
					  (unsigned long long)bk->inp3,
					  (unsigned long long)bk->lver,
					  (unsigned long long)dblock.mbal,
					  (unsigned long long)dblock.bal,
					  (unsigned long long)dblock.inp,
					  (unsigned long long)dblock.inp2,
					  (unsigned long long)dblock.inp3,
					  (unsigned long long)dblock.lver);

				log_token(token, "ballot %llu phase1 read %s",
					  (unsigned long long)next_lver, bk_debug);

				*larger_lver = bk->lver;
				error = SANLK_DBLOCK_LVER;
				goto out;
			}

			/* see "It aborts the ballot" in comment above */

			if (bk->mbal > dblock.mbal) {
				log_warnt(token, "ballot %llu abort1 larger mbal in bk[%d] %llu:%llu:%llu:%llu:%llu:%llu "
					  "our dblock %llu:%llu:%llu:%llu:%llu:%llu",
					  (unsigned long long)next_lver, q,
					  (unsigned long long)bk->mbal,
					  (unsigned long long)bk->bal,
					  (unsigned long long)bk->inp,
					  (unsigned long long)bk->inp2,
					  (unsigned long long)bk->inp3,
					  (unsigned long long)bk->lver,
					  (unsigned long long)dblock.mbal,
					  (unsigned long long)dblock.bal,
					  (unsigned long long)dblock.inp,
					  (unsigned long long)dblock.inp2,
					  (unsigned long long)dblock.inp3,
					  (unsigned long long)dblock.lver);

				log_token(token, "ballot %llu phase1 read %s",
					  (unsigned long long)next_lver, bk_debug);

				*other_max_mbal = bk->mbal;
				error = SANLK_DBLOCK_MBAL;
				goto out;
			}

			/* see choosing inp for phase 2 in comment below */

			if (!bk->inp)
				continue;

			if (!bk->bal) {
				log_errot(token, "ballot %llu zero bal inp[%d] %llu",
					  (unsigned long long)next_lver, q,
					  (unsigned long long)bk->inp);
				continue;
			}

			if (bk->bal > bk_max.bal) {
				bk_max = *bk;
				q_max = q;
			}
		}
	}

	log_token(token, "ballot %llu phase1 read %s",
		  (unsigned long long)next_lver, bk_debug);

	if (!majority_disks(num_disks, num_reads)) {
		log_errot(token, "ballot %llu dblock read error %d",
			  (unsigned long long)next_lver, rv);
		error = SANLK_DBLOCK_READ;
		goto out;
	}


	/*
	 * "When it completes phase 1, p chooses a new value of dblock[p].inp,
	 * sets dblock[p].bal to dblock[p].mbal (its current ballot number),
	 * and begins phase 2."
	 *
	 * "We now describe how processor p chooses the value of dblock[p].inp
	 * that it tries to commit in phase 2. Let blocksSeen be the set
	 * consisting of dblock[p] and all the records disk[d][q] read by p in
	 * phase 1. Let nonInitBlks be the subset of blocksSeen consisting of
	 * those records whose inp field is not NotAnInput.  If nonInitBlks is
	 * empty, then p sets dblock[p].inp to its own input value input[p].
	 * Otherwise, it sets dblock[p].inp to bk.inp for some record bk in
	 * nonInitBlks having the largest value of bk.bal."
	 */

	if (bk_max.inp) {
		/* lver and mbal are already set */
		dblock.inp = bk_max.inp;
		dblock.inp2 = bk_max.inp2;
		dblock.inp3 = bk_max.inp3;
	} else {
		/* lver and mbal are already set */
		dblock.inp = token->host_id;
		dblock.inp2 = token->host_generation;
		dblock.inp3 = monotime();
	}
	dblock.bal = dblock.mbal;
	dblock.checksum = 0; /* set after paxos_dblock_out */

	if (bk_max.inp) {
		log_token(token, "ballot %llu choose bk_max[%d] lver %llu mbal %llu bal %llu inp %llu %llu %llu",
			  (unsigned long long)next_lver, q_max,
			  (unsigned long long)bk_max.lver,
			  (unsigned long long)bk_max.mbal,
			  (unsigned long long)bk_max.bal,
			  (unsigned long long)bk_max.inp,
			  (unsigned long long)bk_max.inp2,
			  (unsigned long long)bk_max.inp3);
	}


	/*
	 * phase 2
	 *
	 * Same description as phase 1, same sequence of writes/reads.
	 */

	phase2 = 1;

	log_token(token, "ballot %llu phase2 write bal %llu inp %llu %llu %llu q_max %d",
		  (unsigned long long)dblock.lver,
		  (unsigned long long)dblock.bal,
		  (unsigned long long)dblock.inp,
		  (unsigned long long)dblock.inp2,
		  (unsigned long long)dblock.inp3,
		  q_max);

	num_writes = 0;

	for (d = 0; d < num_disks; d++) {
		/* acquire io: write 2 (sector) */
		rv = write_dblock(task, token, &token->disks[d], token->host_id, &dblock);
		if (rv < 0)
			continue;
		num_writes++;
	}

	if (!majority_disks(num_disks, num_writes)) {
		log_errot(token, "ballot %llu our dblock write2 error %d",
			  (unsigned long long)next_lver, rv);
		error = SANLK_DBLOCK_WRITE;
		goto out;
	}

	memset(bk_debug, 0, sizeof(bk_debug));
	bk_debug_count = 0;

	num_reads = 0;

	for (d = 0; d < num_disks; d++) {
		disk = &token->disks[d];

		memset(iobuf, 0, read_len);

		/* acquire io: read 3 (lease area) */
		rv = read_iobuf(disk->fd, disk->offset, iobuf, read_len, task, task->use_aio, token->io_timeout, NULL);
		if (rv == SANLK_AIO_TIMEOUT) {
			iobuf = NULL;
			/* areabuf will be freed when aio completes */
			task->paxos_areabuf = NULL;
			task->paxos_areabuf_size = 0;
		}
		if (rv < 0)
			continue;
		num_reads++;

		for (q = 0; q < num_hosts; q++) {
			bk_end = (struct paxos_dblock *)(iobuf + ((2 + q)*sector_size));

			checksum = dblock_checksum(bk_end);

			paxos_dblock_in(bk_end, &bk_in);
			bk = &bk_in;

			if (bk->mbal && ((flags & PAXOS_ACQUIRE_DEBUG_ALL) || (bk->lver >= dblock.lver))) {
				if (bk_debug_count >= BK_DEBUG_COUNT) {
					log_token(token, "ballot %llu phase2 read %s",
						  (unsigned long long)next_lver, bk_debug);
					memset(bk_debug, 0, sizeof(bk_debug));
					bk_debug_count = 0;
				}

				memset(bk_str, 0, sizeof(bk_str));
				snprintf(bk_str, BK_STR_SIZE, "%d:%llu:%llu:%llu:%llu:%llu:%llu:%x,", q,
					 (unsigned long long)bk->mbal,
					 (unsigned long long)bk->bal,
					 (unsigned long long)bk->inp,
					 (unsigned long long)bk->inp2,
					 (unsigned long long)bk->inp3,
					 (unsigned long long)bk->lver,
					 bk->flags);
				bk_str[BK_STR_SIZE-1] = '\0';
				bk_debug_append(bk_debug, bk_str);
				bk_debug_count++;
			}

			rv = verify_dblock(token, bk, checksum);
			if (rv < 0)
				continue;

			if (bk->lver < dblock.lver)
				continue;

			if (bk->lver > dblock.lver) {
				/*
				 * This happens when we choose another host's bk, that host
				 * acquires the lease itself, releases it, and reacquires it
				 * with a new lver, all before we get here, at which point
				 * we see the larger lver.  I believe case this would always
				 * also be caught the the bk->mbal > dblock.mbal condition
				 * below.
				 */
				log_warnt(token, "ballot %llu abort2 larger lver in bk[%d] %llu:%llu:%llu:%llu:%llu:%llu "
					  "our dblock %llu:%llu:%llu:%llu:%llu:%llu",
					  (unsigned long long)next_lver, q,
					  (unsigned long long)bk->mbal,
					  (unsigned long long)bk->bal,
					  (unsigned long long)bk->inp,
					  (unsigned long long)bk->inp2,
					  (unsigned long long)bk->inp3,
					  (unsigned long long)bk->lver,
					  (unsigned long long)dblock.mbal,
					  (unsigned long long)dblock.bal,
					  (unsigned long long)dblock.inp,
					  (unsigned long long)dblock.inp2,
					  (unsigned long long)dblock.inp3,
					  (unsigned long long)dblock.lver);

				log_token(token, "ballot %llu phase2 read %s",
					  (unsigned long long)next_lver, bk_debug);

				*larger_lver = bk->lver;
				error = SANLK_DBLOCK_LVER;
				goto out;
			}

			/* see "It aborts the ballot" in comment above */

			if (bk->mbal > dblock.mbal) {
				log_warnt(token, "ballot %llu abort2 larger mbal in bk[%d] %llu:%llu:%llu:%llu:%llu:%llu "
					  "our dblock %llu:%llu:%llu:%llu:%llu:%llu",
					  (unsigned long long)next_lver, q,
					  (unsigned long long)bk->mbal,
					  (unsigned long long)bk->bal,
					  (unsigned long long)bk->inp,
					  (unsigned long long)bk->inp2,
					  (unsigned long long)bk->inp3,
					  (unsigned long long)bk->lver,
					  (unsigned long long)dblock.mbal,
					  (unsigned long long)dblock.bal,
					  (unsigned long long)dblock.inp,
					  (unsigned long long)dblock.inp2,
					  (unsigned long long)dblock.inp3,
					  (unsigned long long)dblock.lver);

				log_token(token, "ballot %llu phase2 read %s",
					  (unsigned long long)next_lver, bk_debug);

				*other_max_mbal = bk->mbal;
				error = SANLK_DBLOCK_MBAL;
				goto out;
			}
		}
	}

	log_token(token, "ballot %llu phase2 read %s",
		  (unsigned long long)next_lver, bk_debug);

	if (!majority_disks(num_disks, num_reads)) {
		log_errot(token, "ballot %llu dblock read2 error %d",
			  (unsigned long long)next_lver, rv);
		error = SANLK_DBLOCK_READ;
		goto out;
	}

	/* "When it completes phase 2, p has committed dblock[p].inp." */

	memcpy(dblock_out, &dblock, sizeof(struct paxos_dblock));
	error = SANLK_OK;
 out:
	if (phase2 && (error < 0) &&
	    ((error == SANLK_DBLOCK_READ) || (error == SANLK_DBLOCK_WRITE))) {
		/*
		 * After phase2 we might "win" the ballot even if we don't complete it
		 * because another host could could pick and commit our dblock values.
		 * If we abort the acquire, but are granted the lease, this would leave
		 * us owning the lease on disk.  With this flag, the release path will
		 * try to ensure we are not and do not become the lease owner.
		 */
		token->flags |= T_RETRACT_PAXOS;

		log_errot(token, "ballot %llu retract error %d",
			  (unsigned long long)next_lver, error);
	}

	memcpy(dblock_out, &dblock, sizeof(struct paxos_dblock));
	return error;
}

static void log_leader_error(int result,
			     struct token *token,
			     struct sync_disk *disk,
			     struct leader_record *lr,
			     const char *caller)
{
	log_errot(token, "leader1 %s error %d sn %.48s rn %.48s",
		  caller ? caller : "unknown",
		  result,
		  token->r.lockspace_name,
		  token->r.name);

	log_errot(token, "leader2 path %s offset %llu fd %d",
		  disk->path,
		  (unsigned long long)disk->offset,
		  disk->fd);

	log_errot(token, "leader3 m %x v %x ss %u nh %llu mh %llu oi %llu og %llu lv %llu",
		  lr->magic,
		  lr->version,
		  lr->sector_size,
		  (unsigned long long)lr->num_hosts,
		  (unsigned long long)lr->max_hosts,
		  (unsigned long long)lr->owner_id,
		  (unsigned long long)lr->owner_generation,
		  (unsigned long long)lr->lver);

	log_errot(token, "leader4 sn %.48s rn %.48s ts %llu cs %x",
		  lr->space_name,
		  lr->resource_name,
		  (unsigned long long)lr->timestamp,
		  lr->checksum);

	log_errot(token, "leader5 wi %llu wg %llu wt %llu",
		  (unsigned long long)lr->write_id,
		  (unsigned long long)lr->write_generation,
		  (unsigned long long)lr->write_timestamp);
}

static int _verify_leader(struct token *token,
			 struct sync_disk *disk,
			 struct leader_record *lr,
			 uint32_t checksum,
			 const char *caller,
			 int print_error)
{
	int result;

	if (lr->magic == PAXOS_DISK_CLEAR)
		return SANLK_LEADER_MAGIC;

	if (lr->magic != PAXOS_DISK_MAGIC) {
		result = SANLK_LEADER_MAGIC;
		goto fail;
	}

	if ((lr->version & 0xFFFF0000) != PAXOS_DISK_VERSION_MAJOR) {
		result = SANLK_LEADER_VERSION;
		goto fail;
	}

	if (strncmp(lr->space_name, token->r.lockspace_name, NAME_ID_SIZE)) {
		result = SANLK_LEADER_LOCKSPACE;
		goto fail;
	}

	if (strncmp(lr->resource_name, token->r.name, NAME_ID_SIZE)) {
		result = SANLK_LEADER_RESOURCE;
		goto fail;
	}

	if (lr->num_hosts < token->host_id) {
		result = SANLK_LEADER_NUMHOSTS;
		goto fail;
	}

	if (lr->checksum != checksum) {
		result = SANLK_LEADER_CHECKSUM;
		goto fail;
	}

	return SANLK_OK;

 fail:
	if (!print_error)
		return result;

	switch (result) {
	case SANLK_LEADER_MAGIC:
		log_errot(token, "verify_leader wrong magic %x %s",
			  lr->magic, disk->path);
		break;
	case SANLK_LEADER_VERSION:
		log_errot(token, "verify_leader wrong version %x %s",
			  lr->version, disk->path);
		break;
	case SANLK_LEADER_LOCKSPACE:
		log_errot(token, "verify_leader wrong space name %.48s %.48s %s",
			  lr->space_name, token->r.lockspace_name, disk->path);
		break;
	case SANLK_LEADER_RESOURCE:
		log_errot(token, "verify_leader wrong resource name %.48s %.48s %s",
			  lr->resource_name, token->r.name, disk->path);
		break;
	case SANLK_LEADER_NUMHOSTS:
		log_errot(token, "verify_leader num_hosts too small %llu %llu %s",
			  (unsigned long long)lr->num_hosts,
			  (unsigned long long)token->host_id, disk->path);
		break;
	case SANLK_LEADER_CHECKSUM:
		log_errot(token, "verify_leader wrong checksum %x %x %s",
			  lr->checksum, checksum, disk->path);
		break;
	};

	log_leader_error(result, token, disk, lr, caller);

	/*
	memset(&leader_end, 0, sizeof(struct leader_record));

	rv = read_sector(disk, token->sector_size, 0, (char *)&leader_end,
			 sizeof(struct leader_record),
			 NULL, NO_AIO, 1, "paxos_verify");

	leader_record_in(&leader_end, &leader_rr);

	log_leader_error(rv, token, disk, &leader_rr, "paxos_verify");
	*/

	return result;
}

static int verify_leader(struct token *token,
			 struct sync_disk *disk,
			 struct leader_record *lr,
			 uint32_t checksum,
			 const char *caller)
{
	return _verify_leader(token, disk, lr, checksum, caller, 1);
}

static int verify_leader_no_error(struct token *token,
			 struct sync_disk *disk,
			 struct leader_record *lr,
			 uint32_t checksum,
			 const char *caller)
{
	return _verify_leader(token, disk, lr, checksum, caller, 0);
}

int paxos_verify_leader(struct token *token,
			 struct sync_disk *disk,
			 struct leader_record *lr,
			 uint32_t checksum,
			 const char *caller)
{
	return verify_leader(token, disk, lr, checksum, caller);
}

/* read the lockspace name and resource name given the disk location */

int paxos_read_resource(struct task *task,
			struct token *token,
			struct sanlk_resource *res)
{
	struct leader_record leader;
	uint32_t checksum;
	int align_size;
	int tmp_sector_size = 0;
	int rv;

	memset(&leader, 0, sizeof(struct leader_record));

	/*
	 * We don't know the sector size, so we don't know if we should read
	 * 512 or 4k, but it doesn't matter since the leader record is all that
	 * we need.  It's probably better to read 4k on a 512 disk than to read 512
	 * on a 4k disk, so always do a 4k read.
	 */
	if (!token->sector_size) {
		token->sector_size = 4096;
		token->align_size = sector_size_to_align_size_old(4096);
		tmp_sector_size = 1;
	}

	rv = read_leader(task, task->use_aio, token, &token->disks[0], &leader, &checksum);
	if (rv < 0)
		return rv;

	if (!res->lockspace_name[0])
		memcpy(token->r.lockspace_name, leader.space_name, NAME_ID_SIZE);

	if (!res->name[0])
		memcpy(token->r.name, leader.resource_name, NAME_ID_SIZE);

	if (token->flags & T_CHECK_EXISTS) {
		if (leader.magic != PAXOS_DISK_MAGIC)
			rv = SANLK_LEADER_MAGIC;
		else
			rv = SANLK_OK;
	} else {
		rv = verify_leader_no_error(token, &token->disks[0], &leader, checksum, "read_resource");
	}

	if (rv == SANLK_OK) {
		memcpy(res->lockspace_name, leader.space_name, NAME_ID_SIZE);
		memcpy(res->name, leader.resource_name, NAME_ID_SIZE);
		res->lver = leader.lver;

		if (leader.flags & LFL_CAW)
			res->flags |= SANLK_RES_USING_CAW;

		if ((leader.sector_size == 512) || (leader.sector_size == 4096)) {
			align_size = leader_align_size_from_flag(leader.flags);
			if (!align_size)
				align_size = sector_size_to_align_size_old(leader.sector_size);

			token->sector_size = leader.sector_size;
			token->align_size = align_size;

			/* The flags set by the user may be wrong. */
			sanlk_res_sector_flags_clear(&res->flags);
			sanlk_res_align_flags_clear(&res->flags);

			res->flags |= sanlk_res_sector_size_to_flag(leader.sector_size);
			res->flags |= sanlk_res_align_size_to_flag(align_size);
		} else if (tmp_sector_size) {
			/* we don't know the correct value, so don't set any */
			/* FIXME: add a note about when this can happen */
			token->sector_size = 0;
			token->align_size = 0;
		}
	}

	return rv;
}

int paxos_read_buf(struct task *task,
		   struct token *token,
		   char **buf_out)
{
	char *iobuf, **p_iobuf;
	struct sync_disk *disk = &token->disks[0];
	int rv, iobuf_len;

	if (!token->sector_size || !token->align_size) {
		log_errot(token, "paxos_read_buf with sector_size %d align_size %d",
			  token->sector_size, token->align_size);
		return -EINVAL;
	}

	iobuf_len = token->align_size;
	if (iobuf_len < 0)
		return iobuf_len;

	p_iobuf = &iobuf;

	rv = posix_memalign((void *)p_iobuf, getpagesize(), iobuf_len);
	if (rv)
		return rv;

	memset(iobuf, 0, iobuf_len);

	rv = read_iobuf(disk->fd, disk->offset, iobuf, iobuf_len, task, task->use_aio, token->io_timeout, NULL);

	*buf_out = iobuf;

	return rv;
}

static int _leader_read_one(struct task *task,
			    int use_aio,
			    struct token *token,
			    struct leader_record *leader_ret,
			    const char *caller)
{
	struct leader_record leader;
	uint32_t checksum;
	int rv;

	memset(&leader, 0, sizeof(struct leader_record));

	rv = read_leader(task, use_aio, token, &token->disks[0], &leader, &checksum);
	if (rv < 0)
		return rv;

	rv = verify_leader(token, &token->disks[0], &leader, checksum, caller);

	/* copy what we read even if verify finds a problem */

	memcpy(leader_ret, &leader, sizeof(struct leader_record));
	return rv;
}

int paxos_lease_leader_read(struct task *task,
			    int use_aio,
			    struct token *token,
			    struct leader_record *leader_ret,
			    const char *caller)
{
	int rv;

	rv = _leader_read_one(task, use_aio, token, leader_ret, caller);

	if (rv == SANLK_OK)
		log_token(token, "%s leader %llu owner %llu %llu %llu", caller,
			  (unsigned long long)leader_ret->lver,
			  (unsigned long long)leader_ret->owner_id,
			  (unsigned long long)leader_ret->owner_generation,
			  (unsigned long long)leader_ret->timestamp);

	return rv;
}

int paxos_lease_leader_read_buf(struct task *task,
			    int use_aio,
			    struct token *token,
			    struct leader_record *leader_ret,
			    char *buf,
			    const char *caller)
{
	struct sync_disk *disk = &token->disks[0];
	struct leader_record leader;
	struct leader_record leader_end;
	uint32_t checksum;
	int rv;

	rv = read_iobuf(disk->fd, disk->offset, buf, token->sector_size, task, task->use_aio, token->io_timeout, NULL);
	if (rv < 0)
		return rv;

	memcpy(&leader_end, buf, sizeof(struct leader_record));
	checksum = leader_checksum(&leader_end);
        leader_record_in(&leader_end, &leader);

	rv = verify_leader(token, disk, &leader, checksum, caller);
	if (rv < 0)
		return rv;

	memcpy(leader_ret, &leader, sizeof(struct leader_record));
	return SANLK_OK;
}

static int _lease_read_one(struct task *task,
			   struct token *token,
			   int use_aio,
			   uint32_t flags,
			   struct leader_record *leader_ret,
			   struct paxos_dblock *our_dblock,
			   uint64_t *max_mbal,
			   int *max_q,
			   const char *caller,
			   int log_bk_vals,
			   int *is_caw)
{
	char bk_debug[BK_DEBUG_SIZE];
	char bk_str[BK_STR_SIZE];
	int bk_debug_count;
	struct sync_disk *disk = &token->disks[0];
	struct leader_record leader_end;
	struct paxos_dblock our_dblock_end;
	struct paxos_dblock bk;
	char *iobuf, **p_iobuf;
	uint32_t host_id = token->host_id;
	uint32_t checksum;
	struct paxos_dblock *bk_end;
	uint64_t tmp_mbal = 0;
	int sector_size;
	int read_len;
	int iobuf_len;
	int use_huge = 0;
	int read_mem_align;
	int q, tmp_q = -1, rv;

	/* FIXME: what case is this? */
	if (token->align_size < 0) {
		log_errot(token, "lease_read_one align_size %d", token->align_size);
		return token->align_size;
	}

	sector_size = token->sector_size;
	/* token->align_size: 1,2,4,8 MB depending on sector_size */
	read_len = token->align_size;    /* remains at 1MB with hugepage */
	iobuf_len = token->align_size;   /* increases to 2MB with hugepage, even with 1MB align_size */
	read_mem_align = getpagesize();  /* increases to 2MB with hugepage, even with 1MB align_size */

	/* Reuse a previously allocated buffer. */
	if (task->paxos_areabuf && (task->paxos_areabuf_size >= iobuf_len)) {
		memset(task->paxos_areabuf, 0, task->paxos_areabuf_size);
		iobuf = task->paxos_areabuf;
		task->paxos_areabuf_used++;
		goto mem_done;
	}

	/* Replace previously allocated buffer with a larger one. */
	if (task->paxos_areabuf && (task->paxos_areabuf_size < iobuf_len)) {
		log_taskw(task, "replacing areabuf with size %u smaller than iobuf_len %u", task->paxos_areabuf_size, iobuf_len);
		free(task->paxos_areabuf);
		task->paxos_areabuf = NULL;
		task->paxos_areabuf_size = 0;
	}

	/* Determine if hugepages is applicable. */
	if (com.use_hugepages && ((iobuf_len == ONE_MB_IN_BYTES) || !(iobuf_len % TWO_MB_IN_BYTES))) {
		read_mem_align = TWO_MB_IN_BYTES;
		if (iobuf_len < TWO_MB_IN_BYTES)
			iobuf_len = TWO_MB_IN_BYTES;
		use_huge = 1;
	}

	/* Allocate a new buffer. */
	p_iobuf = &iobuf;
	rv = posix_memalign((void *)p_iobuf, read_mem_align, iobuf_len);
	if (rv)
		return rv;

	/* Make the new buffer use hugepages. */
	if (use_huge) {
		char *bp;
		madvise(iobuf, iobuf_len, MADV_HUGEPAGE);
		/* allocate the pages */
		for (bp = iobuf; bp < iobuf+iobuf_len; bp += TWO_MB_IN_BYTES)
			memset(bp, 0, 1);
	}

	/* Save the new buffer to reuse if the task doesn't have one. */
	if (!task->paxos_areabuf) {
		task->paxos_areabuf = iobuf;
		task->paxos_areabuf_size = iobuf_len;
		task->paxos_areabuf_created++;
		log_taskd(task, "created areabuf %p size %u %u", iobuf, iobuf_len, task->paxos_areabuf_created);
	}

 mem_done:

	rv = read_iobuf(disk->fd, disk->offset, iobuf, read_len, task, use_aio, token->io_timeout, NULL);
	if (rv < 0)
		goto out;

	memcpy(&leader_end, iobuf, sizeof(struct leader_record));
	checksum = leader_checksum(&leader_end);
	leader_record_in(&leader_end, leader_ret);

	rv = verify_leader(token, disk, leader_ret, checksum, caller);
	if (rv < 0)
		goto out;

	if (leader_ret->flags & LFL_CAW) {
		if (is_caw)
			*is_caw = 1;
		goto out;
	}

	memcpy(&our_dblock_end, iobuf + ((host_id + 1) * sector_size), sizeof(struct paxos_dblock));
	paxos_dblock_in(&our_dblock_end, our_dblock);

	memset(bk_debug, 0, sizeof(bk_debug));
	bk_debug_count = 0;

	for (q = 0; q < leader_ret->num_hosts; q++) {
		bk_end = (struct paxos_dblock *)(iobuf + ((2 + q) * sector_size));

		checksum = dblock_checksum(bk_end);

		paxos_dblock_in(bk_end, &bk);

		if (log_bk_vals && bk.mbal &&
		    ((flags & PAXOS_ACQUIRE_DEBUG_ALL) || (bk.lver >= leader_ret->lver))) {
			if (bk_debug_count >= BK_DEBUG_COUNT) {
				log_token(token, "leader %llu dblocks %s",
					  (unsigned long long)leader_ret->lver, bk_debug);
				memset(bk_debug, 0, sizeof(bk_debug));
				bk_debug_count = 0;
			}

			memset(bk_str, 0, sizeof(bk_str));
			snprintf(bk_str, BK_STR_SIZE, "%d:%llu:%llu:%llu:%llu:%llu:%llu:%x,", q,
				 (unsigned long long)bk.mbal,
				 (unsigned long long)bk.bal,
				 (unsigned long long)bk.inp,
				 (unsigned long long)bk.inp2,
				 (unsigned long long)bk.inp3,
				 (unsigned long long)bk.lver,
				 bk.flags);
			bk_str[BK_STR_SIZE-1] = '\0';
			bk_debug_append(bk_debug, bk_str);
			bk_debug_count++;
		}

		rv = verify_dblock(token, &bk, checksum);
		if (rv < 0)
			goto out;

		if (!tmp_mbal || bk.mbal > tmp_mbal) {
			tmp_mbal = bk.mbal;
			tmp_q = q;
		}
	}
	*max_mbal = tmp_mbal;
	*max_q = tmp_q;

	if (log_bk_vals)
		log_token(token, "leader %llu owner %llu %llu %llu dblocks %s",
			  (unsigned long long)leader_ret->lver,
			  (unsigned long long)leader_ret->owner_id,
			  (unsigned long long)leader_ret->owner_generation,
			  (unsigned long long)leader_ret->timestamp,
			  bk_debug);

 out:
	if (rv == SANLK_AIO_TIMEOUT) {
		/* areabuf will be freed when aio completes */
		task->paxos_areabuf = NULL;
		task->paxos_areabuf_size = 0;
	}
	return rv;
}

/*
 * read all the initial values needed to start disk paxos:
 * - the leader record
 * - our own dblock
 * - the max mbal from all dblocks
 *
 * Read the entire lease area in one i/o and copy all those
 * values from it.
 */

int paxos_lease_read(struct task *task, int use_aio, struct token *token, uint32_t flags,
			    struct leader_record *leader_ret,
			    uint64_t *max_mbal, const char *caller, int log_bk_vals,
			    int *is_caw)
{
	struct paxos_dblock our_dblock;
	int rv, q = -1;

	rv = _lease_read_one(task, token, use_aio, flags,
			     leader_ret, &our_dblock, max_mbal, &q, caller, log_bk_vals,
			     is_caw);
	if (rv != SANLK_OK)
		return rv;

	if (is_caw && *is_caw)
		log_token(token, "%s leader %llu owner %llu %llu %llu caw",
			  caller,
			  (unsigned long long)leader_ret->lver,
			  (unsigned long long)leader_ret->owner_id,
			  (unsigned long long)leader_ret->owner_generation,
			  (unsigned long long)leader_ret->timestamp);
	else
		log_token(token, "%s leader %llu owner %llu %llu %llu max mbal[%d] %llu "
			  "our_dblock %llu %llu %llu %llu %llu %llu",
			  caller,
			  (unsigned long long)leader_ret->lver,
			  (unsigned long long)leader_ret->owner_id,
			  (unsigned long long)leader_ret->owner_generation,
			  (unsigned long long)leader_ret->timestamp,
			  q,
			  (unsigned long long)*max_mbal,
			  (unsigned long long)our_dblock.mbal,
			  (unsigned long long)our_dblock.bal,
			  (unsigned long long)our_dblock.inp,
			  (unsigned long long)our_dblock.inp2,
			  (unsigned long long)our_dblock.inp3,
			  (unsigned long long)our_dblock.lver);

	return SANLK_OK;
}

static int write_new_leader(struct task *task,
			    struct token *token,
			    struct leader_record *nl,
			    const char *caller)
{
	int num_disks = token->r.num_disks;
	int num_writes = 0;
	int timeout = 0;
	int rv = 0;
	int d;

	for (d = 0; d < num_disks; d++) {
		rv = write_leader(task, token, &token->disks[d], nl);
		if (rv == SANLK_AIO_TIMEOUT)
			timeout = 1;
		if (rv < 0) 
			continue;
		num_writes++;
	}

	if (!majority_disks(num_disks, num_writes)) {
		log_errot(token, "%s write_new_leader error %d timeout %d owner %llu %llu %llu",
			  caller, rv, timeout,
			  (unsigned long long)nl->owner_id,
			  (unsigned long long)nl->owner_generation,
			  (unsigned long long)nl->timestamp);
		if (timeout)
			return SANLK_AIO_TIMEOUT;
		if (rv < 0)
			return rv;
		return SANLK_LEADER_WRITE;
	}

	return SANLK_OK;
}

void close_token_caw(struct token *token)
{
	if (token->caw_dev_fd) {
		close(token->caw_dev_fd);
		token->caw_dev_fd = 0;
	}
}

int open_token_caw(struct token *token)
{
	char caw_path[PATH_MAX] = { 0 };
	uint32_t disk_major = token->resource->disk_major;
	uint32_t disk_minor = token->resource->disk_minor;
	uint32_t caw_major;
	uint32_t caw_minor;
	uint64_t disk_offset_sect512;
	uint64_t caw_offset_sect512;
	int caw_fd;
	int rv;

	if (token->caw_dev_fd) {
		/* shouldn't happen */
		log_errot(token, "open_token_caw already open %d", token->caw_dev_fd);
		return -1;
	}

 restart:

	if (token->resource->caw_state == CAW_STATE_USE_DISK) {
		if ((caw_fd = open(token->disks[0].path, O_RDWR | O_DIRECT | O_SYNC, 0)) < 0) {
			log_errot(token, "open_token_caw disk %u:%u %s errno %d",
				  disk_major, disk_minor, token->disks[0].path, errno);
			return -1;
		}
		if (!caw_fd) {
			log_errot(token, "open_token_caw disk %u:%u %s fd zero",
				  disk_major, disk_minor, token->disks[0].path);
			return -1;
		}

		token->caw_dev_fd = caw_fd;
		return 0;
	}

	if (token->resource->caw_state == CAW_STATE_FOUND_DEV) {
		caw_major = token->resource->caw_major;
		caw_minor = token->resource->caw_minor;

		if (devno_to_path(caw_major, caw_minor, caw_path) < 0) {
			log_errot(token, "open_token_caw: disk %u:%u devno_to_path %u:%u error",
				  disk_major, disk_minor, caw_major, caw_minor);
			return -1;
		}

		if ((caw_fd = open(caw_path, O_RDWR | O_DIRECT | O_SYNC, 0)) < 0) {
			log_errot(token, "open_token_caw disk %u:%u caw %u:%u %s open errno %d",
				  disk_major, disk_minor, caw_major, caw_minor, caw_path, errno);
			return -1;
		}
		if (!caw_fd) {
			log_errot(token, "open_token_caw disk %u:%u caw %u:%u %s open fd zero",
				  disk_major, disk_minor, caw_major, caw_minor, caw_path);
			return -1;
		}

		token->caw_dev_fd = caw_fd;
		return 0;
	}

	if (token->resource->caw_state == CAW_STATE_UNKNOWN) {
		if (!dev_is_lv(disk_major, disk_minor)) {
			if (dev_is_scsi(disk_major, disk_minor) || dev_is_mpath(disk_major, disk_minor)) {
				token->resource->caw_major = token->resource->disk_major;
				token->resource->caw_minor = token->resource->disk_minor;
				token->resource->caw_offset = token->disks[0].offset;
				token->resource->caw_state = CAW_STATE_USE_DISK;
				goto restart;
			}
			/* we shouldn't get this far */
			log_errot(token, "open_caw_dev disk %u:%u does not support caw", disk_major, disk_minor);
			return -1;
		}

		/*
		 * SGIO compare-and-write must be submitted to a scsi LUN or mpath device.
		 * dm rejects SGIO to an LV, so get the scsi or mpath dev used by the LV
		 * (dm table) at the given offset.
		 */

		/* dm units are always 512 byte sectors */
		disk_offset_sect512 = token->disks[0].offset / 512;

		rv = lv_to_target(disk_major, disk_minor, disk_offset_sect512, &caw_major, &caw_minor, &caw_offset_sect512);
		if (rv < 0) {
			log_errot(token, "open_token_caw: unsupported LV %u:%u error %d", disk_major, disk_minor, rv);
			return -1;
		}

		token->resource->caw_major = caw_major;
		token->resource->caw_minor = caw_minor;
		token->resource->caw_offset = caw_offset_sect512 * 512;
		token->resource->caw_state = CAW_STATE_FOUND_DEV;
		goto restart;
	}

	log_errot(token, "open_caw_dev disk %u:%u invalid state %u", disk_major, disk_minor, token->resource->caw_state);
	return -1;
}

static int token_acquire_compare_and_write(struct task *task, struct token *token, char *area_buf, struct leader_record *leader_end)
{
	char *caw_data;
	char **p_caw_data;
	int sector_size = token->sector_size;
	int rv;

	p_caw_data = &caw_data;
	rv = posix_memalign((void *)p_caw_data, getpagesize(), sector_size * 2);
	if (rv)
		return -ENOMEM;

	memcpy(caw_data, area_buf, sector_size);
	memcpy(caw_data + sector_size, area_buf, sector_size);
	memcpy(caw_data + sector_size, leader_end, sizeof(struct leader_record));

	/* fd will already be open when retried */
	if (!token->caw_dev_fd) {
		rv = open_token_caw(token);
		if (rv < 0) {
			log_errot(token, "paxos_acquire open caw error %d", rv);
			rv = SANLK_CAW_OPEN;
			goto out;
		}
	}

	rv = compare_and_write(task, token->caw_dev_fd, token->io_timeout,
			       token->resource->caw_major, token->resource->caw_minor,
			       sector_size, token->resource->caw_offset, caw_data, NULL, NULL);
out:
	/* on success, keep caw_data so it can be used for release without rereading disk */
	if (!rv)
		token->caw_data = caw_data;
	else
		free(caw_data);
	return rv;
}

/* caw alternative for check_mode_block() */

static void check_mode_blocks(struct token *token, int num_hosts, char *area_buf)
{

	struct mode_block *mb_end;
	struct mode_block mb;
	char *dblock;
	int i;

	for (i = 0; i < num_hosts; i++) {
		dblock = (char *)(area_buf + ((2 + i) * token->sector_size));
		mb_end = (struct mode_block *)(dblock + MBLOCK_OFFSET);

		mode_block_in(mb_end, &mb);

		if (mb.flags & MBLOCK_SHARED) {
			set_id_bit(i + 1, token->shared_bitmap, NULL);
			token->shared_count++;
			log_token(token, "mode_block[%d] shared %d gen %llu",
				  i, token->shared_count,
				  (unsigned long long)mb.generation);
		}
	}
}

/*
 * If we hang or crash after completing a ballot successfully, but before
 * committing the leader_record, then the next host that runs a ballot (with the
 * same lver since we did not commit the new lver to the leader_record) will
 * commit the same inp values that we were about to commit.  If the inp values
 * they commit indicate we (who crashed or hung) are the new owner, then the
 * other hosts will begin monitoring the liveness of our host_id.  Once enough
 * time has passed, they assume we're dead, and go on with new versions.  The
 * "enough time" ensures that if we hung before writing the leader, that we
 * won't wake up and finally write what will then be an old invalid leader.
 */

/*
 * acquire io: disk ios needed to acquire a free and uncontended lease
 *
 * paxos_lease_acquire()
 * R1   paxos_lease_read()	1 read   1 MB (lease area size)
 *      [ delta_lease_leader_read() N reads of 512 bytes (if held) ]
 *      run_ballot()
 * W1           write_dblock()	1 write  512 bytes (1 dblock sector)
 * R2           read_iobuf()	1 read   1 MB (lease area size)
 * W2           write_dblock()  1 write  512 bytes (1 dblock sector)
 * R3           read_iobuf()	1 read   1 MB (lease area size)
 * W3   write_new_leader()	1 write  512 bytes (1 leader sector)
 *
 * 				6 i/os = 3 1MB reads, 3 512 byte writes
 * 				(assuming 512 sector size and 1MB area size)
 */

/*
 * io when using caw
 *
 * acquire ex lease
 * ----------------
 * paxos_lease_acquire()
 *      paxos_lease_read()              1 read   1 MB      (lease area)
 *      [ delta_lease_leader_read()     N reads  512 bytes (delta leader block of owner) ] 
 *      compare_and_write()             1 ioctl  512 bytes (leader block)
 *
 * release ex lease
 * ----------------
 * paxos_lease_release()
 *      compare_and_write()             1 ioctl  512 bytes (leader block)
 *
 *
 *
 * acquire sh lease
 * ----------------
 * paxos_lease_acquire()
 *      paxos_lease_read()              1 read   1 MB      (lease area)
 *      [ delta_lease_leader_read()     N reads  512 bytes (delta leader block of owner) ]
 *      compare_and_write()             1 ioctl  512 bytes (leader block)
 * write_mblock_shared_dblock_release() 1 write  512 bytes (mode block)
 * paxos_lease_release()
 *      compare_and_write()             1 ioctl  512 bytes (leader block)
 *
 * release sh lease
 * ----------------
 * write_mblock_shared_dblock_release() 1 write  512 bytes (mode block)
 */

int paxos_lease_acquire(struct task *task,
			struct token *token,
			uint32_t flags,
		        struct leader_record *leader_ret,
			struct paxos_dblock *dblock_ret,
		        uint64_t acquire_lver,
		        int new_num_hosts)
{
	struct sync_disk host_id_disk;
	struct leader_record host_id_leader;
	struct leader_record cur_leader;
	struct leader_record tmp_leader;
	struct leader_record new_leader;
	struct leader_record leader_end;
	struct paxos_dblock dblock;
	struct paxos_dblock owner_dblock;
	struct host_status hs;
	uint32_t checksum;
	uint64_t wait_start, now;
	uint64_t last_timestamp;
	uint64_t next_lver;
	uint64_t larger_lver = 0;
	uint64_t max_mbal;
	uint64_t num_mbal;
	uint64_t our_mbal;
	uint64_t other_max_mbal;
	uint64_t set_dead_ext_gen = 0;
	int copy_cur_leader;
	int disk_open = 0;
	int error, rv, us;
	int align_size;
	int ls_sector_size;
	int is_caw = 0;
	int other_io_timeout, other_host_dead_seconds;

	memset(&dblock, 0, sizeof(dblock)); /* shut up compiler */

	log_token(token, "paxos_acquire begin offset %llu 0x%x %d %d",
		  (unsigned long long)token->disks[0].offset, flags,
		  token->sector_size, token->align_size);

	if (!token->sector_size) {
		log_errot(token, "paxos_acquire with zero sector_size");
		return -EINVAL;
	}

 restart:
	memset(&tmp_leader, 0, sizeof(tmp_leader));
	copy_cur_leader = 0;

	/*
	 * acquire io: read 1 (lease area)
	 *
	 * We don't know if this lease uses caw until we read the
	 * leader record and look for LFL_CAW.
	 *
	 * paxos_lease_read checks for the LFL_CAW flag, and if found
	 * sets is_caw = 1.  From paxos_lease_read, task->paxos_areabuf
	 * contains the original lease area data that was read.
	 * The caw code below needs the lease area data to check for
	 * mblock shared flags, and for prev_data in compare-and-write.
	 */
	error = paxos_lease_read(task, task->use_aio, token, flags, &cur_leader, &max_mbal, "paxos_acquire", 1, &is_caw);
	if (error < 0)
		goto out;

	align_size = leader_align_size_from_flag(cur_leader.flags);
	if (!align_size)
		align_size = sector_size_to_align_size_old(cur_leader.sector_size);

	/*
	 * token sector_size/align_size are initially set from the lockspace values.
	 * It's possible but unusual that the paxos lease leader record will have
	 * different sector/align sizes than we used initially.
	 */
	if ((cur_leader.sector_size != token->sector_size) ||
	    (align_size != token->align_size)) {
		log_token(token, "paxos_acquire restart with different sizes was %d %d now %d %d",
			  token->sector_size, token->align_size,
			  cur_leader.sector_size, align_size);
		token->sector_size = cur_leader.sector_size;
		token->align_size = align_size;
		goto restart;
	}

	if (is_caw) {
		token->flags |= T_CAW;
		token->resource->flags |= R_CAW;
		if (com.use_compare_and_write == USE_CAW_NO) {
			error = SANLK_CAW_CONFIG;
			goto out;
		}
	}

	if (flags & PAXOS_ACQUIRE_FORCE) {
		copy_cur_leader = 1;
		goto run;
	}

	if (acquire_lver && cur_leader.lver != acquire_lver) {
		log_errot(token, "paxos_acquire acquire_lver %llu cur_leader %llu",
			  (unsigned long long)acquire_lver,
			  (unsigned long long)cur_leader.lver);
		error = SANLK_ACQUIRE_LVER;
		goto out;
	}

	if (cur_leader.timestamp == LEASE_FREE) {
		log_token(token, "paxos_acquire leader %llu free",
			  (unsigned long long)cur_leader.lver);
		copy_cur_leader = 1;
		goto run;
	}

	if (cur_leader.owner_id == token->host_id &&
	    cur_leader.owner_generation == token->host_generation) {
		log_token(token, "paxos_acquire owner %llu %llu %llu is already local %llu %llu",
			  (unsigned long long)cur_leader.owner_id,
			  (unsigned long long)cur_leader.owner_generation,
			  (unsigned long long)cur_leader.timestamp,
			  (unsigned long long)token->host_id,
			  (unsigned long long)token->host_generation);
		copy_cur_leader = 1;
		goto run;
	}

	/*
	 * We were the last host to hold this lease, but in a previous
	 * lockspace generation in which we didn't cleanly release the
	 * paxos lease.
	 */

	if (cur_leader.owner_id == token->host_id &&
	    cur_leader.owner_generation < token->host_generation) {
		log_token(token, "paxos_acquire owner %llu %llu %llu was old local new is %llu",
			  (unsigned long long)cur_leader.owner_id,
			  (unsigned long long)cur_leader.owner_generation,
			  (unsigned long long)cur_leader.timestamp,
			  (unsigned long long)token->host_generation);
		copy_cur_leader = 1;
		goto run;
	}

	/*
	 * Check if current owner is alive based on its host_id renewals.
	 * If the current owner has been dead long enough we can assume that
	 * its watchdog has triggered and we can go for the paxos lease.
	 */

	if (!disk_open) {
		memset(&host_id_disk, 0, sizeof(host_id_disk));

		rv = lockspace_disk(cur_leader.space_name, &host_id_disk, &ls_sector_size);
		if (rv < 0) {
			log_errot(token, "paxos_acquire no lockspace info %.48s",
			  	  cur_leader.space_name);
			error = SANLK_ACQUIRE_LOCKSPACE;
			goto out;
		}
		host_id_disk.fd = -1;

		rv = open_disks_fd(&host_id_disk, 1);
		if (rv < 0) {
			log_errot(token, "paxos_acquire open host_id_disk error %d", rv);
			error = SANLK_ACQUIRE_IDDISK;
			goto out;
		}
		disk_open = 1;
	}

	rv = host_info(cur_leader.space_name, cur_leader.owner_id, &hs);

	if (!rv && hs.last_check && hs.last_live &&
	    hs.owner_id == cur_leader.owner_id &&
	    hs.owner_generation == cur_leader.owner_generation) {
		wait_start = hs.last_live;
		last_timestamp = hs.timestamp;

		log_token(token, "paxos_acquire owner %llu %llu %llu host_info %llu %llu %llu dead_ext %d wait_start %llu",
			  (unsigned long long)cur_leader.owner_id,
			  (unsigned long long)cur_leader.owner_generation,
			  (unsigned long long)cur_leader.timestamp,
			  (unsigned long long)hs.owner_id,
			  (unsigned long long)hs.owner_generation,
			  (unsigned long long)hs.timestamp,
			  hs.dead_ext,
			  (unsigned long long)wait_start);
	} else {
		wait_start = monotime();
		last_timestamp = 0;

		/* Happens if local lockspace has not been running long enough to collect info on other host. */
		log_token(token, "paxos_acquire owner %llu %llu %llu host_info rv %d not ready %llu,%llu,%llu,%llu,%llu,%llu,%u wait_start %llu",
			  (unsigned long long)cur_leader.owner_id,
			  (unsigned long long)cur_leader.owner_generation,
			  (unsigned long long)cur_leader.timestamp,
			  rv,
			  (unsigned long long)hs.owner_id,
			  (unsigned long long)hs.owner_generation,
			  (unsigned long long)hs.timestamp,
			  (unsigned long long)hs.first_check,
			  (unsigned long long)hs.last_check,
			  (unsigned long long)hs.last_live,
			  hs.dead_ext,
			  (unsigned long long)wait_start);
	}

	/*
	 * If an external entity has said that the owner is dead,
	 * then we don't need to use the delta lease timeout of
	 * the owner as the basis for deciding the owner is dead.
	 *
	 * Note: hs from host_info() was last updated from our
	 * previous renewal, and may not reflect the latest
	 * info from the owner's delta lease.
	 */
	if (hs.dead_ext && (hs.owner_generation == cur_leader.owner_generation)) {
		log_token(token, "paxos_acquire owner %llu %llu %llu "
			  "delta %llu %llu %llu host_status is dead_ext",
			  (unsigned long long)cur_leader.owner_id,
			  (unsigned long long)cur_leader.owner_generation,
			  (unsigned long long)cur_leader.timestamp,
			  (unsigned long long)host_id_leader.owner_id,
			  (unsigned long long)host_id_leader.owner_generation,
			  (unsigned long long)host_id_leader.timestamp);
		goto run;
	}

	/*
	 * If the paxos leader shows gen N and host_id leader shows
	 * gen N-1 with DEAD_EXT, then the host's latest delta lease was
	 * clobbered by another host setting DEAD_EXT with the prev gen,
	 * causing the generation to go back one.  Return an error specific
	 * to this case and retry.  Eventually we should see the owner delta
	 * lease gen N with DEAD_EXT or gen N+1.
	 */
	if (hs.dead_ext && (hs.owner_generation < cur_leader.owner_generation))  {
		log_errot(token, "paxos_acquire owner %llu %llu %llu reverted delta %llu %llu %llu dead_ext",
			  (unsigned long long)cur_leader.owner_id,
			  (unsigned long long)cur_leader.owner_generation,
			  (unsigned long long)cur_leader.timestamp,
			  (unsigned long long)host_id_leader.owner_id,
			  (unsigned long long)host_id_leader.owner_generation,
			  (unsigned long long)host_id_leader.timestamp);
		memcpy(leader_ret, &cur_leader, sizeof(struct leader_record));
		return SANLK_ACQUIRE_OWNER_GEN_REVERTED;
	}

	/*
	 * These would have been set by a local entity calling
	 * set_host with dead_ext on the failed host.  If a local
	 * entity has not does this, then eventually we would see
	 * the hs.dead_ext set from another host.
	 */
	if (hs.found_dead_ext_gen)
		set_dead_ext_gen = hs.found_dead_ext_gen;
	else if (hs.wrote_dead_ext_gen)
		set_dead_ext_gen = hs.wrote_dead_ext_gen;

	/*
	 * A local entity called set_host+dead_ext on this host,
	 * so we know it's dead from that, even if our local
	 * renewal thread has not yet seen it and set hs.dead_ext
	 * in check_other_leases.
	 */
	if (set_dead_ext_gen && (set_dead_ext_gen == cur_leader.owner_generation)) {
		log_token(token, "paxos_acquire owner %llu %llu %llu "
			  "delta %llu %llu %llu is dead_ext (found %llu wrote %llu)",
			  (unsigned long long)cur_leader.owner_id,
			  (unsigned long long)cur_leader.owner_generation,
			  (unsigned long long)cur_leader.timestamp,
			  (unsigned long long)host_id_leader.owner_id,
			  (unsigned long long)host_id_leader.owner_generation,
			  (unsigned long long)host_id_leader.timestamp,
			  (unsigned long long)hs.found_dead_ext_gen,
			  (unsigned long long)hs.wrote_dead_ext_gen);
		goto run;
	}

	/*
	 * Loop waiting for the owner of this paxos lease to be dead by timeout.
	 * The paxos lease owner will be dead when the owner's delta lease expires,
	 * indicating that the owner's watchdog has fired.
	 */
	while (1) {
		error = delta_lease_leader_read(task, ls_sector_size, token->io_timeout,
						&host_id_disk,
						cur_leader.space_name,
						cur_leader.owner_id,
						&host_id_leader,
						"paxos_acquire");
		if (error < 0) {
			log_errot(token, "paxos_acquire owner %llu %llu %llu "
				  "delta read %d fd %d path %s off %llu",
				  (unsigned long long)cur_leader.owner_id,
				  (unsigned long long)cur_leader.owner_generation,
				  (unsigned long long)cur_leader.timestamp,
				  error, host_id_disk.fd, host_id_disk.path,
				  (unsigned long long)host_id_disk.offset);
			goto out;
		}

		/* a host_id cannot become free in less than
		   host_dead_seconds after the final renewal because
		   a host_id must first be acquired before being freed,
		   and acquiring cannot take less than host_dead_seconds */

		if (host_id_leader.timestamp == LEASE_FREE) {
			log_token(token, "paxos_acquire owner %llu delta free",
				  (unsigned long long)cur_leader.owner_id);
			goto run;
		}

		/* another host has acquired the host_id of the host that
		   owned this paxos lease; acquiring a host_id also cannot be
		   done in less than host_dead_seconds, or

		   the host_id that owns this lease may be alive, but it
		   owned the lease in a previous generation without freeing it,
		   and no longer owns it */

		if (host_id_leader.owner_id != cur_leader.owner_id ||
		    host_id_leader.owner_generation > cur_leader.owner_generation) {
			log_token(token, "paxos_acquire owner %llu %llu %llu "
				  "delta %llu %llu %llu mismatch",
				  (unsigned long long)cur_leader.owner_id,
				  (unsigned long long)cur_leader.owner_generation,
				  (unsigned long long)cur_leader.timestamp,
				  (unsigned long long)host_id_leader.owner_id,
				  (unsigned long long)host_id_leader.owner_generation,
				  (unsigned long long)host_id_leader.timestamp);
			goto run;
		}

		if (!last_timestamp) {
			last_timestamp = host_id_leader.timestamp;
			goto skip_live_check;
		}

		/*
		 * The local renewal thread has not yet read the owner's delta
		 * lease since DEAD_EXT was set, so hs.dead_ext was not seen above.
		 * But we've read the owner's delta lease here, and see it here.
		 */
		if ((host_id_leader.flags & LFL_DEAD_EXT) &&
		    (host_id_leader.owner_generation == cur_leader.owner_generation)) {
			log_token(token, "paxos_acquire owner %llu %llu %llu "
				  "delta %llu %llu %llu read flag dead_ext",
				  (unsigned long long)cur_leader.owner_id,
				  (unsigned long long)cur_leader.owner_generation,
				  (unsigned long long)cur_leader.timestamp,
				  (unsigned long long)host_id_leader.owner_id,
				  (unsigned long long)host_id_leader.owner_generation,
				  (unsigned long long)host_id_leader.timestamp);
			goto run;
		}

		/*
		 * Check if the owner is alive:
		 *
		 * 1. We just read the delta lease of the owner (host_id_leader).
		 * If that has a newer timestamp than the timestamp last seen by
		 * our own renewal thread (last_timestamp), then the owner is alive.
		 *
		 * 2. If our own renewal thread saw the owner's timestamp change
		 * the last time it was checked, then consider the owner to be alive.
		 *
		 * first_check != last_live: avoid returning IDLIVE when this
		 * checks the host state soon after add_lockspace, before more
		 * than one check_other_leases() has happened.  Before two
		 * checks of the other lease, a dead host and live host look
		 * the same (and first_check == last_live).  If the lease owner
		 * is dead before acquire, then acquire will wait for the lease
		 * to expire.  If the caller doesn't want acquire to wait for a
		 * lease to expire, they include the NOWAIT flag.
		 */

		if ((host_id_leader.timestamp != last_timestamp) ||
		    (hs.last_live && (hs.last_check == hs.last_live) && (hs.first_check != hs.last_live))) {

			log_token(token, "paxos_acquire owner %llu delta %llu %llu %llu alive",
				  (unsigned long long)cur_leader.owner_id,
				  (unsigned long long)host_id_leader.owner_id,
				  (unsigned long long)host_id_leader.owner_generation,
				  (unsigned long long)host_id_leader.timestamp);

			log_token(token, "paxos_acquire alive host info %llu %llu timestamp %llu first_check %llu last_check %llu last_live %llu",
				  (unsigned long long)hs.owner_id,
				  (unsigned long long)hs.owner_generation,
				  (unsigned long long)hs.timestamp,
				  (unsigned long long)hs.first_check,
				  (unsigned long long)hs.last_check,
				  (unsigned long long)hs.last_live);

			memcpy(leader_ret, &cur_leader, sizeof(struct leader_record));

			/* It's possible that the live owner has released the
			   lease, but its release was clobbered by another host
			   that was running the ballot with it and wrote it as
			   the owner.  If the leader writer was not the owner,
			   check if the owner's dblock is cleared.  If so, then
			   the owner released the lease and we can run a
			   ballot.  Comparing the write_id and owner_id is not
			   required; we could always read the owner dblock
			   here, but comparing the writer and owner can
			   eliminate many unnecessary dblock reads. */

			if (!is_caw && (cur_leader.write_id != cur_leader.owner_id)) {
				rv = read_dblock(task, token, &token->disks[0],
						 cur_leader.owner_id, &owner_dblock);
				if (!rv && (owner_dblock.flags & DBLOCK_FL_RELEASED)) {
					/* not an error, but interesting to see */
					log_warnt(token, "paxos_acquire owner %llu %llu %llu writer %llu owner dblock released",
						  (unsigned long long)cur_leader.owner_id,
						  (unsigned long long)cur_leader.owner_generation,
						  (unsigned long long)cur_leader.timestamp,
						  (unsigned long long)cur_leader.write_id);
					goto run;
				}
			}

			error = SANLK_ACQUIRE_IDLIVE;
			goto out;
		}

		/* If the owner hasn't renewed its host_id lease for
		   host_dead_seconds then its watchdog should have fired by
		   now. */

		now = monotime();

		other_io_timeout = hs.io_timeout;
		other_host_dead_seconds = calc_host_dead_seconds(other_io_timeout);

		if (now - wait_start > other_host_dead_seconds) {

			/*
			 * This host would be considered dead if it weren't using NO_TIMEOUT,
			 * and no dead_ext has been set for it yet allowing us to procede.
			 */
			if (host_id_leader.flags & LFL_NO_TIMEOUT) {
				log_token(token, "paxos_acquire owner %llu %llu %llu "
					  "delta %llu %llu %llu no_timeout_dead %llu-%llu>%d",
					  (unsigned long long)cur_leader.owner_id,
					  (unsigned long long)cur_leader.owner_generation,
				          (unsigned long long)cur_leader.timestamp,
				          (unsigned long long)host_id_leader.owner_id,
				          (unsigned long long)host_id_leader.owner_generation,
				          (unsigned long long)host_id_leader.timestamp,
				          (unsigned long long)now,
				          (unsigned long long)wait_start,
				          other_host_dead_seconds);
				memcpy(leader_ret, &cur_leader, sizeof(struct leader_record));
				error = SANLK_ACQUIRE_OWNED_NO_TIMEOUT;
				goto out;
			}

			log_token(token, "paxos_acquire owner %llu %llu %llu "
				  "delta %llu %llu %llu dead %llu-%llu>%d",
				  (unsigned long long)cur_leader.owner_id,
				  (unsigned long long)cur_leader.owner_generation,
				  (unsigned long long)cur_leader.timestamp,
				  (unsigned long long)host_id_leader.owner_id,
				  (unsigned long long)host_id_leader.owner_generation,
				  (unsigned long long)host_id_leader.timestamp,
				  (unsigned long long)now,
				  (unsigned long long)wait_start,
				  other_host_dead_seconds);
			goto run;
		}

		if (flags & PAXOS_ACQUIRE_OWNER_NOWAIT) {
			log_token(token, "paxos_acquire owner %llu %llu %llu no wait",
				  (unsigned long long)cur_leader.owner_id,
				  (unsigned long long)cur_leader.owner_generation,
				  (unsigned long long)cur_leader.timestamp);
			memcpy(leader_ret, &cur_leader, sizeof(struct leader_record));
			error = SANLK_ACQUIRE_OWNED_RETRY;
			goto out;
		}

 skip_live_check:
		/* TODO: test with sleep(2) here */
		sleep(1);

		if (external_shutdown) {
			error = -1;
			goto out;
		}

		/*
		 * In this while loop we are waiting for an indication that the
		 * current owner is alive or dead, but if we see the leader
		 * owner change in the meantime, we'll restart the entire
		 * process.
		 */
		error = paxos_lease_leader_read(task, task->use_aio, token, &tmp_leader, "paxos_acquire");
		if (error < 0)
			goto out;

		if (memcmp(&cur_leader, &tmp_leader, sizeof(struct leader_record))) {
			log_token(token, "paxos_acquire restart leader changed1 from "
				  "%llu %llu %llu to %llu %llu %llu",
				  (unsigned long long)cur_leader.owner_id,
				  (unsigned long long)cur_leader.owner_generation,
				  (unsigned long long)cur_leader.timestamp,
				  (unsigned long long)tmp_leader.owner_id,
				  (unsigned long long)tmp_leader.owner_generation,
				  (unsigned long long)tmp_leader.timestamp);
			goto restart;
		}
	}
	/* end of loop waiting for the owner to be dead */

 run:
	if (is_caw) {
		memcpy(&new_leader, &cur_leader, sizeof(struct leader_record));
		new_leader.lver += 1;
		new_leader.owner_id = token->host_id;
		new_leader.owner_generation = token->host_generation;
		new_leader.timestamp = monotime();

		new_leader.write_id = token->host_id;
		new_leader.write_generation = token->host_generation;
		new_leader.write_timestamp = new_leader.timestamp;

		if (new_num_hosts)
			new_leader.num_hosts = new_num_hosts;
		if (flags & PAXOS_ACQUIRE_SHARED)
			new_leader.flags |= LFL_SHORT_HOLD;
		else
			new_leader.flags &= ~LFL_SHORT_HOLD;
		new_leader.checksum = 0; /* set after leader_record_out */

		leader_record_out(&new_leader, &leader_end);
		checksum = leader_checksum(&leader_end);
		new_leader.checksum = checksum;
		leader_end.checksum = cpu_to_le32(checksum);

		/* acquire io: compare-and-write 1 */
		error = token_acquire_compare_and_write(task, token, task->paxos_areabuf, &leader_end);
		if (error == SANLK_CAW_MISCOMPARE) {
			log_token(token, "paxos_acquire %llu caw miscompare",
				  (unsigned long long)new_leader.lver);
			goto restart;
		}
		if (error < 0) {
			log_errot(token, "paxos_acquire %llu caw error %d",
				  (unsigned long long)new_leader.lver, error);
			goto out;
		}
#if 0
		/* verify compare and write worked */
		usleep(100);
		rv = paxos_lease_leader_read(task, task->use_aio, token, &tmp_leader, "paxos_caw_acquire_verify");
		if (rv < 0) {
			log_errot(token, "paxos_caw_acquire_verify failed to read leader %d", rv);
			goto out;
		}
		if ((tmp_leader.owner_id != token->host_id) ||
		    (tmp_leader.owner_generation != token->host_generation) ||
		    (tmp_leader.timestamp == LEASE_FREE)) {
			log_errot(token, "paxos_caw_acquire_verify wrong leader %llu %llu %llu expect %llu %llu %llu",
				 (unsigned long long)tmp_leader.owner_id,
				 (unsigned long long)tmp_leader.owner_generation,
				 (unsigned long long)tmp_leader.timestamp,
				 (unsigned long long)new_leader.owner_id,
				 (unsigned long long)new_leader.owner_generation,
				 (unsigned long long)new_leader.timestamp);
		}
#endif

		/* caw alternative to run_ballot/check_mode_block */
		check_mode_blocks(token, new_leader.num_hosts, task->paxos_areabuf);

		memcpy(leader_ret, &new_leader, sizeof(struct leader_record));
		error = SANLK_OK;
		goto out;
	}

	/*
	 * Use the disk paxos algorithm to attempt to commit a new leader.
	 *
	 * If we complete a ballot successfully, we can commit a leader record
	 * with next_lver.  If we find a higher mbal during a ballot, we increase
	 * our own mbal and try the ballot again.
	 *
	 * next_lver is derived from cur_leader with a zero or timed out owner.
	 * We need to monitor the leader record to see if another host commits
	 * a new leader_record with next_lver.
	 *
	 * TODO: may not need to increase mbal if dblock.inp and inp2 match
	 * current host_id and generation?
	 */

	/* This next_lver assignment is based on the original cur_leader, not a
	   re-reading of the leader here, i.e. we cannot just re-read the leader
	   here, and make next_lver one more than that.  This is because another
	   node may have made us the owner of next_lver as it is now. */

	next_lver = cur_leader.lver + 1;

	if (!max_mbal) {
		our_mbal = token->host_id;
	} else {
		num_mbal = max_mbal - (max_mbal % cur_leader.max_hosts);
		our_mbal = num_mbal + cur_leader.max_hosts + token->host_id;
	}

 retry_ballot:

	if (copy_cur_leader) {
		/* reusing the initial read removes an iop in the common case */
		copy_cur_leader = 0;
		memcpy(&tmp_leader, &cur_leader, sizeof(struct leader_record));
	} else {
		/* acquire io: read 1 (for retry) */
		error = paxos_lease_leader_read(task, task->use_aio, token, &tmp_leader, "paxos_acquire");
		if (error < 0)
			goto out;
	}

	if (tmp_leader.lver == next_lver) {
		/*
		 * another host has committed a leader_record for next_lver,
		 * check which inp (owner_id) they committed (possibly us).
		 */

		if (tmp_leader.owner_id == token->host_id &&
		    tmp_leader.owner_generation == token->host_generation) {
			/* not a problem, but interesting to see */

			log_warnt(token, "paxos_acquire %llu owner is our inp "
				  "%llu %llu %llu committed by %llu",
				  (unsigned long long)next_lver,
				  (unsigned long long)tmp_leader.owner_id,
				  (unsigned long long)tmp_leader.owner_generation,
				  (unsigned long long)tmp_leader.timestamp,
				  (unsigned long long)tmp_leader.write_id);

			memcpy(leader_ret, &tmp_leader, sizeof(struct leader_record));
			memcpy(dblock_ret, &dblock, sizeof(struct paxos_dblock));
			error = SANLK_OK;
		} else {
			/* not a problem, but interesting to see */

			log_warnt(token, "paxos_acquire %llu owner is %llu %llu %llu",
				  (unsigned long long)next_lver,
				  (unsigned long long)tmp_leader.owner_id,
				  (unsigned long long)tmp_leader.owner_generation,
				  (unsigned long long)tmp_leader.timestamp);

			memcpy(leader_ret, &tmp_leader, sizeof(struct leader_record));
			error = SANLK_ACQUIRE_OWNED;
		}
		goto out;
	}

	if (tmp_leader.lver > next_lver) {
		/*
		 * A case where this was observed: for next_lver 65 we abort1, and delay.
		 * While sleeping, the lease v65 (which was acquired during our abort1) is
		 * released and then reacquired as v66.  When we goto retry_ballot, our
		 * next_lver is 65, but the current lver on disk is 66, causing us to
		 * we fail in the larger1 check.)
		 */
		log_token(token, "paxos_acquire %llu restart new lver %llu from "
			  "%llu %llu %llu to %llu %llu %llu",
			  (unsigned long long)next_lver,
			  (unsigned long long)tmp_leader.lver,
			  (unsigned long long)cur_leader.owner_id,
			  (unsigned long long)cur_leader.owner_generation,
			  (unsigned long long)cur_leader.timestamp,
			  (unsigned long long)tmp_leader.owner_id,
			  (unsigned long long)tmp_leader.owner_generation,
			  (unsigned long long)tmp_leader.timestamp);
		goto restart;
	}

	if (memcmp(&cur_leader, &tmp_leader, sizeof(struct leader_record))) {
		log_token(token, "paxos_acquire %llu restart leader changed2 from "
			  "%llu %llu %llu to %llu %llu %llu",
			  (unsigned long long)next_lver,
			  (unsigned long long)cur_leader.owner_id,
			  (unsigned long long)cur_leader.owner_generation,
			  (unsigned long long)cur_leader.timestamp,
			  (unsigned long long)tmp_leader.owner_id,
			  (unsigned long long)tmp_leader.owner_generation,
			  (unsigned long long)tmp_leader.timestamp);
		goto restart;
	}

	other_max_mbal = 0;

	/*
	 * Another host wrote an lver into their dblock, and then failed before
	 * that lver could be committed to the leader record.  Advance the next
	 * lver past the larger lver last seen in a dblock, otherwise ballots
	 * will continue to be aborted.
	 */
	if (larger_lver > next_lver) {
		log_warnt(token, "paxos_acquire larger lver %llu found beyond next lver %llu",
			  (unsigned long long)larger_lver, (unsigned long long)next_lver);
		next_lver = larger_lver + 1;
	}

	/* acquire io: write 1 (sector), read 2 (lease area), write 2 (sector), read 3 (lease area) */
	error = run_ballot(task, token, flags, cur_leader.num_hosts, next_lver, our_mbal, &dblock, &other_max_mbal, &larger_lver);

	if ((error == SANLK_DBLOCK_MBAL) || (error == SANLK_DBLOCK_LVER)) {
		us = get_rand(0, 1000000);
		if (us < 0)
			us = token->host_id * 100;

		log_token(token, "paxos_acquire %llu retry delay %d us",
			  (unsigned long long)next_lver, us);

		usleep(us);

		/*
		 * If other hosts have advanced max_mbal far beyond where we're at, then
		 * skip ahead to the base max_mbal seen in the last ballot, rather than
		 * adding just adding max_hosts to our_mbal repeatedly until we catch up.
		 * (Note: run_ballot returns the first larger mbal it finds, and does not
		 * scan all dblocks for the largest mbal that exists.  So this doesn't
		 * always skip ahead as far as it could.  All dblocks could be scanned
		 * to find the largest mbal, but it would be a bigger code change.)
		 */
		if (other_max_mbal > (our_mbal + (2 * cur_leader.max_hosts))) {
			uint64_t other_host_id, other_base_mbal, our_next_mbal;
			other_host_id = other_max_mbal % cur_leader.max_hosts;
			other_base_mbal = other_max_mbal - other_host_id;
			our_next_mbal = other_base_mbal + token->host_id;

			log_warnt(token, "paxos_acquire %llu other_max_mbal %llu advance our_mbal from %llu to %llu",
				  (unsigned long long)next_lver,
				  (unsigned long long)other_max_mbal,
				  (unsigned long long)our_mbal,
				  (unsigned long long)our_next_mbal);

			our_mbal = our_next_mbal;
		} else {
			our_mbal += cur_leader.max_hosts;
		}

		goto retry_ballot;
	}

	if (error < 0) {
		log_errot(token, "paxos_acquire %llu ballot error %d",
			  (unsigned long long)next_lver, error);
		goto out;
	}

	/* ballot success, commit next_lver with dblock values */

	memcpy(&new_leader, &cur_leader, sizeof(struct leader_record));
	new_leader.lver = dblock.lver;
	new_leader.owner_id = dblock.inp;
	new_leader.owner_generation = dblock.inp2;
	new_leader.timestamp = dblock.inp3;

	new_leader.write_id = token->host_id;
	new_leader.write_generation = token->host_generation;
	new_leader.write_timestamp = monotime();

	if (new_num_hosts)
		new_leader.num_hosts = new_num_hosts;

	if (new_leader.owner_id == token->host_id) {
		/*
		 * The LFL_SHORT_HOLD flag is just a "hint" to help
		 * other nodes be more intelligent about retrying
		 * due to transient failures when acquiring shared
		 * leases.  Only modify SHORT_HOLD if we're committing
		 * ourself as the new owner.  If we're committing another
		 * host as owner, we don't know if they are acquiring
		 * shared or not.
		 *
		 * When acquiring a shared lease, we acquire the paxos
		 * lease (which is inherently ex), set the shared flag
		 * in our mode_block, then release the paxos lease.
		 * The SHORT flag indicates that this paxos lease will
		 * be released as soon as our mode_block is written.
		 * to hold the lock for a long period (e.g. for an LV.)
		 */
		if (flags & PAXOS_ACQUIRE_SHARED)
			new_leader.flags |= LFL_SHORT_HOLD;
		else
			new_leader.flags &= ~LFL_SHORT_HOLD;
	}

	new_leader.checksum = 0; /* set after leader_record_out */

	/* acquire io: write 3 (sector) */
	error = write_new_leader(task, token, &new_leader, "paxos_acquire");
	if (error < 0) {
		/* See comment in run_ballot about this flag. */
		token->flags |= T_RETRACT_PAXOS;
		memcpy(leader_ret, &new_leader, sizeof(struct leader_record));
		goto out;
	}

	if (new_leader.owner_id != token->host_id) {
		/* not a problem, but interesting to see */

		/* It's possible that we commit an outdated owner id/gen here.
		   If we go back to the top and retry, we may find that the
		   owner host_id is alive but with a newer generation, and
		   we'd be able to get the lease by running the ballot again. */

		log_warnt(token, "ballot %llu commit other owner %llu %llu %llu",
			  (unsigned long long)new_leader.lver,
			  (unsigned long long)new_leader.owner_id,
			  (unsigned long long)new_leader.owner_generation,
			  (unsigned long long)new_leader.timestamp);

		memcpy(leader_ret, &new_leader, sizeof(struct leader_record));
		error = SANLK_ACQUIRE_OTHER;
		goto out;
	}

	log_token(token, "ballot %llu commit self owner %llu %llu %llu",
		  (unsigned long long)next_lver,
		  (unsigned long long)new_leader.owner_id,
		  (unsigned long long)new_leader.owner_generation,
		  (unsigned long long)new_leader.timestamp);

	memcpy(leader_ret, &new_leader, sizeof(struct leader_record));
	memcpy(dblock_ret, &dblock, sizeof(struct paxos_dblock));
	error = SANLK_OK;

 out:
	if (disk_open)
		close_disks(&host_id_disk, 1);

	return error;
}

#if 0
int paxos_lease_renew(struct task *task,
		      struct token *token,
		      struct leader_record *leader_last,
		      struct leader_record *leader_ret)
{
	struct leader_record new_leader;
	int rv, d;
	int error;

	for (d = 0; d < token->r.num_disks; d++) {
		memset(&new_leader, 0, sizeof(struct leader_record));

		rv = read_leader(task, task->use_aio, token, &token->disks[d], &new_leader);
		if (rv < 0)
			continue;

		if (memcmp(&new_leader, leader_last,
			   sizeof(struct leader_record))) {
			log_errot(token, "leader changed between renewals");
			return SANLK_BAD_LEADER;
		}
	}

	new_leader.timestamp = monotime();
	new_leader.checksum = 0; /* set after leader_record_out */

	error = write_new_leader(task, token, &new_leader);
	if (error < 0)
		goto out;

	memcpy(leader_ret, &new_leader, sizeof(struct leader_record));
 out:
	return error;
}
#endif

int paxos_caw_lease_release(struct task *task,
			struct token *token,
			struct sanlk_resource *resrename,
		        struct leader_record *leader_ret)
{
	struct leader_record new_leader;
	struct leader_record leader_end;
	int sector_size = token->sector_size;
	char *iobuf = NULL, **p_iobuf = NULL;
	char *rdbuf = NULL, **p_rdbuf = NULL;
	char *caw_data;
	uint32_t checksum;
	int read_data = 0;
	int not_owned;
	int rv;

	/*
	 * caw_data is allocated by token_acquire_compare_and_write(),
	 * and if the acquire is successful, that caw_data buffer is
	 * saved as token->caw_data to be reused here for release.
	 *
	 * When release is done by resource_thread_release(), then
	 * the token arg is a fake/temp token, and token->caw_data
	 * may be set when the caw_data buffer was saved and passed
	 * from the original token->caw_data.  Sometimes token->caw_data
	 * may be NULL when this is called by resource_thread_release.
	 *
	 * The caw_data buffer is not passed to aio read/write functions
	 * so that it does not need to be reallocated on AIO_TIMEOUT.
	 */

	if (!token->caw_data) {
		p_iobuf = &iobuf;
		rv = posix_memalign((void *)p_iobuf, getpagesize(), sector_size * 2);
		if (rv)
			return -ENOMEM;
		memset(iobuf, 0, sector_size * 2);
		caw_data = iobuf;
		read_data = 1;
	} else {
		caw_data = token->caw_data;

		/* previously written data is in second half, copy to first half */
		memmove(caw_data, caw_data + sector_size, sector_size);

		memcpy(&leader_end, caw_data, sizeof(struct leader_record));
		checksum = leader_checksum(&leader_end);
		leader_record_in(&leader_end, &new_leader);

		rv = verify_leader(token, &token->disks[0], &new_leader, checksum, "paxos_caw_release");
		if (rv < 0) {
			log_token(token, "paxos_caw_release prev caw_data invalid");
			memset(&new_leader, 0, sizeof(new_leader));
			read_data = 1;
		}
	}

	/*
	 * CLEANUP_OWNER: uncertain if we are owner, but release if we are.
	 * If we are not the owner, it's not an error.
	 */
	if (token->flags & T_CLEANUP_OWNER)
		read_data = 1;

 retry:
	if (read_data) {
		log_token(token, "paxos_caw_release reading leader sector");

		if (!rdbuf) {
			p_rdbuf = &rdbuf;
			rv = posix_memalign((void *)p_rdbuf, getpagesize(), sector_size);
			if (rv) {
				rv = -ENOMEM;
				goto out;
			}
		}

		rv = paxos_lease_leader_read_buf(task, task->use_aio, token, &new_leader, rdbuf, "paxos_caw_release");
		if (rv == SANLK_AIO_TIMEOUT) {
			log_errot(token, "paxos_caw_release failed to read leader sector %d", rv);
			rdbuf = NULL;
			goto out;
		}
		if (rv < 0) {
			log_errot(token, "paxos_caw_release failed to read leader sector %d", rv);
			goto out;
		}

		not_owned = ((new_leader.owner_id != token->host_id) ||
			     (new_leader.owner_generation != token->host_generation) ||
			     (new_leader.timestamp == LEASE_FREE));

		if (not_owned && (token->flags & T_CLEANUP_OWNER)) {
			/* CLEANUP_OWNER: we expect that we might not own the lease, so not an error */
			log_warnt(token, "paxos_caw_release cleanup found %llu %llu %llu %llu for token %llu %llu %llu",
				  (unsigned long long)new_leader.owner_id,
				  (unsigned long long)new_leader.owner_generation,
				  (unsigned long long)new_leader.timestamp,
				  (unsigned long long)new_leader.lver,
				  (unsigned long long)token->host_id,
				  (unsigned long long)token->host_generation,
				  (unsigned long long)token->r.lver);
			rv = SANLK_OK;
			goto out;
		}

		if (not_owned) {
			log_errot(token, "paxos_caw_release reread wrong lease state %llu %llu %llu %llu token %llu %llu %llu",
				  (unsigned long long)new_leader.owner_id,
				  (unsigned long long)new_leader.owner_generation,
				  (unsigned long long)new_leader.timestamp,
				  (unsigned long long)new_leader.lver,
				  (unsigned long long)token->host_id,
				  (unsigned long long)token->host_generation,
				  (unsigned long long)token->r.lver);
			rv = SANLK_RELEASE_OWNER;
			goto out;
		}

		/*
		 * the current data has been read into rdbuf,
		 * so update both prev/next parts of caw_data
		 * with rdbuf before udpating the second half
		 * with the new leader.
		 */
		memmove(caw_data, rdbuf, sector_size);
		memmove(caw_data + sector_size, rdbuf, sector_size);
	}

	if (resrename)
		memcpy(new_leader.resource_name, resrename->name, NAME_ID_SIZE);

	new_leader.timestamp = LEASE_FREE;
	new_leader.write_id = token->host_id;
	new_leader.write_generation = token->host_generation;
	new_leader.write_timestamp = monotime();
	new_leader.flags &= ~LFL_SHORT_HOLD;
	new_leader.checksum = 0; /* set after leader_record_out */

	leader_record_out(&new_leader, &leader_end);
	checksum = leader_checksum(&leader_end);
	new_leader.checksum = checksum;
	leader_end.checksum = cpu_to_le32(checksum);

	memcpy(caw_data + sector_size, &leader_end, sizeof(struct leader_record));

	if (!token->caw_dev_fd) {
		rv = open_token_caw(token);
		if (rv < 0) {
			log_errot(token, "paxos_release open caw error %d", rv);
			rv = SANLK_CAW_OPEN;
			goto out;
		}
	}

	/*
	 * This release can race with another host doing compare and write in acquire:
	 * 1. other: acquire read
	 * 2. local: acquire read
	 * 3. local: acquire caw
	 * 4. local: release caw
	 * 5. other: acquire caw
	 *
	 * steps 4 and 5 can race.  step 4 would succeed, step 5 would miscompare.
	 */

	rv = compare_and_write(task, token->caw_dev_fd, token->io_timeout,
			       token->resource->caw_major, token->resource->caw_minor,
			       token->sector_size, token->resource->caw_offset, caw_data, NULL, NULL);
	if (rv == SANLK_CAW_MISCOMPARE) {
		log_errot(token, "paxos_caw_release miscompare new_leader %llu %llu %llu",
			  (unsigned long long)new_leader.owner_id,
			  (unsigned long long)new_leader.owner_generation,
			  (unsigned long long)new_leader.timestamp);

		/*
		 * generally shouldn't happen...
		 * a case where our token->caw_data was not up to date with what's on disk,
		 * e.g. we're trying to release after a failed attempt to acquire?
		 */
		if (!read_data) {
			read_data = 1;
			goto retry;
		}

		if (!rdbuf) {
			rv = SANLK_RELEASE_OWNER;
			goto out;
		}

		/* read here so we can log info about what the state is on disk */
		rv = paxos_lease_leader_read_buf(task, task->use_aio, token, &new_leader, rdbuf, "paxos_caw_release");
		if (rv == SANLK_AIO_TIMEOUT) {
			log_errot(token, "paxos_caw_release miscompare read error %d", rv);
			rdbuf = NULL;
		} else if (rv < 0) {
			log_errot(token, "paxos_caw_release miscompare read error %d", rv);
		} else {
			log_errot(token, "paxos_caw_release miscompare read %llu %llu %llu writer %llu %llu %llu",
				  (unsigned long long)new_leader.owner_id,
				  (unsigned long long)new_leader.owner_generation,
				  (unsigned long long)new_leader.timestamp,
				  (unsigned long long)new_leader.write_id,
				  (unsigned long long)new_leader.write_generation,
				  (unsigned long long)new_leader.write_timestamp);
		}
		rv = SANLK_RELEASE_OWNER;
		goto out;
	}

	if (rv < 0) {
		log_errot(token, "paxos_caw_release compare_and_write error %d", rv);
		goto out;
	}

	log_token(token, "paxos_caw_release done");

	memcpy(leader_ret, &new_leader, sizeof(struct leader_record));
	rv = SANLK_OK;
out:
	/* iobuf/caw_data is not used for aio which might time out */

	/* set to NULL above for io timeout so we don't free it */
	if (rdbuf)
		free(rdbuf);

	/* this function allocated caw_data because the token had none */
	if (iobuf)
		free(iobuf);

	/* caw_data is also freed by free_token in case this function isn't called */
	if (token->caw_data) {
		free(token->caw_data);
		token->caw_data = NULL;
	}

	return rv;
}

int paxos_lease_release(struct task *task,
			struct token *token,
			struct sanlk_resource *resrename,
		        struct leader_record *leader_last,
		        struct leader_record *leader_ret)
{
	struct leader_record leader;
	struct leader_record *last;
	int not_owned;
	int error;

	if (token->flags & T_CAW)
		return paxos_caw_lease_release(task, token, resrename, leader_ret);

	error = paxos_lease_leader_read(task, task->use_aio, token, &leader, "paxos_release");
	if (error < 0) {
		log_errot(token, "paxos_release leader_read error %d", error);
		goto out;
	}

	/*
	 * Used when the caller does not know who the owner is, but
	 * wants to ensure it is not the owner.
	 */
	if (!leader_last)
		last = &leader;
	else
		last = leader_last;

	/*
	 * This will happen when two hosts finish the same ballot
	 * successfully, the second committing the same inp values
	 * that the first did, as it should.  But the second will
	 * write its own write_id/gen/timestamp, which will differ
	 * from what the first host wrote.  So when the first host
	 * rereads here in the release, it will find different
	 * write_id/gen/timestamp from what it wrote.  This is
	 * perfectly fine (use log warn since it's interesting
	 * to see when this happens.)
	 *
	 * If another host was the writer and committed us as the
	 * owner, then we don't zero the leader record when we release,
	 * we just release our dblock (by setting the release flag,
	 * already done prior to calling paxos_lease_release).  This is
	 * because other hosts will ignore our leader record if we were
	 * not the writer once we release our dblock.  Those other
	 * hosts will then run a ballot and commit/write a new leader.
	 * If we are also zeroing the leader, that can race with
	 * another host writing a new leader, and we could clobber the
	 * new leader.
	 */
	if (leader.write_id != token->host_id) {
		log_warnt(token, "paxos_release skip write "
			  "last lver %llu owner %llu %llu %llu writer %llu %llu %llu "
			  "disk lver %llu owner %llu %llu %llu writer %llu %llu %llu",
			  (unsigned long long)last->lver,
			  (unsigned long long)last->owner_id,
			  (unsigned long long)last->owner_generation,
			  (unsigned long long)last->timestamp,
			  (unsigned long long)last->write_id,
			  (unsigned long long)last->write_generation,
			  (unsigned long long)last->write_timestamp,
			  (unsigned long long)leader.lver,
			  (unsigned long long)leader.owner_id,
			  (unsigned long long)leader.owner_generation,
			  (unsigned long long)leader.timestamp,
			  (unsigned long long)leader.write_id,
			  (unsigned long long)leader.write_generation,
			  (unsigned long long)leader.write_timestamp);
		return 0;
	}

	not_owned = ((leader.owner_id != token->host_id) ||
		     (leader.owner_generation != token->host_generation) ||
		     (leader.timestamp == LEASE_FREE));

	/*
	 * CLEANUP_OWNER indicates the caller isn't sure if we are the
	 * owner (likely due to a previous io timeout/error), and wants
	 * to release if we are.
	 */
	if (not_owned && (token->flags & T_CLEANUP_OWNER)) {
		log_warnt(token, "paxos_release cleanup found %llu %llu %llu %llu for token %llu %llu",
			  (unsigned long long)leader.owner_id,
			  (unsigned long long)leader.owner_generation,
			  (unsigned long long)leader.timestamp,
			  (unsigned long long)leader.lver,
			  (unsigned long long)token->host_id,
			  (unsigned long long)token->host_generation);
		return 0;
	}

	/*
	 * When we were the writer of our own leader record, then
	 * releasing the lease includes both setting the RELEASED flag
	 * in our dblock and clearing out timestamp in the leader.
	 * When we reread the leader here in release, we should find
	 * it the same as we last saw in acquire.
	 */

	if (leader.lver != last->lver) {
		log_errot(token, "paxos_release other lver "
			  "last lver %llu owner %llu %llu %llu writer %llu %llu %llu "
			  "disk lver %llu owner %llu %llu %llu writer %llu %llu %llu",
			  (unsigned long long)last->lver,
			  (unsigned long long)last->owner_id,
			  (unsigned long long)last->owner_generation,
			  (unsigned long long)last->timestamp,
			  (unsigned long long)last->write_id,
			  (unsigned long long)last->write_generation,
			  (unsigned long long)last->write_timestamp,
			  (unsigned long long)leader.lver,
			  (unsigned long long)leader.owner_id,
			  (unsigned long long)leader.owner_generation,
			  (unsigned long long)leader.timestamp,
			  (unsigned long long)leader.write_id,
			  (unsigned long long)leader.write_generation,
			  (unsigned long long)leader.write_timestamp);
		return SANLK_RELEASE_LVER;
	}

	if (leader.timestamp == LEASE_FREE) {
		log_errot(token, "paxos_release already free "
			  "last lver %llu owner %llu %llu %llu writer %llu %llu %llu "
			  "disk lver %llu owner %llu %llu %llu writer %llu %llu %llu",
			  (unsigned long long)last->lver,
			  (unsigned long long)last->owner_id,
			  (unsigned long long)last->owner_generation,
			  (unsigned long long)last->timestamp,
			  (unsigned long long)last->write_id,
			  (unsigned long long)last->write_generation,
			  (unsigned long long)last->write_timestamp,
			  (unsigned long long)leader.lver,
			  (unsigned long long)leader.owner_id,
			  (unsigned long long)leader.owner_generation,
			  (unsigned long long)leader.timestamp,
			  (unsigned long long)leader.write_id,
			  (unsigned long long)leader.write_generation,
			  (unsigned long long)leader.write_timestamp);
		return SANLK_RELEASE_OWNER;
	}

	if (leader.owner_id != token->host_id ||
	    leader.owner_generation != token->host_generation) {
		log_errot(token, "paxos_release other owner "
			  "last lver %llu owner %llu %llu %llu writer %llu %llu %llu "
			  "disk lver %llu owner %llu %llu %llu writer %llu %llu %llu",
			  (unsigned long long)last->lver,
			  (unsigned long long)last->owner_id,
			  (unsigned long long)last->owner_generation,
			  (unsigned long long)last->timestamp,
			  (unsigned long long)last->write_id,
			  (unsigned long long)last->write_generation,
			  (unsigned long long)last->write_timestamp,
			  (unsigned long long)leader.lver,
			  (unsigned long long)leader.owner_id,
			  (unsigned long long)leader.owner_generation,
			  (unsigned long long)leader.timestamp,
			  (unsigned long long)leader.write_id,
			  (unsigned long long)leader.write_generation,
			  (unsigned long long)leader.write_timestamp);
		return SANLK_RELEASE_OWNER;
	}

	if (memcmp(&leader, last, sizeof(struct leader_record))) {
		log_errot(token, "paxos_release different vals "
			  "last lver %llu owner %llu %llu %llu writer %llu %llu %llu "
			  "disk lver %llu owner %llu %llu %llu writer %llu %llu %llu",
			  (unsigned long long)last->lver,
			  (unsigned long long)last->owner_id,
			  (unsigned long long)last->owner_generation,
			  (unsigned long long)last->timestamp,
			  (unsigned long long)last->write_id,
			  (unsigned long long)last->write_generation,
			  (unsigned long long)last->write_timestamp,
			  (unsigned long long)leader.lver,
			  (unsigned long long)leader.owner_id,
			  (unsigned long long)leader.owner_generation,
			  (unsigned long long)leader.timestamp,
			  (unsigned long long)leader.write_id,
			  (unsigned long long)leader.write_generation,
			  (unsigned long long)leader.write_timestamp);
		return SANLK_RELEASE_OWNER;
	}

	if (resrename)
		memcpy(leader.resource_name, resrename->name, NAME_ID_SIZE);

	leader.timestamp = LEASE_FREE;
	leader.write_id = token->host_id;
	leader.write_generation = token->host_generation;
	leader.write_timestamp = monotime();
	leader.flags &= ~LFL_SHORT_HOLD;
	leader.checksum = 0; /* set after leader_record_out */

	error = write_new_leader(task, token, &leader, "paxos_release");
	if (error < 0)
		goto out;

	memcpy(leader_ret, &leader, sizeof(struct leader_record));
 out:
	return error;
}

int paxos_lease_init(struct task *task,
		     struct token *token,
		     int num_hosts, int write_clear, int try_caw)
{
	char *iobuf, **p_iobuf;
	struct leader_record leader;
	struct leader_record leader_end;
	struct request_record rr;
	struct request_record rr_end;
	uint32_t checksum;
	int iobuf_len;
	int sector_size = 0;
	int align_size = 0;
	int max_hosts = 0;
	int aio_timeout = 0;
	int write_io_timeout = 0;
	int use_caw = try_caw;
	int rv, d;

	rv = sizes_from_flags(token->r.flags, &sector_size, &align_size, &max_hosts, "RES");
	if (rv)
		return rv;

	if (!sector_size) {
		/* sector/align flags were not set, use historical defaults */
		sector_size = token->disks[0].sector_size;
		align_size = sector_size_to_align_size_old(sector_size);
		max_hosts = DEFAULT_MAX_HOSTS;
	}

	if (try_caw && (compare_and_write_test(task, token->disks[0].path, token->disks[0].offset, sector_size, com.paxos_caw_verify) < 0))
		use_caw = 0;

	if (!num_hosts || (num_hosts > max_hosts))
		num_hosts = max_hosts;

	token->sector_size = sector_size;
	token->align_size = align_size;

	iobuf_len = align_size;

	p_iobuf = &iobuf;

	rv = posix_memalign((void *)p_iobuf, getpagesize(), iobuf_len);
	if (rv)
		return rv;

	memset(iobuf, 0, iobuf_len);

	memset(&leader, 0, sizeof(leader));

	if (write_clear) {
		leader.magic = PAXOS_DISK_CLEAR;
		leader.write_timestamp = monotime();
	} else {
		leader.magic = PAXOS_DISK_MAGIC;
	}

	if (com.create_paxos_64)
		leader.version = PAXOS_DISK_VERSION_64;
	else
		leader.version = PAXOS_DISK_VERSION_MAJOR | PAXOS_DISK_VERSION_MINOR;
	leader.flags = leader_align_flag_from_size(align_size);
	if (use_caw)
		leader.flags |= LFL_CAW;
	leader.sector_size = sector_size;
	leader.num_hosts = num_hosts;
	leader.max_hosts = max_hosts;
	leader.timestamp = LEASE_FREE;
	memcpy(leader.space_name, token->r.lockspace_name, NAME_ID_SIZE);
	memcpy(leader.resource_name, token->r.name, NAME_ID_SIZE);
	leader.checksum = 0; /* set after leader_record_out */

	memset(&rr, 0, sizeof(rr));
	rr.magic = REQ_DISK_MAGIC;
	rr.version = REQ_DISK_VERSION_MAJOR | REQ_DISK_VERSION_MINOR;

	leader_record_out(&leader, &leader_end);

	/*
	 * N.B. must compute checksum after the data has been byte swapped.
	 */
	checksum = leader_checksum(&leader_end);
	leader.checksum = checksum;
	leader_end.checksum = cpu_to_le32(checksum);

	request_record_out(&rr, &rr_end);

	memcpy(iobuf, &leader_end, sizeof(struct leader_record));
	memcpy(iobuf + sector_size, &rr_end, sizeof(struct request_record));

	/*
	 * The process of initializing the lease on disk can use a
	 * longer timeout than the algorithm uses.
	 */
	if (com.write_init_io_timeout)
		write_io_timeout = com.write_init_io_timeout;

	for (d = 0; d < token->r.num_disks; d++) {
		if (!write_io_timeout)
			write_io_timeout = token->io_timeout;

		rv = write_iobuf(token->disks[d].fd, token->disks[d].offset,
				 iobuf, iobuf_len, task, task->use_aio, write_io_timeout, NULL);

		if (rv == SANLK_AIO_TIMEOUT)
			aio_timeout = 1;

		if (rv < 0) {
			if (!aio_timeout) {
				free(iobuf);
				iobuf = NULL;
			}
			return rv;
		}
	}

	if (!aio_timeout)
		free(iobuf);

	return 0;
}
