/*
 * Copyright 1999-2006 University of Chicago
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "globus_i_xio_http.h"
#include <limits.h>

#ifndef GLOBUS_DONT_DOCUMENT_INTERNAL
/**
 * @defgroup globus_i_xio_http_transform Internal Transform Implementation
 * @brief Internal Transform Implementation
 */
#endif

globus_mutex_t                          globus_i_xio_http_cached_handle_mutex;
globus_mutex_t                          globus_i_xio_http_cancel_mutex;
globus_list_t *                         globus_i_xio_http_cached_handles = NULL;
globus_list_t *                         globus_i_xio_http_cancellable_handles = NULL;
static
globus_i_xio_http_handle_t *
globus_l_xio_http_find_cached_handle(
    globus_i_xio_http_target_t *        target,
    globus_i_xio_http_attr_t *          attr);

static
globus_result_t
globus_l_xio_http_reopen(
    globus_i_xio_http_handle_t *        http_handle,
    globus_i_xio_http_target_t *        http_target,
    globus_i_xio_http_attr_t *          http_attr,
    globus_xio_operation_t              op);

static
globus_result_t
globus_l_xio_http_open(
    const globus_xio_contact_t *        contact_info,
    void *                              target,
    void *                              attr,
    globus_xio_operation_t              op);

static
void
globus_l_xio_http_read_callback(
    globus_xio_operation_t              op,
    globus_result_t                     result,
    globus_size_t                       nbytes,
    void *                              user_arg);

static
globus_result_t
globus_l_xio_http_parse_chunk_header(
    globus_i_xio_http_handle_t *        http_handle,
    globus_bool_t *                     done);

static
void
globus_l_xio_http_read_chunk_header_callback(
    globus_xio_operation_t              op,
    globus_result_t                     result,
    globus_size_t                       nbytes,
    void *                              user_arg);

static
void
globus_l_xio_http_copy_residue(
    globus_i_xio_http_handle_t *        http_handle);

static
void
globus_l_xio_http_client_cache_kickout(
    void *                              user_arg);

static
void
globus_l_xio_http_read_cancel_callback(
    globus_xio_operation_t              op,
    void *                              user_arg,
    globus_xio_error_type_t             reason);

static
void
globus_l_xio_http_read_timeout_callback(
    void *                              user_arg);

/**
 * Open an HTTP URI
 * @ingroup globus_i_xio_http_transform 
 *
 * Opens a new connection to handle an HTTP request. Allocates a handle
 * and then passes the open on to the transport. In the callback called
 * by the transport-level open, the HTTP driver will begin sending or
 * receiving the HTTP request metadata.
 *
 * @param target
 *     HTTP-specific target contact information.
 * @param attr
 *     HTTP attributes to associate with this open. On the client side,
 *     this will allow overriding some information in the target.
 * @param op
 *     XIO Operation associated with this open.
 *
 * @returns This function directly returns GLOBUS_SUCCESS or
 * GLOBUS_XIO_ERROR_MEMORY. Other errors may be generated by
 * globus_i_xio_http_handle_init() or globus_xio_driver_pass_open().
 *
 * @retval GLOBUS_SUCCESS
 *     Open successfully passed to transport.
 * @retval GLOBUS_XIO_ERROR_MEMORY
 *     Unable to allocate an HTTP handle.
 */
globus_result_t
globus_i_xio_http_open(
    const globus_xio_contact_t *        contact_info,
    void *                              link,
    void *                              attr,
    globus_xio_operation_t              op)
{
    globus_result_t                     result;
    globus_i_xio_http_target_t *        target = NULL;
    globus_i_xio_http_handle_t *        http_handle;
    GlobusXIOName(globus_i_xio_http_open);
    
    if(link)
    {
        target = (globus_i_xio_http_target_t *) link;
    }
    else
    {
        result = globus_i_xio_http_target_init(&target, contact_info);
        if(result != GLOBUS_SUCCESS)
        {
            goto error;
        }
    }
    
    http_handle = globus_l_xio_http_find_cached_handle(target, attr);

    if (http_handle != NULL)
    {
        result = globus_l_xio_http_reopen(http_handle, target, attr, op);
    }
    else
    {
        result = globus_l_xio_http_open(contact_info, target, attr, op);
    }
    
    if(!link && target)
    {
        globus_i_xio_http_target_destroy(target);
    }

error:
    return result;
}
/* globus_i_xio_http_open() */

static
globus_result_t
globus_l_xio_http_reopen(
    globus_i_xio_http_handle_t *        http_handle,
    globus_i_xio_http_target_t *        http_target,
    globus_i_xio_http_attr_t *          http_attr,
    globus_xio_operation_t              op)
{
    globus_result_t                     result;

    globus_mutex_lock(&http_handle->mutex);

    if (http_handle->pending_error)
    {
        result = globus_error_put(
            globus_object_copy(http_handle->pending_error));
        goto error_exit;
    }

    globus_assert(http_handle->target_info.is_client == http_target->is_client);
    globus_assert(http_handle->request_info.http_version
            == GLOBUS_XIO_HTTP_VERSION_1_1);

    result = globus_i_xio_http_handle_reinit(
            http_handle,
            http_attr,
            http_target);
    if (result != GLOBUS_SUCCESS)
    {
        goto error_exit;
    }
    result = globus_xio_driver_merge_handle(
            op,
            http_handle->handle);
    if (result != GLOBUS_SUCCESS)
    {
        goto error_exit;
    }
    globus_assert(http_target->is_client);

    http_handle->send_state = GLOBUS_XIO_HTTP_REQUEST_LINE;
    http_handle->parse_state = GLOBUS_XIO_HTTP_STATUS_LINE;
    http_handle->reopen_in_progress = GLOBUS_TRUE;

    result = globus_i_xio_http_client_write_request(op, http_handle);
error_exit:
    globus_mutex_unlock(&http_handle->mutex);

    return result;
}
/* globus_l_xio_http_reopen() */

static
globus_result_t
globus_l_xio_http_open(
    const globus_xio_contact_t *        contact_info,
    void *                              target,
    void *                              attr,
    globus_xio_operation_t              op)
{
    globus_result_t                     result;
    globus_i_xio_http_handle_t *        http_handle;
    globus_i_xio_http_attr_t *          http_attr = attr;
    globus_xio_driver_callback_t        open_callback;
    globus_xio_contact_t                new_contact_info;
    char                                port_buf[12];
    GlobusXIOName(globus_l_xio_http_open);

    http_handle = globus_libc_calloc(1, sizeof(globus_i_xio_http_handle_t));

    if (http_handle == NULL)
    {
        result = GlobusXIOErrorMemory("http_handle");

        goto error_exit;
    }
    result = globus_i_xio_http_handle_init(
            http_handle,
            http_attr,
            target);

    if (result != GLOBUS_SUCCESS)
    {
        goto free_http_handle_exit;

        return result;
    }

    if (http_handle->target_info.is_client)
    {
        open_callback = globus_i_xio_http_client_open_callback;
        http_handle->send_state = GLOBUS_XIO_HTTP_PRE_REQUEST_LINE;

        if (http_handle->request_info.http_version
                == GLOBUS_XIO_HTTP_VERSION_1_0)
        {
            if (!GLOBUS_I_XIO_HTTP_HEADER_IS_CONTENT_LENGTH_SET(
                        &http_handle->request_info.headers) &&
                !http_handle->delay_write_header)
            {
                result = GlobusXIOHttpErrorInvalidHeader(
                        "Content-Length",
                        "not set");

                goto destroy_http_handle;
            }
        }
    }
    else
    {
        open_callback = globus_i_xio_http_server_open_callback;
        http_handle->send_state = GLOBUS_XIO_HTTP_STATUS_LINE;
    }
    
    memcpy(&new_contact_info, contact_info, sizeof(new_contact_info));
    snprintf(port_buf, sizeof(port_buf), "%hu", http_handle->target_info.port);
    new_contact_info.port = port_buf;
    
    http_handle->handle = globus_xio_operation_get_driver_handle(op);

    result = globus_xio_driver_pass_open(
        op,
        &new_contact_info,
        open_callback,
        http_handle);

    if (result != GLOBUS_SUCCESS)
    {
        goto destroy_http_handle;
    }
    return result;

destroy_http_handle:
    globus_i_xio_http_handle_destroy(http_handle);
free_http_handle_exit:
    globus_libc_free(http_handle);
error_exit:
    return result;
}
/* globus_l_xio_http_open() */

static
globus_i_xio_http_handle_t *
globus_l_xio_http_find_cached_handle(
    globus_i_xio_http_target_t *        http_target,
    globus_i_xio_http_attr_t *          http_attr)
{
    globus_i_xio_http_handle_t *        http_handle = NULL;
    globus_list_t *                     iter;

    if (http_attr == NULL)
    {
        return NULL;
    }
    if (http_attr->request.http_version == GLOBUS_XIO_HTTP_VERSION_1_0 ||
        !http_target->is_client)
    {
        return NULL;
    }

    globus_mutex_lock(&globus_i_xio_http_cached_handle_mutex);
    iter = globus_i_xio_http_cached_handles;
    while (! globus_list_empty(iter))
    {
        http_handle = globus_list_first(iter);

        if (strcmp(http_target->host, http_handle->target_info.host) == 0 &&
            http_target->port == http_handle->target_info.port)
        {
            globus_list_remove(&globus_i_xio_http_cached_handles, iter);

            break;
        }
        else
        {
            iter = globus_list_rest(iter);
        }
        http_handle = NULL;
    }
    globus_mutex_unlock(&globus_i_xio_http_cached_handle_mutex);

    return http_handle;
}
/* globus_l_xio_http_find_cached_handle() */

/**
 * Read from an HTTP stream
 * @ingroup globus_i_xio_http_transform
 *
 * Begins processing a read. If this is called before the headers have been
 * parsed, then the information pertaining to the read is just stored to
 * be processed later. Otherwise globus_i_xio_http_parse_residue() is called
 * to handle any residual data in the header buffer and pass the read
 * to the transport if necessary.
 *
 * If there is sufficient data in the buffer residue to satisfy the "wait_for"
 * value passed to the read function, then this will cause
 * globus_xio_driver_finished_read() to be called. Otherwise, the driver
 * will pass the read to the transport driver.
 *
 * @param handle
 *     Void pointer to a globus_i_xio_http_handle_t.
 * @param iovec
 *     Pointer to user's iovec array. Can't be changed, so a copy is made
 *     in the http_handle.
 * @param iovec_count
 *     Length of the iovec array.
 * @param op
 *     Operation associated with the read.
 *
 * @return
 *     This function returns GLOBUS_SUCCESS, 
 *     GLOBUS_XIO_ERROR_ALREADY_REGISTERED, and GLOBUS_XIO_ERROR_EOF
 *     errors directly.
 *
 * @retval GLOBUS_SUCCESS
 *     Read handled successfully. Either the read was passed to the transport,
 *     or globus_xio_driver_finished_read() was be called.
 * @retval GLOBUS_XIO_ERROR_ALREADY_REGISTERED
 *     A read is already registered with the HTTP handle, so we can't deal
 *     with this new one.
 * @retval GLOBUS_XIO_ERROR_EOF
 *     A read is being registered on a handle which won't have any more
 *     entity body available (either none was present or the entire content
 *     length or final chunk has been read).
 */
globus_result_t
globus_i_xio_http_read(
    void *                              handle,
    const globus_xio_iovec_t *          iovec,
    int                                 iovec_count,
    globus_xio_operation_t              op)
{
    globus_i_xio_http_handle_t *        http_handle = handle;
    globus_result_t                     result = GLOBUS_SUCCESS;
    globus_i_xio_http_header_info_t *   header_info;
    globus_size_t                       nbytes;
    globus_bool_t                       registered_again = GLOBUS_FALSE;
    int                                 i;
    globus_i_xio_http_attr_t *          descriptor;
    GlobusXIOName(globus_i_xio_http_read);

    if (http_handle->target_info.is_client)
    {
        header_info = &http_handle->response_info.headers;
    }
    else
    {
        header_info = &http_handle->request_info.headers;
    }

    globus_mutex_lock(&http_handle->mutex);

    if (http_handle->pending_error)
    {
        result = globus_error_put(
            globus_object_copy(http_handle->pending_error));
        goto error_exit;
    }
    
    if (http_handle->read_operation.operation != NULL)
    {
        /* Only one read in progress per handle, sorry */
        result = GlobusXIOErrorAlreadyRegistered();

        goto error_exit;
    }

    /* Copy user's iovec into non-const version in the handle, so that
     * we can deal with premature reads or reads from the header residue.
     */
    http_handle->read_operation.iov = globus_libc_calloc(
            iovec_count, sizeof(globus_xio_iovec_t));
    http_handle->read_operation.iovcnt = iovec_count;
    http_handle->read_operation.operation = op;
    http_handle->read_operation.nbytes = 0;
    http_handle->read_operation.wait_for =
            globus_xio_operation_get_wait_for(op);

    for (i = 0; i < iovec_count; i++)
    {
        http_handle->read_operation.iov[i].iov_base = 
            iovec[i].iov_base;
        http_handle->read_operation.iov[i].iov_len =
            iovec[i].iov_len;
    }

    if (http_handle->parse_state == GLOBUS_XIO_HTTP_REQUEST_LINE ||
            http_handle->parse_state == GLOBUS_XIO_HTTP_STATUS_LINE ||
            http_handle->parse_state == GLOBUS_XIO_HTTP_HEADERS)
    {
        /* Still reading header information---read will be handled after
         * headers are parsed
         */

        globus_bool_t                   cancelled;

        globus_assert(http_handle->cancellation == NULL);

        http_handle->cancellation =
            malloc(sizeof(globus_i_xio_http_cancellation_t));
        if (http_handle->cancellation == NULL)
        {
            globus_mutex_unlock(&http_handle->mutex);

            return GlobusXIOErrorMemory("cancellation");
        }

        http_handle->cancellation->user_read_op = op;
        http_handle->cancellation->internal_op = 
                http_handle->response_read_operation;
        http_handle->cancellation->http_handle = http_handle;
        http_handle->cancellation->driver_handle =
            globus_xio_operation_get_driver_handle(op);

        globus_mutex_lock(&globus_i_xio_http_cancel_mutex);
        globus_list_insert(
                &globus_i_xio_http_cancellable_handles,
                http_handle->cancellation);
        globus_mutex_unlock(&globus_i_xio_http_cancel_mutex);

        cancelled = globus_xio_operation_enable_cancel(
                    op,
                    globus_l_xio_http_read_cancel_callback,
                    http_handle->cancellation);

        if (cancelled)
        {
            free(http_handle->read_operation.iov);
            http_handle->read_operation.iov = NULL;
            http_handle->read_operation.iovcnt = 0;
            http_handle->read_operation.operation = NULL;
            http_handle->read_operation.driver_handle = NULL;
            http_handle->read_operation.nbytes = 0;
            http_handle->read_operation.wait_for = 0;
            free(http_handle->cancellation);
            http_handle->cancellation = NULL;
            result = GlobusXIOErrorCanceled();
        }

        globus_mutex_unlock(&http_handle->mutex);
        return result;
    }
    else if ((! http_handle->target_info.is_client) &&
        (http_handle->parse_state == GLOBUS_XIO_HTTP_PRE_REQUEST_LINE))
    {
        /* Haven't started reading header information yet---register that
         * read
         */
        if (http_handle->read_buffer.iov_base == NULL)
        {
            http_handle->read_buffer.iov_len = GLOBUS_XIO_HTTP_CHUNK_SIZE;
            http_handle->read_buffer.iov_base = globus_libc_malloc(
                                        GLOBUS_XIO_HTTP_CHUNK_SIZE);
            if (http_handle->read_buffer.iov_base == NULL)
            {
                result = GlobusXIOErrorMemory("read_buffer");
    
                goto error_exit;
            }
        }
        else
        {
            result = globus_i_xio_http_clean_read_buffer(http_handle);

            if (result != GLOBUS_SUCCESS)
            {
                goto error_exit;
            }
            http_handle->parse_state = GLOBUS_XIO_HTTP_REQUEST_LINE;
        }
    
        result = globus_xio_driver_pass_read(
                op,
                &http_handle->read_buffer,
                1,
                1,
                globus_i_xio_http_server_read_request_callback,
                http_handle);

        if (result == GLOBUS_SUCCESS)
        {
            http_handle->parse_state = GLOBUS_XIO_HTTP_REQUEST_LINE;
        }
        else
        {
            goto error_exit;
        }

        globus_mutex_unlock(&http_handle->mutex);
        return result;
    }

    /* Parse any residual information in our buffer, maybe copying to use
     * buffer, maybe registering a read
     */
    result = globus_i_xio_http_parse_residue(http_handle, &registered_again);

    if ((http_handle->read_operation.wait_for <= 0 && !registered_again) ||
        result != GLOBUS_SUCCESS)
    {
        if (header_info->transfer_encoding !=
                GLOBUS_XIO_HTTP_TRANSFER_ENCODING_CHUNKED &&
            GLOBUS_I_XIO_HTTP_HEADER_IS_CONTENT_LENGTH_SET(header_info) &&
            header_info->content_length == 0)
        {
            /* Synthesize EOF if we've read all of the entity content */
            http_handle->parse_state = GLOBUS_XIO_HTTP_EOF;
            result = GlobusXIOErrorEOF();
        }
        /*
         * Either we've read enough, hit end of chunk, no entity was present,
         * or pass to transport failed. Call finished_read 
         */
        op = http_handle->read_operation.operation;

        nbytes = http_handle->read_operation.nbytes;
        globus_libc_free(http_handle->read_operation.iov);
        http_handle->read_operation.iov = NULL;
        http_handle->read_operation.iovcnt = 0;
        http_handle->read_operation.operation = NULL;
        http_handle->read_operation.driver_handle = NULL;
        http_handle->read_operation.nbytes = 0;

        if (http_handle->target_info.is_client && !http_handle->read_response)
        {
            /* Set metadata on this read to contain the response info */
            descriptor = globus_xio_operation_get_data_descriptor(
                    op,
                    GLOBUS_TRUE);
            if (descriptor == NULL)
            {
                result = GlobusXIOErrorMemory("descriptor");

                goto error_exit;
            }
            globus_i_xio_http_response_destroy(&descriptor->response);
            result = globus_i_xio_http_response_copy(
                    &descriptor->response,
                    &http_handle->response_info);

            if (result != GLOBUS_SUCCESS)
            {
                goto error_exit;
            }
            http_handle->read_response = GLOBUS_TRUE;
        }
        globus_mutex_unlock(&http_handle->mutex);
        globus_xio_driver_finished_read(op, result, nbytes);

        result = GLOBUS_SUCCESS;
    }
    else
    {
        globus_mutex_unlock(&http_handle->mutex);
    }
    return result;

error_exit:
    globus_mutex_unlock(&http_handle->mutex);
    return result;
}
/* globus_i_xio_http_read() */

/**
 * Parse residual data from header read into user buffers.
 * @ingroup globus_i_xio_http_transform
 *
 * Parses the data in the @a http_handle's read_buffer---if content is
 * present, it will be copied into the user buffer
 * passed to the http driver via globus_xio_http_driver_pass_read().
 *
 * Called with the handle's mutex locked.
 *
 * @param http_handle
 *     Handle associated with the read.
 * @param registered_again
 *     Set to GLOBUS_TRUE by this function if the read operation was passed
 *     down again as a result of a partial read of data.
 * 
 * @return void
 */
globus_result_t
globus_i_xio_http_parse_residue(
    globus_i_xio_http_handle_t *        http_handle,
    globus_bool_t *                     registered_again)
{
    int                                 i;
    globus_result_t                     result = GLOBUS_SUCCESS;
    globus_i_xio_http_header_info_t *   headers;
    globus_size_t                       nbytes;
    globus_bool_t                       done;
    globus_off_t                        max_content = 0;
    GlobusXIOName(globus_i_xio_http_parse_residue);

    *registered_again = GLOBUS_FALSE;

    if (http_handle->target_info.is_client)
    {
        headers = &http_handle->response_info.headers;
    }
    else
    {
        headers = &http_handle->request_info.headers;
    }

    if (http_handle->read_operation.iovcnt == 0 || 
        http_handle->parse_state == GLOBUS_XIO_HTTP_EOF)
    {
        http_handle->read_operation.wait_for = 0; 
        goto finish;
    }

    if (http_handle->parse_state != GLOBUS_XIO_HTTP_EOF)
    {
        /* Deal with read if an entity is expected. If it isn't, then
         * any data in the read buffer is likely a pipelined request or
         * response.
         */

        switch (http_handle->parse_state)
        {
            case GLOBUS_XIO_HTTP_CHUNK_CRLF:
            case GLOBUS_XIO_HTTP_CHUNK_LINE:
            case GLOBUS_XIO_HTTP_CHUNK_FOOTERS:
                /* Get another chunk header out of the residue */
                result = globus_l_xio_http_parse_chunk_header(
                        http_handle,
                        &done);

                if (result == GLOBUS_SUCCESS && (!done))
                {
                    /*
                     * Didn't get the complete chunk header, we'll need to
                     * register another read.
                     */
                    result = globus_i_xio_http_clean_read_buffer(http_handle);

                    if (result != GLOBUS_SUCCESS)
                    {
                        break;
                    }

                    result = globus_xio_driver_pass_read(
                            http_handle->read_operation.operation,
                            &http_handle->read_iovec,
                            1,
                            1,
                            globus_l_xio_http_read_chunk_header_callback,
                            http_handle);
                    if (result == GLOBUS_SUCCESS)
                    {
                        *registered_again = GLOBUS_TRUE;
                    }
                    break;
                }
                else if (result != GLOBUS_SUCCESS)
                {
                    /* Error parsing chunk size */
                    break;
                }

                /* FALLSTHROUGH */
            case GLOBUS_XIO_HTTP_CHUNK_BODY:
            case GLOBUS_XIO_HTTP_IDENTITY_BODY:
                globus_l_xio_http_copy_residue(http_handle);

                if (http_handle->parse_state != GLOBUS_XIO_HTTP_EOF &&
                        http_handle->read_operation.wait_for > 0)
                {
                    /* Haven't pre-read enough, pass to transport */
                    if (http_handle->parse_state
                            == GLOBUS_XIO_HTTP_IDENTITY_BODY &&
                        GLOBUS_I_XIO_HTTP_HEADER_IS_CONTENT_LENGTH_SET(headers))
                    {
                        max_content = headers->content_length;
                    }
                    else if (http_handle->parse_state
                            == GLOBUS_XIO_HTTP_CHUNK_BODY)
                    {
                        max_content = http_handle->read_chunk_left;
                    }

                    if (max_content > 0)
                    {
                        /*
                         * Need to truncate iovecs here, so that we don't try
                         * to read past end of content and have to copy
                         * data back in to our read buffer. XIO will repost the
                         * read to us if we finish with < wait_for bytes.
                         */
                        nbytes = 0;

                        for (i = 0; i < http_handle->read_operation.iovcnt; i++)
                        {
                            if ((http_handle->read_operation.iov[i].iov_len +
                                        nbytes) > max_content)
                            {
                                http_handle->read_operation.iov[i].iov_len =
                                    max_content - nbytes;
                            }
                            nbytes +=
                                http_handle->read_operation.iov[i].iov_len;
                        }
                        if (nbytes > http_handle->read_operation.wait_for)
                        {
                            nbytes = http_handle->read_operation.wait_for;
                        }
                    }
                    else
                    {
                        nbytes = http_handle->read_operation.wait_for;
                    }
                    
                    result = globus_xio_driver_pass_read(
                            http_handle->read_operation.operation,
                            http_handle->read_operation.iov,
                            http_handle->read_operation.iovcnt,
                            nbytes,
                            globus_l_xio_http_read_callback,
                            http_handle);
                    if (result == GLOBUS_SUCCESS)
                    {
                        *registered_again = GLOBUS_TRUE;
                    }  
                          
                }
                break;
            case GLOBUS_XIO_HTTP_STATUS_LINE:
                result = globus_l_xio_http_client_parse_response(http_handle, &done);
                if (result != GLOBUS_SUCCESS)
                {
                    goto finish;
                }
                if(!done)
                {
                    result = globus_i_xio_http_clean_read_buffer(http_handle);

                    if (result != GLOBUS_SUCCESS)
                    {
                        goto finish;
                    }
                
                    result = globus_xio_driver_pass_read(
                            http_handle->response_read_operation,
                            &http_handle->read_iovec,
                            1,
                            1,
                            globus_l_xio_http_client_read_response_callback,
                            http_handle);
                
                    if (result != GLOBUS_SUCCESS)
                    {
                        goto finish;
                    }
                }
                break;

            case GLOBUS_XIO_HTTP_PRE_REQUEST_LINE:
                globus_assert(http_handle->parse_state
                        != GLOBUS_XIO_HTTP_PRE_REQUEST_LINE);
            case GLOBUS_XIO_HTTP_REQUEST_LINE:
                globus_assert(http_handle->parse_state
                        != GLOBUS_XIO_HTTP_REQUEST_LINE);
            case GLOBUS_XIO_HTTP_HEADERS:
                globus_assert(http_handle->parse_state
                        != GLOBUS_XIO_HTTP_HEADERS);
            case GLOBUS_XIO_HTTP_EOF:
                globus_assert(http_handle->parse_state
                        != GLOBUS_XIO_HTTP_EOF);

            case GLOBUS_XIO_HTTP_CLOSE:
                globus_assert(http_handle->parse_state
                        != GLOBUS_XIO_HTTP_CLOSE);

                result = GlobusXIOErrorParameter("handle [invalid state]");
                break;
        }
    }

finish:
    return result;
}
/* globus_i_xio_http_parse_residue() */

/**
 * Called with mutex locked.
 */
static
void
globus_l_xio_http_copy_residue(
    globus_i_xio_http_handle_t *        http_handle)
{
    globus_i_xio_http_header_info_t *   headers;
    int                                 to_copy;
    int                                 i;

    if (http_handle->target_info.is_client)
    {
        headers = &http_handle->response_info.headers;
    }
    else
    {
        headers = &http_handle->request_info.headers;
    }
    if (GLOBUS_I_XIO_HTTP_HEADER_IS_CONTENT_LENGTH_SET(headers))
    {
        /* Don't wait beyond content-length */
        if (http_handle->read_operation.wait_for >
                headers->content_length)
        {
            http_handle->read_operation.wait_for =
                headers->content_length;
        }
    }
    to_copy = 0;
    /*
     * Copy already read data into the user's buffers.
     */
    for (i = 0;
         (i < http_handle->read_operation.iovcnt)
             && (http_handle->read_buffer_valid > 0);
         i++)
    {
        if (http_handle->read_operation.iov[i].iov_len
                > http_handle->read_buffer_valid)
        {
            to_copy = http_handle->read_buffer_valid;
        }
        else
        {
            to_copy = http_handle->read_operation.iov[i].iov_len;
        }
        /* Don't copy more than content-length */
        if (headers->transfer_encoding !=
                GLOBUS_XIO_HTTP_TRANSFER_ENCODING_CHUNKED &&
            GLOBUS_I_XIO_HTTP_HEADER_IS_CONTENT_LENGTH_SET(headers) &&
            headers->content_length < to_copy)
        {
            to_copy = headers->content_length;
        }
        else if (headers->transfer_encoding
                == GLOBUS_XIO_HTTP_TRANSFER_ENCODING_CHUNKED &&
                http_handle->read_chunk_left < to_copy)
        {
            to_copy = http_handle->read_chunk_left;
        }

        if (to_copy == 0)
        {
            continue;
        }
        memcpy(http_handle->read_operation.iov[i].iov_base,
                (globus_byte_t *)http_handle->read_buffer.iov_base +
                http_handle->read_buffer_offset,
                to_copy);
        http_handle->read_buffer_valid -= to_copy;
        http_handle->read_buffer_offset += to_copy;
        http_handle->read_operation.iov[i].iov_len -= to_copy;
        http_handle->read_operation.iov[i].iov_base =
          (char *) http_handle->read_operation.iov[i].iov_base + to_copy;
        http_handle->read_operation.nbytes += to_copy;
        if (http_handle->read_operation.wait_for >= to_copy)
        {
            http_handle->read_operation.wait_for -= to_copy;
        }
        else
        {
            http_handle->read_operation.wait_for = 0;
        }
        http_handle->read_chunk_left -= to_copy;

        if (headers->transfer_encoding
                != GLOBUS_XIO_HTTP_TRANSFER_ENCODING_CHUNKED &&
                GLOBUS_I_XIO_HTTP_HEADER_IS_CONTENT_LENGTH_SET(headers))
        {
            headers->content_length -= to_copy;
        }
        else if (headers->transfer_encoding
                == GLOBUS_XIO_HTTP_TRANSFER_ENCODING_CHUNKED &&
                http_handle->read_chunk_left == 0)
        {
            http_handle->parse_state = GLOBUS_XIO_HTTP_CHUNK_CRLF;
        }
    }
}
/* globus_l_xio_http_copy_residue() */

/**
 * Read callback for user data
 * @ingroup globus_i_xio_http_transform
 *
 * Returns data to the XIO layer when transport has read the part of a chunk
 * or entity body for the user.
 *
 * @param op
 *     Operation associated with this read.
 * @param result
 *     Transport result for this read.
 * @param nbytes
 *     Number of bytes read by the transport.
 * @param user_arg
 *     Void pointer pointing to an HTTP handle.
 *
 * @return void
 */
static
void
globus_l_xio_http_read_callback(
    globus_xio_operation_t              op,
    globus_result_t                     result,
    globus_size_t                       nbytes,
    void *                              user_arg)
{
    globus_i_xio_http_handle_t *        http_handle = user_arg;
    globus_i_xio_http_header_info_t *   headers;
    globus_size_t                       nbytes_result = nbytes;
    GlobusXIOName(globus_l_xio_http_read_callback);

    if (http_handle->target_info.is_client)
    {
        headers = &http_handle->response_info.headers;
    }
    else
    {
        headers = &http_handle->request_info.headers;
    }

    globus_mutex_lock(&http_handle->mutex);

    globus_libc_free(http_handle->read_operation.iov);

    /* Add in amt of data pre-read when chunk header was read */
    nbytes_result += http_handle->read_operation.nbytes;

    http_handle->read_operation.iov = NULL;
    http_handle->read_operation.iovcnt = 0;
    http_handle->read_operation.operation = NULL;
    http_handle->read_operation.driver_handle = NULL;
    http_handle->read_operation.nbytes = 0;

    if (headers->transfer_encoding
            != GLOBUS_XIO_HTTP_TRANSFER_ENCODING_CHUNKED &&
        GLOBUS_I_XIO_HTTP_HEADER_IS_CONTENT_LENGTH_SET(headers))
    {
        headers->content_length -= nbytes;

        if (headers->content_length == 0 && result == GLOBUS_SUCCESS)
        {
            http_handle->parse_state = GLOBUS_XIO_HTTP_EOF;
            if (http_handle->target_info.is_client)
            {
                result = GlobusXIOErrorEOF();
            }
            else
            {
                result = GlobusXIOHttpErrorEOF();
            }
        }
    }
    else if (headers->transfer_encoding
            == GLOBUS_XIO_HTTP_TRANSFER_ENCODING_CHUNKED)
    {
        http_handle->read_chunk_left -= nbytes;

        if (http_handle->read_chunk_left == 0)
        {
            http_handle->parse_state = GLOBUS_XIO_HTTP_CHUNK_CRLF;
        }
    }

    globus_mutex_unlock(&http_handle->mutex);
    globus_xio_driver_finished_read(op, result, nbytes_result);
}
/* globus_l_xio_http_read_callback() */

/**
 * Read callback for chunk header information
 * @ingroup globus_i_xio_http_transform
 *
 * Called when the driver has read (at least part of) an http chunk line.
 * Any residue in the driver's buffer will be copied to a user's buffer later.
 *
 * @param op
 *     Operation associated with this read.
 * @param result
 *     Transport result for this read.
 * @param nbytes
 *     Number of bytes read by the transport.
 * @param user_arg
 *     Void pointer pointing to an HTTP handle.
 *
 * @return void
 */
static
void
globus_l_xio_http_read_chunk_header_callback(
    globus_xio_operation_t              op,
    globus_result_t                     result,
    globus_size_t                       nbytes,
    void *                              user_arg)
{
    globus_i_xio_http_handle_t *        http_handle = user_arg;
    globus_i_xio_http_header_info_t *   headers;
    globus_bool_t                       registered_again = GLOBUS_FALSE;
    GlobusXIOName(globus_l_xio_http_read_chunk_header_callback);

    globus_mutex_lock(&http_handle->mutex);
    http_handle->read_buffer_valid += nbytes;

    if (http_handle->target_info.is_client)
    {
        headers = &http_handle->response_info.headers;
    }
    else
    {
        headers = &http_handle->request_info.headers;
    }
    if (result == GLOBUS_SUCCESS)
    {
        result = globus_i_xio_http_parse_residue(
                http_handle,
                &registered_again);
    }
    if ((http_handle->read_operation.wait_for <= 0 && !registered_again) ||
        result != GLOBUS_SUCCESS)
    {
        if (headers->transfer_encoding !=
                GLOBUS_XIO_HTTP_TRANSFER_ENCODING_CHUNKED &&
            GLOBUS_I_XIO_HTTP_HEADER_IS_CONTENT_LENGTH_SET(headers) &&
            headers->content_length == 0)
        {
            http_handle->parse_state = GLOBUS_XIO_HTTP_EOF;
            /* Synthesize EOF if we've read all of the entity content */
            if (http_handle->target_info.is_client)
            {
                result = GlobusXIOErrorEOF();
            }
            else
            {
                result = GlobusXIOHttpErrorEOF();
            }
        }
        /*
         * Either we've read enough, hit end of chunk, no entity was present,
         * or pass to transport failed. Call finished_read 
         */
        op = http_handle->read_operation.operation;

        nbytes = http_handle->read_operation.nbytes;
        globus_libc_free(http_handle->read_operation.iov);
        http_handle->read_operation.iov = NULL;
        http_handle->read_operation.iovcnt = 0;
        http_handle->read_operation.operation = NULL;
        http_handle->read_operation.driver_handle = NULL;
        http_handle->read_operation.nbytes = 0;

        globus_mutex_unlock(&http_handle->mutex);
        globus_xio_driver_finished_read(op, result, nbytes);
    }
    else
    {
        globus_mutex_unlock(&http_handle->mutex);
    }
}
/* globus_l_xio_http_read_chunk_header_callback() */

/**
 * Parse the framing for an http chunk
 * @ingroup globus_i_xio_http_transform
 *
 * Parse the chunk size, end-of-chunk CRLF, or chunk footers, out of the
 * handle's read buffer, depending on the current parse state. 
 *
 * @param http_handle
 *     Handle associated with this parsing.
 * @param done
 *     Pointer to a boolean. This will be set to GLOBUS_TRUE if we are
 *     able to parse an entire chunk header line, or have read the
 *     final footer line. Otherwise, will be set to GLOBUS_FALSE.
 *
 * @return This function returns GLOBUS_SUCCESS, GLOBUS_XIO_HTTP_ERROR_PARSE,
 * or GLOBUS_XIO_ERROR_EOF. It also modifies the value pointed to by the
 * @a done parameter as described above.
 *
 * Called with mutex locked.
 *
 * @retval GLOBUS_SUCCESS
 *     No parsing errors encountered.
 * @retval <driver>::GLOBUS_XIO_HTTP_ERROR_PARSE
 *     Improperly formed chunk header.
 * @retval GLOBUS_XIO_ERROR_EOF
 *     Final chunk footer was read.
 */
static
globus_result_t
globus_l_xio_http_parse_chunk_header(
    globus_i_xio_http_handle_t *        http_handle,
    globus_bool_t *                     done)
{
    char *                              current_offset;
    char *                              eol;
    unsigned long                       chunk_size;
    globus_result_t                     result = GLOBUS_SUCCESS;
    size_t                              parsed;
    GlobusXIOName(globus_l_xio_http_parse_chunk_header);

    current_offset = ((char *) (http_handle->read_buffer.iov_base))
        + http_handle->read_buffer_offset;

    eol = globus_i_xio_http_find_eol(current_offset,
            http_handle->read_buffer_valid);

    if (eol == NULL)
    {
        /* Not enough data to parse */
        *done = GLOBUS_FALSE;
        return result;
    }

    switch (http_handle->parse_state)
    {
        case GLOBUS_XIO_HTTP_CHUNK_CRLF:
            if (current_offset != eol)
            {
                result = GlobusXIOHttpErrorParse("chunk", current_offset);

                break;
            }

            current_offset += 2;
            http_handle->read_buffer_valid -= 2;
            http_handle->read_buffer_offset += 2;
            eol = globus_i_xio_http_find_eol(current_offset,
                    http_handle->read_buffer_valid);
            http_handle->parse_state = GLOBUS_XIO_HTTP_CHUNK_LINE;

            if (eol == NULL)
            {
                *done = GLOBUS_FALSE;

                result = GLOBUS_SUCCESS;

                break;
            }
            
            /* FALLSTHROUGH */
        case GLOBUS_XIO_HTTP_CHUNK_LINE:
            *eol = '\0';

            globus_libc_lock();
            errno = 0;
            chunk_size = strtoul(current_offset, NULL, 16);
            if (chunk_size == ULONG_MAX && errno != 0)
            {
                result = GlobusXIOHttpErrorParse("Chunk-size", current_offset);

                globus_libc_unlock();

                break;
            }
            globus_libc_unlock();

            if (chunk_size > UINT_MAX)
            {
                result = GlobusXIOHttpErrorParse("Chunk-size", current_offset);

                globus_libc_unlock();

                break;
            }
            http_handle->read_chunk_left = (globus_size_t) chunk_size;

            current_offset = eol + 2;
            parsed = current_offset -
                ((char *) http_handle->read_buffer.iov_base
                 + http_handle->read_buffer_offset);

            http_handle->read_buffer_valid -= parsed;
            http_handle->read_buffer_offset += parsed;

            if (http_handle->read_chunk_left != 0)
            {
                *done = GLOBUS_TRUE;
                http_handle->parse_state = GLOBUS_XIO_HTTP_CHUNK_BODY;

                result = GLOBUS_SUCCESS;

                break;
            }

            /* 0 byte chunk is the Last chunk. Parse footers. */
            http_handle->parse_state = GLOBUS_XIO_HTTP_CHUNK_FOOTERS;

            /* FALLSTHROUGH */

        case GLOBUS_XIO_HTTP_CHUNK_FOOTERS:
            /* We parsed the last chunk size line '0'. Now parse any footers
             * and the final CRLF
             */
            while ((eol = globus_i_xio_http_find_eol(
                            current_offset,
                            http_handle->read_buffer_valid)) != current_offset)
            {
                if (eol == NULL)
                {
                    /* final headers not all found */
                    *done = GLOBUS_FALSE;

                    break;
                }
                /* Ignore footers for now */
                current_offset = eol + 2;
                parsed = current_offset -
                    ((char *) http_handle->read_buffer.iov_base
                     + http_handle->read_buffer_offset);
                http_handle->read_buffer_valid -= parsed;
                http_handle->read_buffer_offset += parsed;
            }

            if (eol != NULL)
            {
                /* We found an empty line---end of footers found. */
                *done = GLOBUS_TRUE;
                current_offset = eol + 2;

                parsed = current_offset -
                    ((char *) http_handle->read_buffer.iov_base
                        + http_handle->read_buffer_offset);
                http_handle->read_buffer_valid -= parsed;
                http_handle->read_buffer_offset += parsed;
                http_handle->parse_state = GLOBUS_XIO_HTTP_EOF;

                if (http_handle->target_info.is_client)
                {
                    result = GlobusXIOErrorEOF();
                }
                else
                {
                    result = GlobusXIOHttpErrorEOF();
                }
            }
            else
            {
                /* Didn't finish parsing */
                *done = GLOBUS_FALSE;
            }
            break;
        case GLOBUS_XIO_HTTP_PRE_REQUEST_LINE:
            globus_assert(http_handle->parse_state !=
                    GLOBUS_XIO_HTTP_PRE_REQUEST_LINE);
        case GLOBUS_XIO_HTTP_REQUEST_LINE:
            globus_assert(http_handle->parse_state !=
                    GLOBUS_XIO_HTTP_REQUEST_LINE);
        case GLOBUS_XIO_HTTP_STATUS_LINE:
            globus_assert(http_handle->parse_state !=
                    GLOBUS_XIO_HTTP_STATUS_LINE);
        case GLOBUS_XIO_HTTP_HEADERS:
            globus_assert(http_handle->parse_state !=
                    GLOBUS_XIO_HTTP_HEADERS);
        case GLOBUS_XIO_HTTP_CHUNK_BODY:
            globus_assert(http_handle->parse_state !=
                    GLOBUS_XIO_HTTP_CHUNK_BODY);
        case GLOBUS_XIO_HTTP_IDENTITY_BODY:
            globus_assert(http_handle->parse_state !=
                    GLOBUS_XIO_HTTP_IDENTITY_BODY);
        case GLOBUS_XIO_HTTP_EOF:
            globus_assert(http_handle->parse_state !=
                    GLOBUS_XIO_HTTP_EOF);
        case GLOBUS_XIO_HTTP_CLOSE:
            globus_assert(http_handle->parse_state
                    != GLOBUS_XIO_HTTP_CLOSE);
    }

    return result;
}
/* globus_l_xio_http_parse_chunk_header() */

/**
 * Write to an HTTP stream
 * @ingroup globus_i_xio_http_transform
 *
 * Begins processing a write. If this is called in a server implementation
 * before the response has been sent, then the information pertaining to the
 * write is just stored to be processed later, and the response will be sent
 * now. Otherwise, if this "chunked" transfer-encoding is being used,
 * globus_i_xio_http_write_chunk() is called to frame and send this data.
 * If "chunked" encoding is not being used, then the data buffer is simply
 * passed to the transport.
 *
 * @param handle
 *     Void pointer to a globus_i_xio_http_handle_t.
 * @param iovec
 *     Pointer to user's iovec array. Can't be changed, so a copy is made
 *     in the http_handle when we are delaying the write.
 * @param iovec_count
 *     Length of the iovec array.
 * @param op
 *     Operation associated with the write.
 *
 * @return
 *     This function returns GLOBUS_SUCCESS, 
 *     GLOBUS_XIO_ERROR_ALREADY_REGISTERED, and GLOBUS_XIO_ERROR_EOF
 *     errors directly. Other errors may be returned from
 *     globus_i_xio_http_server_write_response(),
 *     globus_i_xio_http_write_chunk(), or globus_xio_driver_pass_write().
 *
 * @retval GLOBUS_SUCCESS
 *     Write handled successfully. At some point later,
 *     globus_xio_driver_finished_read() will be called.
 * @retval <driver>::GLOBUS_XIO_HTTP_ERROR_NO_ENTITY
 *     A write was attempted for an operation which does not require a
 *     message body (such as a "GET" request) or after the end of the
 *     entity has already been sent.
 * @retval GLOBUS_XIO_ERROR_ALREADY_REGISTERED
 *     A write is already registered with the HTTP handle, so we can't deal
 *     with this new one.
 */
globus_result_t
globus_i_xio_http_write(
    void *                                  handle,
    const globus_xio_iovec_t *              iovec,
    int                                     iovec_count,
    globus_xio_operation_t                  op)
{
    globus_i_xio_http_handle_t *            http_handle = handle;
    globus_result_t                         result;
    globus_i_xio_http_header_info_t *       headers;
    GlobusXIOName(globus_i_xio_http_write);

    if (http_handle->target_info.is_client)
    {
        headers = &http_handle->request_info.headers;
    }
    else
    {
        headers = &http_handle->response_info.headers;
    }

    globus_mutex_lock(&http_handle->mutex);

    if (http_handle->pending_error)
    {
        result = globus_error_put(
            globus_object_copy(http_handle->pending_error));
    }
    else
    {
        
        switch (http_handle->send_state)
        {
            case GLOBUS_XIO_HTTP_STATUS_LINE:
                result = globus_i_xio_http_server_write_response(
                    http_handle,
                    iovec,
                    iovec_count,
                    op);
                break;
            case GLOBUS_XIO_HTTP_EOF:
            case GLOBUS_XIO_HTTP_CLOSE:
                result = GlobusXIOHttpErrorNoEntity();
                break;
    
            case GLOBUS_XIO_HTTP_CHUNK_BODY:
                if (http_handle->write_operation.operation != NULL)
                {
                    result = GlobusXIOErrorAlreadyRegistered();
                    break;
                }
                result = globus_i_xio_http_write_chunk(
                        handle,
                        iovec,
                        iovec_count,
                        op);
                break;
            case GLOBUS_XIO_HTTP_IDENTITY_BODY:
                if (http_handle->write_operation.operation != NULL)
                {
                    result = GlobusXIOErrorAlreadyRegistered();
                    break;
                }
                result = globus_xio_driver_pass_write(
                        op,
                        (globus_xio_iovec_t *) iovec,
                        iovec_count,
                        globus_xio_operation_get_wait_for(op),
                        globus_i_xio_http_write_callback,
                        http_handle);
                break;
    
            case GLOBUS_XIO_HTTP_PRE_REQUEST_LINE:
                if(http_handle->delay_write_header)
                {
                    http_handle->first_write_iovec = iovec;
                    http_handle->first_write_iovec_count = iovec_count;
    
                    result = globus_i_xio_http_client_write_request(
                        op,
                        http_handle);
                    break;
                }
                /* FALLSTHROUGH */
            case GLOBUS_XIO_HTTP_REQUEST_LINE:
            case GLOBUS_XIO_HTTP_HEADERS:
            case GLOBUS_XIO_HTTP_CHUNK_CRLF:
            case GLOBUS_XIO_HTTP_CHUNK_LINE:
            case GLOBUS_XIO_HTTP_CHUNK_FOOTERS:
                globus_assert(http_handle->send_state
                        != GLOBUS_XIO_HTTP_PRE_REQUEST_LINE);
                globus_assert(http_handle->send_state
                        != GLOBUS_XIO_HTTP_REQUEST_LINE);
                globus_assert(http_handle->send_state
                        != GLOBUS_XIO_HTTP_HEADERS);
                globus_assert(http_handle->send_state
                        != GLOBUS_XIO_HTTP_CHUNK_CRLF);
                globus_assert(http_handle->send_state
                        != GLOBUS_XIO_HTTP_CHUNK_LINE);
                globus_assert(http_handle->send_state
                        != GLOBUS_XIO_HTTP_CHUNK_FOOTERS);
                result = GlobusXIOErrorParameter(handle);
                break;
        }
    }
    globus_mutex_unlock(&http_handle->mutex);

    return result;
}
/* globus_i_xio_http_write() */

/**
 * Write a chunk of data on an HTTP stream
 * @ingroup globus_i_xio_http_transform
 *
 * Frames an iovec that the user wishes to write with a chunk header and
 * passes it to the transport driver. When the transport returns,
 * globus_i_xio_http_write_callback() will be called to finish the write
 * operation.
 *
 * Called with the handle's mutex locked.
 *
 * @param http_handle
 *     Handle to the HTTP stream.
 * @param iovec
 *     User's iovec of data to be written.
 * @param iovec_count
 *     Length of the @a iovec array
 * @param op
 *     Operation associated with the write.
 *
 * @return This function returns GLOBUS_SUCCESS, GLOBUS_XIO_ERROR_MEMORY, or
 * an error result from globus_xio_driver_pass_write().
 *
 * @retval GLOBUS_SUCCESS
 *     Write framed and passed to the transport. When the transport completes,
 *     it will call the globus_i_xio_write_callback() function to finish
 *     the write operation.
 * @retval GLOBUS_XIO_ERROR_MEMORY
 *     Unable to process write due to memory constraints.
 */
globus_result_t
globus_i_xio_http_write_chunk(
    globus_i_xio_http_handle_t *        http_handle,
    const globus_xio_iovec_t *          iovec,
    int                                 iovec_count,
    globus_xio_operation_t              op)
{
    int                                 i;
    globus_result_t                     result;
    GlobusXIOName(globus_l_xio_http_write_chunk);

    for (i = 0, http_handle->write_operation.wait_for = 0; i < iovec_count; i++)
    {
        http_handle->write_operation.wait_for += iovec[i].iov_len;
    }

    if (http_handle->write_operation.wait_for == 0)
    {
        /* No content! Return done now (sending 0 length chunk will signal
         * eof!)
         */
        globus_xio_driver_finished_write(op, GLOBUS_SUCCESS, 0);

        return GLOBUS_SUCCESS;
    }
    http_handle->write_operation.operation = op;
    http_handle->write_operation.driver_handle = globus_xio_operation_get_driver_handle(op);
    http_handle->write_operation.iov = globus_libc_malloc((iovec_count + 2) * 
            sizeof(globus_xio_iovec_t));

    if (http_handle->write_operation.iov == NULL)
    {
        result = GlobusXIOErrorMemory("iovec");

        goto error_exit;
    }

    /* Chunk header to beginning of iovec we pass */
    http_handle->write_operation.iov[0].iov_base =
        http_handle->write_operation.chunk_size_buffer;
    http_handle->write_operation.iov[0].iov_len =
        globus_libc_sprintf(
                http_handle->write_operation.iov[0].iov_base,
                "%x\r\n",
                http_handle->write_operation.wait_for);
    /* Assert we haven't overwritten our buffer--64 bytes of hex string length
     * should be sufficient for now
     */
    globus_assert(http_handle->write_operation.iov[0].iov_len <
            sizeof(http_handle->write_operation.chunk_size_buffer));
    for (i = 0; i < iovec_count; i++)
    {
        http_handle->write_operation.iov[i+1].iov_base = iovec[i].iov_base;
        http_handle->write_operation.iov[i+1].iov_len = iovec[i].iov_len;
    }

    /* Chunk end (CRLF) to end of iovec we pass */
    http_handle->write_operation.iov[iovec_count+1].iov_base = "\r\n";
    http_handle->write_operation.iov[iovec_count+1].iov_len = 2;

    http_handle->write_operation.iovcnt = iovec_count + 2;

    /* Add to the "wait for" the size of framing tokens */
    http_handle->write_operation.wait_for +=
        http_handle->write_operation.iov[0].iov_len +
        http_handle->write_operation.iov[iovec_count+1].iov_len;

    return globus_xio_driver_pass_write(
            http_handle->write_operation.operation,
            http_handle->write_operation.iov,
            http_handle->write_operation.iovcnt,
            http_handle->write_operation.wait_for,
            globus_i_xio_http_write_callback,
            http_handle);
error_exit:
    http_handle->write_operation.wait_for = 0;
    http_handle->write_operation.operation = NULL;
    http_handle->write_operation.driver_handle = NULL;
    return result;
}
/* globus_i_xio_http_write_chunk() */

/**
 * Write complete callback
 * @ingroup globus_i_xio_http_transform
 *
 * Callback indicating that the transport has finished writing http data.
 * In chunked mode, we will adjust the nbytes to remove the size of the
 * chunk framing, otherwise we will finish the write with the value passed
 * to this callback.
 * 
 * @param op
 *     Operation associated with the write.
 * @param result
 *     Transport-level result of writing the data.
 * @param nbytes
 *     Number of bytes written by the transport.
 * @param user_arg
 *     Void pointer to a globus_i_xio_http_handle_t.
 *
 * @return void
 */
void
globus_i_xio_http_write_callback(
    globus_xio_operation_t              op,
    globus_result_t                     result,
    globus_size_t                       nbytes,
    void *                              user_arg)
{
    globus_i_xio_http_handle_t *        http_handle = user_arg;
    globus_i_xio_http_header_info_t *   headers;
    GlobusXIOName(globus_i_xio_http_write_callback);

    if (http_handle->target_info.is_client)
    {
        headers = &http_handle->request_info.headers;
    }
    else
    {
        headers = &http_handle->response_info.headers;
    }

    globus_mutex_lock(&http_handle->mutex);
    if (headers->transfer_encoding == GLOBUS_XIO_HTTP_TRANSFER_ENCODING_CHUNKED)
    {
        /* Subtract off the amount of data which can be attributed to chunk
         * header and footer information
         */
        if (result == GLOBUS_SUCCESS)
        {
            nbytes -= (http_handle->write_operation.iov[0].iov_len +
                    http_handle->write_operation.iov[
                        http_handle->write_operation.iovcnt-1].iov_len);
        }
        else if (nbytes <= http_handle->write_operation.iov[0].iov_len)
        {
            nbytes = 0;
        }
        else
        {
            nbytes -= http_handle->write_operation.iov[0].iov_len;
        }
        /*
         * Free our copy of the iovec with the chunk framing
         */
        globus_libc_free(http_handle->write_operation.iov);
    }
    else if (GLOBUS_I_XIO_HTTP_HEADER_IS_CONTENT_LENGTH_SET(headers))
    {
        headers->content_length -= nbytes;

        if (headers->content_length == 0)
        {
            http_handle->send_state = GLOBUS_XIO_HTTP_EOF;
        }
    }
    http_handle->write_operation.iov = NULL;
    http_handle->write_operation.iovcnt = 0;
    http_handle->write_operation.operation = NULL;
    http_handle->write_operation.driver_handle = NULL;
    http_handle->write_operation.nbytes = 0;
    http_handle->write_operation.wait_for = 0;
    if (result != GLOBUS_SUCCESS)
    {
        globus_object_t *               err;

        err = globus_error_get(result);
        
        if(http_handle->reopen_in_progress)
        {
            http_handle->pending_error = 
                GlobusXIOHTTPErrorObjPersistentConnectionDropped(err);
        }
        else
        {
            http_handle->pending_error = err;
        }
        http_handle->send_state = GLOBUS_XIO_HTTP_EOF;
        result = globus_error_put(
            globus_object_copy(http_handle->pending_error));
    }

    globus_mutex_unlock(&http_handle->mutex);

    globus_xio_driver_finished_write(op, result, nbytes);
}
/* globus_i_xio_http_write_callback() */

/**
 * Close an HTTP handle
 * @ingroup globus_i_xio_http_transform
 *
 * Starts the process of closing the handle. If the request or response
 * contains an entity, but end-of-file hasn't been marked on it yet,
 * then we call globus_i_xio_http_handle_cntl() to mark the end of the entity.
 * If the handle is all ready to close, we'll call
 * globus_i_xio_http_close_internal() to register the close events.
 *
 * @param handle
 *     Void pointer to a #globus_i_xio_http_handle_t.
 * @param attr
 *     Close attributes. (Ignored by the HTTP driver).
 * @param op
 *     Operation associated with the close.
 *
 * @retval GLOBUS_XIO_ERROR_ALREADY_REGISTERED
 *     The handle has already been closed.
 * @retval GLOBUS_XIO_ERROR_PARAMETER
 *     The handle is in some unexpected state and the close could not
 *     be handled.
 */
globus_result_t
globus_i_xio_http_close(
    void *                              handle,
    void *                              attr,
    globus_xio_operation_t              op)
{
    globus_result_t                     result;
    globus_i_xio_http_handle_t *        http_handle = handle;
    GlobusXIOName(globus_i_xio_http_close);

    globus_mutex_lock(&http_handle->mutex);
    switch (http_handle->send_state)
    {
        case GLOBUS_XIO_HTTP_CLOSE:
            /* Close already called */
            result = GlobusXIOErrorAlreadyRegistered();
            break;

        case GLOBUS_XIO_HTTP_STATUS_LINE:
            if (! http_handle->target_info.is_client)
            {
                /* On server, when we close we ought to set the
                 * "Connection: close" header when we want to close if
                 * we haven't sent our status line yet.
                 */
                http_handle->response_info.headers.flags |=
                        GLOBUS_I_XIO_HTTP_HEADER_CONNECTION_CLOSE;
            }
            /* FALLSTHROUGH */
        case GLOBUS_XIO_HTTP_CHUNK_BODY:
            http_handle->close_operation = op;
            http_handle->user_close = GLOBUS_TRUE;

            result = globus_i_xio_http_set_end_of_entity(http_handle);

            if (result != GLOBUS_SUCCESS)
            {
                http_handle->close_operation = NULL;
                http_handle->user_close = GLOBUS_FALSE;
            }
            break;

        case GLOBUS_XIO_HTTP_IDENTITY_BODY:
            globus_assert(http_handle->send_state
                    != GLOBUS_XIO_HTTP_IDENTITY_BODY);
        case GLOBUS_XIO_HTTP_REQUEST_LINE:
            globus_assert(http_handle->send_state
                    != GLOBUS_XIO_HTTP_REQUEST_LINE);
        case GLOBUS_XIO_HTTP_HEADERS:
            globus_assert(http_handle->send_state
                    != GLOBUS_XIO_HTTP_HEADERS);
        case GLOBUS_XIO_HTTP_CHUNK_CRLF:
            globus_assert(http_handle->send_state
                    != GLOBUS_XIO_HTTP_CHUNK_CRLF);
        case GLOBUS_XIO_HTTP_CHUNK_LINE:
            globus_assert(http_handle->send_state
                    != GLOBUS_XIO_HTTP_CHUNK_LINE);
        case GLOBUS_XIO_HTTP_CHUNK_FOOTERS:
            globus_assert(http_handle->send_state
                    != GLOBUS_XIO_HTTP_CHUNK_FOOTERS);

            result = GlobusXIOErrorParameter("header");
            break;

        case GLOBUS_XIO_HTTP_PRE_REQUEST_LINE:
        case GLOBUS_XIO_HTTP_EOF:
            http_handle->close_operation = op;
            http_handle->user_close = GLOBUS_TRUE;

            if (http_handle->write_operation.operation != NULL)
            {
                /* EOF writing in progress, return success and eof callback
                 * will finish the close
                 */
                result = GLOBUS_SUCCESS;
                break;
            }

            result = globus_i_xio_http_close_internal(http_handle);

            if (result != GLOBUS_SUCCESS)
            {
                http_handle->close_operation = NULL;
                http_handle->user_close = GLOBUS_FALSE;
            }
            break;
    }
    globus_mutex_unlock(&http_handle->mutex);

    return result;
}
/* globus_i_xio_http_close() */

/**
 * Close an http handle
 * @ingroup globus_i_xio_http_transform
 *
 * Pass the close event to the transport. This is either called by the above
 * or after the end-of-entity callback occurs.  In the server case, we delay
 * 100ms before passing the close to the transport, this seems to help avoid
 * some ECONNRESET errors on the client side---more investigation is needed to
 * remove this hack.
 *
 * Called with mutex locked.
 *
 * @param http_handle
 *     The HTTP handle we will be closing
 *
 * @return
 * This function returns the error result of globus_xio_driver_pass_close().
 */
globus_result_t
globus_i_xio_http_close_internal(
    globus_i_xio_http_handle_t *        http_handle)
{
    globus_i_xio_http_header_info_t *   headers;
    globus_result_t                     result;
    globus_reltime_t                    delay;

    if (http_handle->target_info.is_client)
    {
        headers = &http_handle->request_info.headers;
    }
    else
    {
        headers = &http_handle->response_info.headers;
    }

    http_handle->send_state = GLOBUS_XIO_HTTP_CLOSE;

    if (http_handle->target_info.is_client &&
        http_handle->user_close &&
        http_handle->request_info.http_version == GLOBUS_XIO_HTTP_VERSION_1_1 &&
        (!GLOBUS_I_XIO_HTTP_HEADER_IS_CONNECTION_CLOSE(
                &http_handle->response_info.headers)) &&
        http_handle->parse_state == GLOBUS_XIO_HTTP_EOF)
    {
        GlobusTimeReltimeSet(delay, 0, 0);

        result = globus_callback_register_oneshot(
                NULL,
                &delay,
                globus_l_xio_http_client_cache_kickout,
                http_handle);

        if (result == GLOBUS_SUCCESS)
        {
            goto finish;
        }
    }

    /* If oneshot registration failed, or if the close is "normal", we'll just
     * pass the close down
     */

    result = globus_xio_driver_pass_close(
            http_handle->close_operation,
            globus_i_xio_http_close_callback,
            http_handle);
finish:
    return result;
}
/* globus_i_xio_http_close_internal() */

void
globus_i_xio_http_close_callback(
    globus_xio_operation_t              op,
    globus_result_t                     result,
    void *                              user_arg)
{
    globus_i_xio_http_handle_t *        http_handle = user_arg;

    globus_mutex_lock(&http_handle->mutex);
    /*
     * If the close is generated by a failure during open (after we've
     * opened the transport, then this flag is set to false.
     */
    if (http_handle->user_close)
    {
        globus_mutex_unlock(&http_handle->mutex);
        globus_xio_driver_finished_close(op, result);
    }
    else
    {
        globus_mutex_unlock(&http_handle->mutex);
        globus_xio_driver_operation_destroy(http_handle->close_operation);
    }
    http_handle->close_operation = NULL;

    globus_i_xio_http_handle_destroy(user_arg);

    globus_libc_free(user_arg);
}
/* globus_i_xio_http_close_callback() */

static
void
globus_l_xio_http_client_cache_kickout(
    void *                              user_arg)
{
    globus_i_xio_http_handle_t *        http_handle = user_arg;

    globus_assert(http_handle->target_info.is_client &&
        http_handle->user_close &&
        http_handle->request_info.http_version==GLOBUS_XIO_HTTP_VERSION_1_1);

    globus_xio_driver_finished_close(
            http_handle->close_operation,
            GLOBUS_SUCCESS);

    globus_mutex_lock(&globus_i_xio_http_cached_handle_mutex);
    globus_list_insert(&globus_i_xio_http_cached_handles, http_handle);
    globus_mutex_unlock(&globus_i_xio_http_cached_handle_mutex);

    return;
}
/* globus_l_xio_http_client_cache_kickout() */

static
void
globus_l_xio_http_read_cancel_callback(
    globus_xio_operation_t              op,
    void *                              user_arg,
    globus_xio_error_type_t             reason)
{
    globus_callback_register_oneshot(
            NULL,
            &globus_i_reltime_zero,
            globus_l_xio_http_read_timeout_callback,
            user_arg);
}
/* globus_l_xio_http_read_cancel_callback() */

static
void
globus_l_xio_http_read_timeout_callback(
    void *                              user_arg)
{
    globus_list_t *                     l;
    globus_i_xio_http_cancellation_t *  cancellation = user_arg;

    globus_mutex_lock(&globus_i_xio_http_cancel_mutex);
    l = globus_list_search(globus_i_xio_http_cancellable_handles, user_arg);

    if (l == NULL)
    {
        /* if the cancellation info isn't in the list any more, it was removed
         * by the operation completing before this oneshot fired. We'll free
         * up memory and quit.
         */
        free(user_arg);
    }
    else
    {
        /* Operation hasn't completed yet. Cancel our internal read op, and
         * remove this op from the list
         */
        globus_list_remove(&globus_i_xio_http_cancellable_handles, l);

        globus_xio_driver_operation_cancel(
                cancellation->driver_handle,
                cancellation->internal_op);
    }
    globus_mutex_unlock(&globus_i_xio_http_cancel_mutex);
}
/* globus_l_xio_http_read_timeout_callback() */
