/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2006 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 *
 *  Portions of this code were written by Intel Corporation.
 *  Copyright (C) 2011-2012 Intel Corporation.  Intel provides this material
 *  to Argonne National Laboratory subject to Software Grant and Corporate
 *  Contributor License Agreement dated February 8, 2012.
 */

#include "scif_impl.h"

#undef FUNCNAME
#define FUNCNAME MPID_nem_scif_send_queued
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_scif_send_queued(MPIDI_VC_t * vc, reqq_t * send_queue)
{
    int mpi_errno = MPI_SUCCESS;
    MPID_Request *sreq;
    MPIDI_msg_sz_t offset;
    MPID_IOV *iov;
    int complete;
    uint64_t seqno;
    MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);

    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);

    MPIU_Assert(vc != NULL);

    if (MPIDI_CH3I_Sendq_empty(*send_queue)) {
        if (vc_scif->terminate)
            MPID_nem_scif_vc_terminated(vc);
        goto fn_exit;
    }
    while (!MPIDI_CH3I_Sendq_empty(*send_queue)) {
        sreq = MPIDI_CH3I_Sendq_head(*send_queue);
        seqno = RQ_SCIF(sreq)->seqno;
        complete = 0;
        if (seqno) {
            /* We were just waiting for DMA to finish */
            complete = MPID_nem_scif_chk_seqno(&vc_scif->sc->csend, seqno);
        }
        else if (MPID_nem_scif_poll_send(vc_scif->sc->fd, &vc_scif->sc->csend)) {
            MPIU_DBG_MSG_P(CH3_CHANNEL, VERBOSE, "Sending %p", sreq);

            iov = &sreq->dev.iov[sreq->dev.iov_offset];

            offset = MPID_nem_scif_writev(vc_scif->sc->fd, &vc_scif->sc->csend,
                                          iov, sreq->dev.iov_count, &seqno);
            MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER,
                                 "**scif_writev", "**scif_writev %s", MPIU_Strerror(errno));
            MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "write " MPIDI_MSG_SZ_FMT, offset);
            RQ_SCIF(sreq)->seqno = seqno;
            complete = 1;
            for (iov = &sreq->dev.iov[sreq->dev.iov_offset];
                 iov < &sreq->dev.iov[sreq->dev.iov_offset + sreq->dev.iov_count]; ++iov) {
                if (offset < iov->MPID_IOV_LEN) {
                    iov->MPID_IOV_BUF = (char *) iov->MPID_IOV_BUF + offset;
                    iov->MPID_IOV_LEN -= offset;
                    /* iov_count should be equal to the number of iov's remaining */
                    sreq->dev.iov_count -= ((iov - sreq->dev.iov) - sreq->dev.iov_offset);
                    sreq->dev.iov_offset = iov - sreq->dev.iov;
                    complete = 0;
                    seqno = 0;
                    break;
                }
                offset -= iov->MPID_IOV_LEN;
            }
            if (seqno)
                complete = 0;
        }
        if (!complete) {
            /* writev couldn't write the entire iov, give up for now */
            break;
        }
        else {
            /* sent whole message */
            int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);

            RQ_SCIF(sreq)->seqno = 0;
            reqFn = sreq->dev.OnDataAvail;
            if (!reqFn) {
                MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
                MPIDI_CH3U_Request_complete(sreq);
                MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
                MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq);
                continue;
            }

            complete = 0;
            mpi_errno = reqFn(vc, sreq, &complete);
            if (mpi_errno)
                MPIU_ERR_POP(mpi_errno);

            if (complete) {
                MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
                MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq);
                continue;
            }
            sreq->dev.iov_offset = 0;
        }
    }

  fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
#define FUNCNAME MPID_nem_scif_iStartContigMsg
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_scif_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz,
                                  void *data, MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr)
{
    int mpi_errno = MPI_SUCCESS;
    MPID_Request *sreq = NULL;
    MPIDI_msg_sz_t offset = 0;
    MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
    scifconn_t *sc = vc_scif->sc;
    uint64_t seqno = 0;
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_ISTARTCONTIGMSG);

    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_ISTARTCONTIGMSG);

    MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));

    MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_iStartContigMsg");
    MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr);

    if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) &&
        MPID_nem_scif_poll_send(sc->fd, &sc->csend)) {
        MPID_IOV iov[2];

        iov[0].MPID_IOV_BUF = hdr;
        iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
        iov[1].MPID_IOV_BUF = data;
        iov[1].MPID_IOV_LEN = data_sz;

        offset = MPID_nem_scif_writev(sc->fd, &sc->csend, iov, 2, &seqno);
        MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER,
                             "**scif_writev", "**scif_writev %s", MPIU_Strerror(errno));
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE,
                         (MPIU_DBG_FDEST, "scif_send " MPIDI_MSG_SZ_FMT " fd=%d",
                          offset, sc->fd));

        if (offset == sizeof(MPIDI_CH3_Pkt_t) + data_sz) {
            if (!seqno) {
                /* sent whole message */
                *sreq_ptr = NULL;
                goto fn_exit;
            }
        }
    }

    /* create and enqueue request */
    MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing");

    /* create a request */
    sreq = MPID_Request_create();
    RQ_SCIF(sreq)->seqno = seqno;
    MPIU_Assert(sreq != NULL);
    MPIU_Object_set_ref(sreq, 2);
    sreq->kind = MPID_REQUEST_SEND;

    sreq->dev.OnDataAvail = 0;
    sreq->ch.vc = vc;
    sreq->dev.iov_offset = 0;

    if (!seqno) {
        if (offset < sizeof(MPIDI_CH3_Pkt_t)) {
            sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) hdr;
            sreq->dev.iov[0].MPID_IOV_BUF = (char *) &sreq->dev.pending_pkt + offset;
            sreq->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t) - offset;
            if (data_sz) {
                sreq->dev.iov[1].MPID_IOV_BUF = data;
                sreq->dev.iov[1].MPID_IOV_LEN = data_sz;
                sreq->dev.iov_count = 2;
            }
            else
                sreq->dev.iov_count = 1;
        }
        else {
            sreq->dev.iov[0].MPID_IOV_BUF = (char *) data + (offset - sizeof(MPIDI_CH3_Pkt_t));
            sreq->dev.iov[0].MPID_IOV_LEN = data_sz - (offset - sizeof(MPIDI_CH3_Pkt_t));
            sreq->dev.iov_count = 1;
        }

        MPIU_Assert(sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0);
    }

    if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) {
        /* this will be the first send on the queue: queue it and set
         * the write flag on the pollfd */
        MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
    }
    else {
        /* there are other sends in the queue before this one: try to
         * send from the queue */
        MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
        mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
    }

    *sreq_ptr = sreq;

  fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_ISTARTCONTIGMSG);
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
#define FUNCNAME MPID_nem_scif_iSendContig
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_scif_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
                              MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz)
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_msg_sz_t offset = 0;
    MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
    scifconn_t *sc = vc_scif->sc;
    uint64_t seqno = 0;
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG);

    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG);

    MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));

    MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_iSendContig");

    MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr);

    if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) &&
        MPID_nem_scif_poll_send(sc->fd, &sc->csend)) {
        MPID_IOV iov[2];

        iov[0].MPID_IOV_BUF = hdr;
        iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
        iov[1].MPID_IOV_BUF = data;
        iov[1].MPID_IOV_LEN = data_sz;

        offset = MPID_nem_scif_writev(sc->fd, &sc->csend, iov, 2, &seqno);
        MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER,
                             "**scif_writev", "**scif_writev %s", MPIU_Strerror(errno));
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE,
                         (MPIU_DBG_FDEST, "scif_send " MPIDI_MSG_SZ_FMT " fd=%d",
                          offset, sc->fd));

        if (offset == sizeof(MPIDI_CH3_Pkt_t) + data_sz) {
            /* sent whole message */
            int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);

            if (seqno)
                goto enqueue_request;
            reqFn = sreq->dev.OnDataAvail;
            if (!reqFn) {
                MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
                MPIDI_CH3U_Request_complete(sreq);
                MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
                goto fn_exit;
            }
            else {
                int complete = 0;

                mpi_errno = reqFn(vc, sreq, &complete);
                if (mpi_errno)
                    MPIU_ERR_POP(mpi_errno);

                if (complete) {
                    MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
                    goto fn_exit;
                }

                /* not completed: more to send */
                goto enqueue_request;
            }
        }
    }

    /* save iov */
    if (offset < sizeof(MPIDI_CH3_Pkt_t)) {
        sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) hdr;
        sreq->dev.iov[0].MPID_IOV_BUF = (char *) &sreq->dev.pending_pkt + offset;
        sreq->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t) - offset;
        if (data_sz) {
            sreq->dev.iov[1].MPID_IOV_BUF = data;
            sreq->dev.iov[1].MPID_IOV_LEN = data_sz;
            sreq->dev.iov_count = 2;
        }
        else
            sreq->dev.iov_count = 1;
        seqno = 0;
    }
    else {
        sreq->dev.iov[0].MPID_IOV_BUF = (char *) data + (offset - sizeof(MPIDI_CH3_Pkt_t));
        sreq->dev.iov[0].MPID_IOV_LEN = data_sz - (offset - sizeof(MPIDI_CH3_Pkt_t));
        sreq->dev.iov_count = 1;
        seqno = 0;
    }

  enqueue_request:
    /* enqueue request */
    RQ_SCIF(sreq)->seqno = seqno;
    MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing");
    MPIU_Assert(seqno || (sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0));

    sreq->ch.vc = vc;
    sreq->dev.iov_offset = 0;

    if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) {
        /* this will be the first send on the queue: queue it and set
         * the write flag on the pollfd */
        MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
    }
    else {
        /* there are other sends in the queue before this one: try to
         * send from the queue */
        MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
        mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
    }

  fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG);
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPID_nem_scif_SendNoncontig
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_nem_scif_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *header,
                                MPIDI_msg_sz_t hdr_sz)
{
    int mpi_errno = MPI_SUCCESS;
    int iov_n;
    MPID_IOV iov[MPID_IOV_LIMIT];
    MPID_IOV *iov_p;
    MPIDI_msg_sz_t offset = 0;
    int complete;
    uint64_t seqno = 0;
    MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG);

    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG);

    MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_SendNoncontig");
    MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));

    iov[0].MPID_IOV_BUF = header;
    iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);

    iov_n = MPID_IOV_LIMIT - 1;

    mpi_errno = MPIDI_CH3U_Request_load_send_iov(sreq, &iov[1], &iov_n);
    MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**loadsendiov");

    iov_n += 1;
    offset = 0;

    if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) &&
        MPID_nem_scif_poll_send(vc_scif->sc->fd, &vc_scif->sc->csend)) {
        offset =
            MPID_nem_scif_writev(vc_scif->sc->fd, &vc_scif->sc->csend, iov, iov_n, &seqno);
        MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER,
                             "**scif_writev", "**scif_writev %s", MPIU_Strerror(errno));

        MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "scif_send noncontig " MPIDI_MSG_SZ_FMT, offset);
    }

    if (offset < iov[0].MPID_IOV_LEN) {
        /* header was not yet sent, save it in req */
        sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) header;
        iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) & sreq->dev.pending_pkt;
        iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
    }

    /* check if whole iov was sent, and save any unsent portion of
     * iov */
    sreq->dev.iov_count = 0;
    complete = 1;
    for (iov_p = &iov[0]; iov_p < &iov[iov_n]; ++iov_p) {
        if (offset < iov_p->MPID_IOV_LEN) {
            sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_BUF =
                (MPID_IOV_BUF_CAST) ((char *) iov_p->MPID_IOV_BUF + offset);
            sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_LEN = iov_p->MPID_IOV_LEN - offset;
            offset = 0;
            ++sreq->dev.iov_count;
            complete = 0;
            seqno = 0;
        }
        else
            offset -= iov_p->MPID_IOV_LEN;
    }
    if (seqno)
        complete = MPID_nem_scif_chk_seqno(&vc_scif->sc->csend, seqno);

    if (complete) {
        /* sent whole iov */
        int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);

        reqFn = sreq->dev.OnDataAvail;
        if (!reqFn) {
            MPIDI_CH3U_Request_complete(sreq);
            MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
            goto fn_exit;
        }

        complete = 0;
        mpi_errno = reqFn(vc, sreq, &complete);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);

        if (complete) {
            MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
            goto fn_exit;
        }
        seqno = 0;
    }

    /* enqueue request */
    MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing");
    MPIU_Assert(seqno || (sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0));

    RQ_SCIF(sreq)->seqno = seqno;
    sreq->ch.vc = vc;
    sreq->dev.iov_offset = 0;

    if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) {
        /* this will be the first send on the queue: queue it and set
         * the write flag on the pollfd */
        MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
    }
    else {
        /* there are other sends in the queue before this one: try to
         * send from the queue */
        MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
        mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
    }

  fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG);
    return mpi_errno;
  fn_fail:
    MPIU_Object_set_ref(sreq, 0);
    MPIDI_CH3_Request_destroy(sreq);
    goto fn_exit;
}

#undef FUNCNAME
#define FUNCNAME MPID_nem_scif_error_out_send_queue
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPID_nem_scif_error_out_send_queue(struct MPIDI_VC *const vc, int req_errno)
{
    int mpi_errno = MPI_SUCCESS;
    MPID_Request *req;
    MPID_nem_scif_vc_area *const vc_scif = VC_SCIF(vc);
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_ERROR_OUT_SEND_QUEUE);

    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_ERROR_OUT_SEND_QUEUE);

    /* we don't call onDataAvail or onFinal handlers because this is
     * an error condition and we just want to mark them as complete */

    /* send queue */
    while (!MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) {
        MPIDI_CH3I_Sendq_dequeue(&vc_scif->send_queue, &req);
        req->status.MPI_ERROR = req_errno;

        MPIDI_CH3U_Request_complete(req);
    }

    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_ERROR_OUT_SEND_QUEUE);
    return mpi_errno;
}
