/* -*- C -*-
 *
 * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2011 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2011      Los Alamos National Security, LLC.
 *                         All rights reserved.
 * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
 * Copyright (c) 2017      Research Organization for Information Science
 *                         and Technology (RIST). All rights reserved.
 * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
 * Copyright (c) 2020      IBM Corporation.  All rights reserved.
 * Copyright (c) 2021-2026 Nanook Consulting  All rights reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */
/** @file:
 *
 */

/*
 * includes
 */
#include "prte_config.h"

#include <string.h>
#ifdef HAVE_SYS_TIME_H
#    include <sys/time.h>
#endif

#include "src/mca/mca.h"
#include "src/threads/pmix_threads.h"
#include "src/util/pmix_argv.h"
#include "src/util/pmix_environ.h"

#include "constants.h"
#include "src/mca/errmgr/errmgr.h"
#include "src/mca/ess/ess.h"
#include "src/mca/ras/base/base.h"
#include "src/rml/rml.h"
#include "src/mca/schizo/base/base.h"
#include "src/mca/state/state.h"
#include "src/pmix/pmix-internal.h"
#include "src/runtime/prte_globals.h"
#include "src/runtime/prte_quit.h"
#include "src/util/error_strings.h"
#include "src/util/name_fns.h"
#include "src/util/proc_info.h"
#include "src/util/pmix_show_help.h"
#include "types.h"

#include "src/mca/plm/base/base.h"
#include "src/mca/plm/base/plm_private.h"
#include "src/mca/plm/plm.h"
#include "src/mca/plm/plm_types.h"

#include "src/prted/pmix/pmix_server_internal.h"

static bool recv_issued = false;

int prte_plm_base_comm_start(void)
{
    if (recv_issued) {
        return PRTE_SUCCESS;
    }

    PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                         "%s plm:base:receive start comm", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));

    PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_PLM,
                  PRTE_RML_PERSISTENT, prte_plm_base_recv, NULL);
    if (PRTE_PROC_IS_MASTER) {
        PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_PRTED_CALLBACK,
                      PRTE_RML_PERSISTENT, prte_plm_base_daemon_callback, NULL);
        PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_REPORT_REMOTE_LAUNCH,
                      PRTE_RML_PERSISTENT, prte_plm_base_daemon_failed, NULL);
        PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_TOPOLOGY_REPORT,
                      PRTE_RML_PERSISTENT, prte_plm_base_daemon_topology, NULL);
        PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_STACK_TRACE,
                      PRTE_RML_PERSISTENT, prte_plm_base_stack_trace_recv, NULL);
    }
    recv_issued = true;

    return PRTE_SUCCESS;
}

int prte_plm_base_comm_stop(void)
{
    if (!recv_issued) {
        return PRTE_SUCCESS;
    }

    PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                         "%s plm:base:receive stop comm", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));

    PRTE_RML_CANCEL(PRTE_NAME_WILDCARD, PRTE_RML_TAG_PLM);
    if (PRTE_PROC_IS_MASTER) {
        PRTE_RML_CANCEL(PRTE_NAME_WILDCARD, PRTE_RML_TAG_PRTED_CALLBACK);
        PRTE_RML_CANCEL(PRTE_NAME_WILDCARD, PRTE_RML_TAG_REPORT_REMOTE_LAUNCH);
        PRTE_RML_CANCEL(PRTE_NAME_WILDCARD, PRTE_RML_TAG_TOPOLOGY_REPORT);
        PRTE_RML_CANCEL(PRTE_NAME_WILDCARD, PRTE_RML_TAG_STACK_TRACE);
    }
    recv_issued = false;

    return PRTE_SUCCESS;
}

/* process incoming messages in order of receipt */
void prte_plm_base_recv(int status, pmix_proc_t *sender,
                        pmix_data_buffer_t *buffer,
                        prte_rml_tag_t tag, void *cbdata)
{
    prte_plm_cmd_flag_t command;
    int32_t count;
    pmix_nspace_t job;
    prte_session_t *session;
    prte_job_t *jdata, *parent;
    pmix_data_buffer_t *answer;
    pmix_rank_t vpid;
    prte_proc_t *proc;
    prte_node_t *node;
    prte_proc_state_t state;
    prte_exit_code_t exit_code;
    int32_t rc = PRTE_SUCCESS, ret;
    uint32_t ui32, *ui32_ptr;
    prte_app_context_t *app, *child_app;
    pmix_proc_t name, *nptr;
    pid_t pid;
    bool debugging, found;
    int i, room, *rmptr = &room;
    char *tmp;
    pmix_value_t pidval = PMIX_VALUE_STATIC_INIT;
    PRTE_HIDE_UNUSED_PARAMS(status, tag, cbdata);

    PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                         "%s plm:base:receive processing msg",
                         PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));

    count = 1;
    rc = PMIx_Data_unpack(NULL, buffer, &command, &count, PMIX_UINT8);
    if (PMIX_SUCCESS != rc) {
        PMIX_ERROR_LOG(rc);
        goto CLEANUP;
    }

    switch (command) {
    case PRTE_PLM_TOOL_ATTACHED_CMD:
        /* set default return value */
        ret = PMIX_SUCCESS;

        /* unpack the room number of the request so we can return it to them */
        count = 1;
        rc = PMIx_Data_unpack(NULL, buffer, &room, &count, PMIX_INT);
        if (PMIX_SUCCESS != rc) {
            PMIX_ERROR_LOG(rc);
            goto CLEANUP;
        }
        // see if they need a new jobid assigned
        count = 1;
        rc = PMIx_Data_unpack(NULL, buffer, &found, &count, PMIX_BOOL);
        if (PMIX_SUCCESS != rc) {
            PMIX_ERROR_LOG(rc);
            goto CLEANUP;
        }
        // if found, then the tool already has a jobid and they will
        // be passing us the full procID of the tool
        if (found) {
            // get the full procID of the tool
            count = 1;
            rc = PMIx_Data_unpack(NULL, buffer, &name, &count, PMIX_PROC);
            if (PMIX_SUCCESS != rc) {
                PMIX_ERROR_LOG(rc);
                goto CLEANUP;
            }

        } else {
            // if not found, then they will pass us the rank and we will
            // assign a new jobID for the tool
            count = 1;
            rc = PMIx_Data_unpack(NULL, buffer, &name.rank, &count, PMIX_PROC_RANK);
            if (PMIX_SUCCESS != rc) {
                PMIX_ERROR_LOG(rc);
                goto CLEANUP;
            }
            /* the new nspace is our base nspace with an "@N" extension */
            pmix_asprintf(&tmp, "%s@%u", prte_plm_globals.base_nspace, prte_plm_globals.next_jobid);
            PMIX_LOAD_NSPACE(name.nspace, tmp);
            free(tmp);
            prte_plm_globals.next_jobid++;
        }
        // see if we already have this job
        jdata = prte_get_job_data_object(name.nspace);
        if (NULL != jdata) {
            // we do - check if we already have this proc
            proc = (prte_proc_t*)pmix_pointer_array_get_item(jdata->procs, name.rank);
            if (NULL == proc) {
                // new rank for this tool - add it
                proc = PMIX_NEW(prte_proc_t);
                memcpy(&proc->name, &name, sizeof(pmix_proc_t));
                proc->state = PRTE_PROC_STATE_RUNNING;
                pmix_pointer_array_set_item(jdata->procs, name.rank, proc);
                jdata->num_procs++;
            }
            ret = PMIX_SUCCESS;

        } else {
            // unpack the cmd line
            count = 1;
            rc = PMIx_Data_unpack(NULL, buffer, &tmp, &count, PMIX_STRING);
            if (PMIX_SUCCESS != rc) {
                PMIX_ERROR_LOG(rc);
                goto CLEANUP;
            }
            // and the pid
            count = 1;
            rc = PMIx_Data_unpack(NULL, buffer, &pid, &count, PMIX_PID);
            if (PMIX_SUCCESS != rc) {
                PMIX_ERROR_LOG(rc);
                free(tmp);
                goto CLEANUP;
            }

            // need to add the tool job
            jdata = PMIX_NEW(prte_job_t);
            PMIX_LOAD_NSPACE(jdata->nspace, name.nspace);
            PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_TOOL);
            rc = prte_set_job_data_object(jdata);
            app = PMIX_NEW(prte_app_context_t);
            if (NULL != tmp) {
                app->argv = PMIx_Argv_split(tmp, ' ');
                free(tmp);
                app->app = strdup(app->argv[0]);
            } else {
                app->app = strdup("unknown");
                PMIx_Argv_append_nosize(&app->argv, "unknown");
            }
            pmix_pointer_array_set_item(jdata->apps, 0, app);
            jdata->num_apps++;
            proc = PMIX_NEW(prte_proc_t);
            memcpy(&proc->name, &name, sizeof(pmix_proc_t));
            proc->pid = pid;
            proc->state = PRTE_PROC_STATE_RUNNING;
            pmix_pointer_array_set_item(jdata->procs, name.rank, proc);
            // find the node it is on
            node = (prte_node_t*)pmix_pointer_array_get_item(prte_node_pool, sender->rank);
            if (NULL == node) {
                PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
                rc = PRTE_ERR_NOT_FOUND;
                goto CLEANUP;
            }
            PMIX_RETAIN(node);
            proc->node = node;
            jdata->num_procs = 1;
        }
        /* setup the response */
        PMIX_DATA_BUFFER_CREATE(answer);

        /* pack the status to be returned */
        rc = PMIx_Data_pack(NULL, answer, &ret, 1, PMIX_INT32);
        if (PMIX_SUCCESS != rc) {
            PMIX_ERROR_LOG(rc);
        }

        /* pack the room number of the request */
        rc = PMIx_Data_pack(NULL, answer, &room, 1, PMIX_INT);
        if (PMIX_SUCCESS != rc) {
            PMIX_ERROR_LOG(rc);
        }

        if (PRTE_SUCCESS == ret) {
            // pack the namespace of the tool - we may have provided the namespace,
            // or it may be just the one they sent us
            rc = PMIx_Data_pack(NULL, answer, name.nspace, 1, PMIX_PROC_NSPACE);
            if (PMIX_SUCCESS != rc) {
                PMIX_ERROR_LOG(rc);
            }
        }

        /* send the response back to the sender */
        PRTE_RML_SEND(ret, sender->rank, answer, PRTE_RML_TAG_TCONN_RESP);
        if (PRTE_SUCCESS != ret) {
            PRTE_ERROR_LOG(ret);
            PMIX_DATA_BUFFER_RELEASE(answer);
        }
        break;

    case PRTE_PLM_LAUNCH_JOB_CMD:
        PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                             "%s plm:base:receive job launch command from %s",
                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender)));

        /* unpack the job object */
        count = 1;
        rc = prte_job_unpack(buffer, &jdata);
        if (PRTE_SUCCESS != rc) {
            PRTE_ERROR_LOG(rc);
            goto ANSWER_LAUNCH;
        }

        /* record the sender so we know who to respond to */
        PMIX_LOAD_PROCID(&jdata->originator, sender->nspace, sender->rank);

        /* assign a schizo module */
        if (NULL == jdata->personality) {
            PMIx_Argv_append_nosize(&jdata->personality, "prte");
        }
        tmp = PMIx_Argv_join(jdata->personality, ',');
        jdata->schizo = (struct prte_schizo_base_module_t*)prte_schizo_base_detect_proxy(tmp);
        if (NULL == jdata->schizo) {
            pmix_show_help("help-schizo-base.txt", "no-proxy", true, prte_tool_basename, tmp);
            free(tmp);
            rc = PRTE_ERR_NOT_FOUND;
            goto ANSWER_LAUNCH;
        }
        free(tmp);

        /* get the name of the actual spawn parent - i.e., the proc that actually
         * requested the spawn */
        if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_LAUNCH_PROXY, (void **) &nptr, PMIX_PROC)) {
            PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
            rc = PRTE_ERR_NOT_FOUND;
            goto ANSWER_LAUNCH;
        }

        /* If an alloc id was given specifying the session within which
         * the job is to be spawned, then use it - otherwise default to parent session */
        session = NULL;
        ui32_ptr = &ui32;
        if (prte_get_attribute(&jdata->attributes, PRTE_JOB_SESSION_ID, (void **) &ui32_ptr, PMIX_UINT32)) {
            session = prte_get_session_object(ui32);
            if (NULL == session) {
                /* if the caller specified a session and we don't know about it, then
                 * that is an unrecoverable error */
                rc = PRTE_ERR_NOT_FOUND;
                goto ANSWER_LAUNCH;
            }
            goto moveon;
        }

        tmp = NULL;
        if (prte_get_attribute(&jdata->attributes, PRTE_JOB_ALLOC_ID, (void **) &tmp, PMIX_STRING) &&
            NULL != tmp) {
            session = prte_get_session_object_from_id(tmp);
            free(tmp);
            if (NULL == session) {
                /* if the caller specified a session and we don't know about it, then
                 * that is an unrecoverable error */
                rc = PRTE_ERR_NOT_FOUND;
                goto ANSWER_LAUNCH;
            }
            goto moveon;
        }

        tmp = NULL;
        if (prte_get_attribute(&jdata->attributes, PRTE_JOB_REF_ID, (void **) &tmp, PMIX_STRING) &&
            NULL != tmp) {
            session = prte_get_session_object_from_refid(tmp);
            free(tmp);
            if (NULL == session) {
                /* if the caller specified a session and we don't know about it, then
                 * that is an unrecoverable error */
                rc = PRTE_ERR_NOT_FOUND;
                goto ANSWER_LAUNCH;
            }
            goto moveon;
        }

        /* try defaulting to parent session */
        parent = prte_get_job_data_object(nptr->nspace);
        if (NULL != parent) {
            /* if the proc requesting the spawn is a tool, it does not have a
             * session - so assign it the default session */
            if (PRTE_FLAG_TEST(parent, PRTE_JOB_FLAG_TOOL)) {
                session = prte_default_session;
            } else {
                session = parent->session;
                if (NULL == session) {
                    rc = PRTE_ERR_NOT_FOUND;
                    goto ANSWER_LAUNCH;
                }
            }

        // (RHC) This next clause merits some thought - not sure I fully
        // understand the conditionals
        } else if (!prte_pmix_server_globals.scheduler_connected ||
                   PMIX_CHECK_PROCID(nptr, &prte_pmix_server_globals.scheduler)) {
            /* The proc requesting the spawn is a tool, hence does not have a session.
             * If we don't have a scheduler connected (or the tool itself is the scheduler) we allow
             * it to spawn into the default session, i.e. the global node pool
             */
            session = prte_default_session;
        } else {
            PRTE_ERROR_LOG(PRTE_ERR_PERM);
            rc = PRTE_ERR_PERM;
            goto ANSWER_LAUNCH;
        }

moveon:
        jdata->session = session;
        pmix_pointer_array_add(jdata->session->jobs, jdata);

        /* get the parent's job object */
        if (NULL != (parent = prte_get_job_data_object(nptr->nspace)) &&
            !PMIX_CHECK_NSPACE(parent->nspace, PRTE_PROC_MY_NAME->nspace)) {
            /* link the spawned job to the spawner */
            PMIX_RETAIN(jdata);
            pmix_list_append(&parent->children, &jdata->super);
            /* connect the launcher as well */
            if (PMIX_NSPACE_INVALID(parent->launcher)) {
                /* we are an original spawn */
                PMIX_LOAD_NSPACE(jdata->launcher, nptr->nspace);
            } else {
                PMIX_LOAD_NSPACE(jdata->launcher, parent->launcher);
            }
            /* if the prefix was set in the parent's job, we need to transfer
             * that prefix to the child's app_context so any further launch of
             * procs can find the correct library. There always has to be at
             * least one app_context in both parent and child, so we don't
             * need to check that here. However, be sure not to overwrite
             * the prefix if the user already provided it!
             */
            app = (prte_app_context_t *) pmix_pointer_array_get_item(parent->apps, 0);
            child_app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, 0);
            if (NULL != app && NULL != app->env && NULL != child_app) {
                for (i=0; NULL != app->env[i]; i++) {
                    if (0 == strncmp(app->env[i], "PMIX_PREFIX", strlen("PMIX_PREFIX"))) {
                        PMIx_Argv_append_unique_nosize(&child_app->env, app->env[i]);
                        break;
                    }
                }
            }
        }
        PMIX_PROC_RELEASE(nptr);

        PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                             "%s plm:base:receive adding hosts",
                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));

        /* process any add-hostfile and add-host options that were provided */
        if (PRTE_SUCCESS != (rc = prte_ras_base_add_hosts(jdata))) {
            PRTE_ERROR_LOG(rc);
            goto ANSWER_LAUNCH;
        }

        if (NULL != parent && !PRTE_FLAG_TEST(parent, PRTE_JOB_FLAG_TOOL) &&
            !PMIX_CHECK_NSPACE(parent->nspace, PRTE_PROC_MY_NAME->nspace)) {
            if (NULL == parent->bookmark) {
                /* find the sender's node in the job map */
                proc = (prte_proc_t *) pmix_pointer_array_get_item(parent->procs, sender->rank);
                if (NULL != proc) {
                    /* set the bookmark so the child starts from that place - this means
                     * that the first child process could be co-located with the proc
                     * that called comm_spawn, assuming slots remain on that node. Otherwise,
                     * the procs will start on the next available node
                     */
                    jdata->bookmark = proc->node;
                }
            } else {
                jdata->bookmark = parent->bookmark;
            }
        }

        if (!prte_dvm_ready) {
            pmix_pointer_array_add(prte_cache, jdata);
            return;
        }

        /* launch it */
        PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                             "%s plm:base:receive calling spawn",
                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
        if (PRTE_SUCCESS != (rc = prte_plm.spawn(jdata))) {
            PRTE_ERROR_LOG(rc);
            goto ANSWER_LAUNCH;
        }
        break;


    ANSWER_LAUNCH:
        PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                             "%s plm:base:receive - error on launch: %d",
                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), rc));

        /* setup the response */
        PMIX_DATA_BUFFER_CREATE(answer);

        /* pack the error code to be returned */
        rc = PMIx_Data_pack(NULL, answer, &rc, 1, PMIX_INT32);
        if (PMIX_SUCCESS != rc) {
            PMIX_ERROR_LOG(rc);
        }

        /* pack an invalid jobid */
        PMIX_LOAD_NSPACE(job, NULL);
        rc = PMIx_Data_pack(NULL, answer, &job, 1, PMIX_PROC_NSPACE);
        if (PMIX_SUCCESS != rc) {
            PMIX_ERROR_LOG(rc);
        }

        /* pack the room number of the request */
        if (prte_get_attribute(&jdata->attributes, PRTE_JOB_ROOM_NUM, (void **) &rmptr, PMIX_INT)) {
            rc = PMIx_Data_pack(NULL, answer, &room, 1, PMIX_INT);
            if (PMIX_SUCCESS != rc) {
                PMIX_ERROR_LOG(rc);
            }
        }

        /* send the response back to the sender */
        PRTE_RML_SEND(ret, sender->rank, answer, PRTE_RML_TAG_LAUNCH_RESP);
        if (PRTE_SUCCESS != ret) {
            PRTE_ERROR_LOG(ret);
            PMIX_DATA_BUFFER_RELEASE(answer);
        }
        break;


    case PRTE_PLM_UPDATE_PROC_STATE:
        pmix_output_verbose(5, prte_plm_base_framework.framework_output,
                            "%s plm:base:receive update proc state command from %s",
                            PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender));
        count = 1;
        rc = PMIx_Data_unpack(NULL, buffer, &job, &count, PMIX_PROC_NSPACE);
        while (PMIX_SUCCESS == rc) {
            PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                                "%s plm:base:receive got update_proc_state for job %s",
                                PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(job)));

            PMIX_LOAD_NSPACE(name.nspace, job);
            /* get the job object */
            jdata = prte_get_job_data_object(job);
            count = 1;
            rc = PMIx_Data_unpack(NULL, buffer, &vpid, &count, PMIX_PROC_RANK);
            while (PMIX_SUCCESS == rc) {
                if (PMIX_RANK_INVALID == vpid) {
                    /* flag indicates that this job is complete - move on */
                    break;
                }
                name.rank = vpid;
                /* unpack the pid */
                count = 1;
                rc = PMIx_Data_unpack(NULL, buffer, &pid, &count, PMIX_PID);
                if (PMIX_SUCCESS != rc) {
                    PMIX_ERROR_LOG(rc);
                    goto CLEANUP;
                }
                /* unpack the state */
                count = 1;
                rc = PMIx_Data_unpack(NULL, buffer, &state, &count, PMIX_UINT32);
                if (PMIX_SUCCESS != rc) {
                    PMIX_ERROR_LOG(rc);
                    goto CLEANUP;
                }
                /* unpack the exit code */
                count = 1;
                rc = PMIx_Data_unpack(NULL, buffer, &exit_code, &count, PMIX_INT32);
                if (PMIX_SUCCESS != rc) {
                    PMIX_ERROR_LOG(rc);
                    goto CLEANUP;
                }

                PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                                     "%s plm:base:receive got update_proc_state for vpid %u pid %d state %s exit_code %d",
                                     PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), vpid, (int)pid, prte_proc_state_to_str(state),
                                     (int) exit_code));

                if (NULL != jdata) {
                    /* get the proc data object */
                    proc = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, vpid);
                    if (NULL == proc) {
                        PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
                        PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FORCED_EXIT);
                        goto CLEANUP;
                    }
                    /* NEVER update the proc state before activating the state machine - let
                     * the state cbfunc update it as it may need to compare this
                     * state against the prior proc state */
                    proc->pid = pid;
                    proc->exit_code = exit_code;
                    PRTE_ACTIVATE_PROC_STATE(&name, state);
                }
                /* get entry from next rank */
                rc = PMIx_Data_unpack(NULL, buffer, &vpid, &count, PMIX_PROC_RANK);
            }
            /* prepare for next job */
            count = 1;
            rc = PMIx_Data_unpack(NULL, buffer, &job, &count, PMIX_PROC_NSPACE);
        }
        if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
            PMIX_ERROR_LOG(rc);
            rc = prte_pmix_convert_status(rc);
        } else {
            rc = PRTE_SUCCESS;
        }
        break;

    case PRTE_PLM_READY_FOR_DEBUG_CMD:
            PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                            "%s plm:base:receive ready for debug command from %s",
                            PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender)));
        count = 1;
        rc = PMIx_Data_unpack(NULL, buffer, &job, &count, PMIX_PROC_NSPACE);
        while (PMIX_SUCCESS == rc) {
            PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                                "%s plm:base:receive got ready for debug for job %s",
                                PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(job)));

            PMIX_LOAD_NSPACE(name.nspace, job);
            /* get the job object */
            jdata = prte_get_job_data_object(job);
            debugging = false;
            if (prte_get_attribute(&jdata->attributes, PRTE_JOB_STOP_ON_EXEC, NULL, PMIX_BOOL) ||
                prte_get_attribute(&jdata->attributes, PRTE_JOB_STOP_IN_INIT, NULL, PMIX_BOOL) ||
                prte_get_attribute(&jdata->attributes, PRTE_JOB_STOP_IN_APP, NULL, PMIX_BOOL)) {
                debugging = true;
            }
            count = 1;
            rc = PMIx_Data_unpack(NULL, buffer, &vpid, &count, PMIX_PROC_RANK);
            while (PMIX_SUCCESS == rc) {
                if (PMIX_RANK_INVALID == vpid) {
                    /* flag indicates that this job is complete - move on */
                    break;
                }
                name.rank = vpid;
                /* unpack the pid */
                count = 1;
                rc = PMIx_Data_unpack(NULL, buffer, &pid, &count, PMIX_PID);
                if (PMIX_SUCCESS != rc) {
                    PMIX_ERROR_LOG(rc);
                    goto CLEANUP;
                }

                PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                     "%s plm:base:receive got ready for debug for vpid %u",
                     PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), vpid));

                /* get the proc data object */
                proc = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, vpid);
                if (NULL == proc) {
                    PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
                    PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FORCED_EXIT);
                    goto CLEANUP;
                }
                /* NEVER update the proc state before activating the state machine - let
                 * the state cbfunc update it as it may need to compare this
                 * state against the prior proc state */
                proc->pid = pid;
                if (debugging) {
                    jdata->num_ready_for_debug++;
                }
                /* get entry from next rank */
                rc = PMIx_Data_unpack(NULL, buffer, &vpid, &count, PMIX_PROC_RANK);
            }
            if (debugging && jdata->num_ready_for_debug == jdata->num_procs) {
                PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_READY_FOR_DEBUG);
            }
            /* prepare for next job */
            count = 1;
            rc = PMIx_Data_unpack(NULL, buffer, &job, &count, PMIX_PROC_NSPACE);
        }
        if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
            PMIX_ERROR_LOG(rc);
            rc = prte_pmix_convert_status(rc);
        } else {
            rc = PRTE_SUCCESS;
        }
        break;

    case PRTE_PLM_REGISTERED_CMD:
            PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                            "%s plm:base:receive registered command from %s",
                            PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender)));
        count = 1;
        rc = PMIx_Data_unpack(NULL, buffer, &job, &count, PMIX_PROC_NSPACE);
        if (PMIX_SUCCESS != rc) {
            PMIX_ERROR_LOG(rc);
            goto CLEANUP;
        }
        PMIX_LOAD_NSPACE(name.nspace, job);
            PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                            "%s plm:base:receive got registered for job %s",
                            PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(job)));
        /* get the job object */
        if (NULL == (jdata = prte_get_job_data_object(job))) {
            PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
            rc = PRTE_ERR_NOT_FOUND;
            goto CLEANUP;
        }
        count = 1;
        while (PRTE_SUCCESS == PMIx_Data_unpack(NULL, buffer, &vpid, &count, PMIX_PROC_RANK)) {
            PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                                 "%s plm:base:receive got registered for vpid %u",
                                 PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), vpid));
            proc = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, vpid);
            if (NULL == proc) {
                PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
                PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FORCED_EXIT);
                goto CLEANUP;
            }
            proc->state = PRTE_PROC_STATE_REGISTERED;
            jdata->num_reported++;
            count = 1;
        }
        if (jdata->num_reported == jdata->num_procs) {
            PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_REGISTERED);
        }
        break;

    case PRTE_PLM_LOCAL_LAUNCH_COMP_CMD:
        PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                            "%s plm:base:receive local launch complete command from %s",
                            PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender)));
        count = 1;
        rc = PMIx_Data_unpack(NULL, buffer, &job, &count, PMIX_PROC_NSPACE);
        if (PMIX_SUCCESS != rc) {
            PMIX_ERROR_LOG(rc);
            goto CLEANUP;
        }
        PMIX_LOAD_NSPACE(name.nspace, job);
            PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                            "%s plm:base:receive got local launch complete for job %s",
                            PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(job)));
        /* get the job object */
        if (NULL == (jdata = prte_get_job_data_object(job))) {
            PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
            rc = PRTE_ERR_NOT_FOUND;
            goto CLEANUP;
        }
        count = 1;
        while (PRTE_SUCCESS == PMIx_Data_unpack(NULL, buffer, &vpid, &count, PMIX_PROC_RANK)) {
            PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                                 "%s plm:base:receive got local launch complete for vpid %s",
                                 PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_VPID_PRINT(vpid)));
            proc = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, vpid);
            if (NULL == proc) {
                PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
                PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FORCED_EXIT);
                goto CLEANUP;
            }
            /* unpack the pid */
            count = 1;
            rc = PMIx_Data_unpack(NULL, buffer, &pid, &count, PMIX_PID);
            if (PMIX_SUCCESS != rc) {
                PMIX_ERROR_LOG(rc);
                goto CLEANUP;
            }
            proc->pid = pid;
            pidval.type = PMIX_PID;
            pidval.data.pid = pid;
            /* store the PID for later retrieval */
            rc = PMIx_Store_internal(&proc->name, PMIX_PROC_PID, &pidval);
            if (PMIX_SUCCESS != rc) {
                PMIX_ERROR_LOG(rc);
            }
            /* unpack the state */
            count = 1;
            rc = PMIx_Data_unpack(NULL, buffer, &state, &count, PMIX_UINT32);
            if (PMIX_SUCCESS != rc) {
                PMIX_ERROR_LOG(rc);
                goto CLEANUP;
            }
            PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                                 "%s plm:base:receive got local launch complete for vpid %u state %s",
                                 PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), vpid,
                                 prte_proc_state_to_str(state)));
            /* if the child failed to start, unpack its exit code */
            if (PRTE_PROC_STATE_RUNNING != state) {
                count = 1;
                rc = PMIx_Data_unpack(NULL, buffer, &exit_code, &count, PMIX_INT32);
                if (PMIX_SUCCESS != rc) {
                    PMIX_ERROR_LOG(rc);
                    goto CLEANUP;
                }
                proc->exit_code = exit_code;
            }
            proc->state = state;
            jdata->num_launched++;
            count = 1;
        }
        /* record that we heard back from a daemon during app launch */
        jdata->num_daemons_reported++;
        found = prte_get_attribute(&jdata->attributes, PRTE_JOB_SHOW_PROGRESS, NULL, PMIX_BOOL);
        if (found) {
            if (0 == jdata->num_daemons_reported % 100 ||
                jdata->num_daemons_reported == prte_process_info.num_daemons) {
                PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_REPORT_PROGRESS);
            }
        }
        if (1 == jdata->num_launched) {
            PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_STARTED);
        }
        if (jdata->num_launched == jdata->num_procs) {
            PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_RUNNING);
        }
        break;

    default:
        PRTE_ERROR_LOG(PRTE_ERR_VALUE_OUT_OF_BOUNDS);
        rc = PRTE_ERR_VALUE_OUT_OF_BOUNDS;
        break;
    }

CLEANUP:
    /* see if an error occurred - if so, wakeup the HNP so we can exit */
    if (PRTE_PROC_IS_MASTER && PRTE_SUCCESS != rc) {
        jdata = NULL;
        PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT);
    }

    PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output,
                         "%s plm:base:receive done processing commands",
                         PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
}

/* where HNP messages come */
void prte_plm_base_receive_process_msg(int fd, short event, void *data)
{
    PRTE_HIDE_UNUSED_PARAMS(fd, event, data);
    assert(0);
}
