topodistNodeCommInfo

Provide some information about off-node communication

Synopsis

int topodistNodeCommInfo(topodist_t *td, MPI_Comm comm,
                       int *partnerOnNode, int *partnerOffNode,
                       MPI_Comm *nodecomm_p, int *totalOffNode)

Input Parameters

td
Topodist information returned by topodistInit
comm
Communicator of processes to use. This routine is collective over this communicator. The most common choice of comm is MPI_COMM_WORLD

Output Parameters

partnerOnNode
Number of processes that this process communicates with that are on the same node
partnerOffNode
As above, but for partners that are off node
nodecomm_p
A communicator containing only the processes that share the node with the calling process.
totalOffNode
The total number of processes on this node that are communicating to one or more processes on a different node
*/ int topodistNodeCommInfo(topodist_t *td, MPI_Comm comm, int *partnerOnNode, int *partnerOffNode, MPI_Comm *nodecomm_p, int *totalOffNode) { int nOffNode, /* Number of partners off of the node for each process */ nOnNode, /* Number of partners on the node for each process */ i, j, sameNode; /* */ int nodeOnNode, nodeOffNode; int color, rank, nrank, nsize, anyoff; MPI_Comm nodecomm;

nOffNode = 0; nOnNode = 0;

/* For each partner, compute on/offnode information */ for (i=0; i<td->ntargets; i++) { /* are the topoinfo values comparable? */ sameNode = 0; if (td->tarray[i].nodeidx == td->myarray.nodeidx && td->tarray[i].nlevels == td->myarray.nlevels) { /* Check the topo indices above the node */ sameNode = 1; for (j=td->myarray.nodeidx; j<td->myarray.nlevels; j++) { if (td->tarray[i].mycoords[j] != td->myarray.mycoords[j]) sameNode = 0; } } if (sameNode) { nOnNode++; } else { nOffNode++; } }

*partnerOnNode = nOnNode; *partnerOffNode = nOffNode;

/* Find all of information by node, as defined above */ MPI_Comm_rank(comm, &rank); color = 0; /* Check for the case where no interconnect network is known */ if (td->myarray.nodeidx < td->myarray.nlevels) color = td->myarray.mycoords[td->myarray.nodeidx]; for (i=td->myarray.nodeidx+1; i<td->myarray.nlevels; i++) color = td->myarray.mycoords[i] + td->myarray.maxcoords[i-1]*color; MPI_Comm_split(comm, color, rank, &nodecomm);

anyoff = (nOffNode > 0); MPI_Allreduce(&anyoff, totalOffNode, 1, MPI_INT, MPI_SUM, nodecomm); MPI_Comm_rank(nodecomm, &nrank); *nodecomm_p = nodecomm;

return 0; }

#include <string.h> int topodistPrint(topodist_t *td, FILE *fp) { int nOffNode, nOnNode, totalOffNode, nodeOnNode, nodeOffNode, nrank, nsize, rank, size, err; MPI_Comm nodecomm;

MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size);

if (rank == 0) { printf("Test of on/off node determination\n"); fflush(stdout); }

err = topodistNodeCommInfo(td, MPI_COMM_WORLD, &nOnNode, &nOffNode, &nodecomm, &totalOffNode);

MPI_Allreduce(&nOnNode, &nodeOnNode, 1, MPI_INT, MPI_SUM, nodecomm); MPI_Allreduce(&nOffNode, &nodeOffNode, 1, MPI_INT, MPI_SUM, nodecomm);

MPI_Comm_rank(nodecomm, &nrank); MPI_Comm_size(nodecomm, &nsize); if (nrank == 0) { printf("rank = %d, nsize = %d, Sends: Node On = %d, Off = %d; procs sending off = %d\n", rank, nsize, nodeOnNode, nodeOffNode, totalOffNode); fflush(stdout); } MPI_Comm_free(&nodecomm); MPI_Barrier(MPI_COMM_WORLD);

{ char coordStr[1024]; int j; coordStr[0] = 0; for (j=td->myarray.nodeidx; j<td->myarray.nlevels; j++) { char numstr[20]; snprintf(numstr, 20, "%d", td->myarray.mycoords[j]); strncat(coordStr, numstr, 1024); strncat(coordStr, ",", 1024); } if (rank > 0) { MPI_Recv((void*)0, 0, MPI_INT, rank-1, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } printf("%d:[%s] OnNode = %d, OffNode = %d\n", rank, coordStr, nOnNode, nOffNode); fflush(stdout); if (rank < size-1) { MPI_Send((void*)0, 0, MPI_INT, rank+1, 1, MPI_COMM_WORLD); } } return 0; }

#include <math.h> void topodistTest(FILE *fp) { topodist_t *dt; topoinfo_t *ti; int rank, size, sends[4], m, ns; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); m = sqrt((double)size); ns = 0; if (rank > 0) sends[ns++] = rank - 1; if (rank < size-1) sends[ns++] = rank + 1; if (rank - m >= 0) sends[ns++] = rank - m; if (rank + m <= size-1) sends[ns++] = rank + m; topoInit(1,&ti); topodistInit(MPI_COMM_WORLD, ns, sends, ns, sends, ti, &dt); topodistPrint(dt, fp); topodistFree(dt); }