Nature Language: multipleMultiPingpong:Multiple threads take place in the communication and computation. Each thread of all MPI processes in crossComm = pingNode sends a portion of the message to the neighbouring process in crossComm = pongNode. Each thread of the neighbouring processes then sends the message back to the first process.
  
CPP: int multipleMultiPingpong(int totalReps, int dataSize){ \n    int repIter, i; \n    int lBound; \n    #pragma omp parallel default(none) \\\n    private(i,repIter,status,lBound) \\\n    shared(pingNode,pongNode,pingSendBuf,pingRecvBuf) \\\n    shared(pongSendBuf,pongRecvBuf,finalRecvBuf,sizeofBuffer) \\\n    shared(dataSize,globalIDarray,crossComm) \\\n    shared(totalReps,myMPIRank,crossCommRank) \n    { \n        for (repIter=1; repIter<=totalReps; repIter++){ /* loop totalRep times */ \n            if (crossCommRank == pingNode){ \n                lBound = (myThreadID * dataSize); \n                #pragma omp for nowait schedule(static,dataSize) \n                for (i=0; i<sizeofBuffer; i++){ \n                    pingSendBuf[i] = globalIDarray[myThreadID]; \n                } \n                MPI_Send(&pingSendBuf[lBound], dataSize, MPI_INT, pongNode, \\\n                myThreadID, crossComm); \n                MPI_Recv(&pongRecvBuf[lBound], dataSize, MPI_INT, pongNode, \\\n                myThreadID, crossComm, &status); \n                #pragma omp for nowait schedule(static,dataSize) \n                for (i=0; i<sizeofBuffer; i++){ \n                    finalRecvBuf[i] = pongRecvBuf[i]; \n                } \n            } else if (crossCommRank == pongNode){ \n                lBound = (myThreadID * dataSize); \n                MPI_Recv(&pingRecvBuf[lBound], dataSize, MPI_INT, pingNode, \\\n                myThreadID, crossComm, &status); \n                #pragma omp for nowait schedule(static,dataSize) \n                for (i=0; i<sizeofBuffer; i++){ \n                    pongSendBuf[i] = pingRecvBuf[i]; \n                } \n                MPI_Send(&pongSendBuf[lBound], dataSize, MPI_INT, pingNode, \\\n                myThreadID, crossComm); \n            } \n        } \n    } \n    return 0; \n}",
  
Fortran: SUBROUTINE multipleMultiPingpong(totalReps, dataSize)\n integer, intent(in) :: totalReps, dataSize\n integer :: repIter, i\n integer :: lBound, uBound\n\n !Open parallel region for threads\n!$OMP PARALLEL DEFAULT(NONE), &\n!$OMP PRIVATE(i,repIter,ierr,status,lBound,uBound), &\n!$OMP SHARED(pingNode,pongNode,pingSendBuf,pingRecvBuf),&\n!$OMP SHARED(pongSendBuf,pongRecvBuf,finalRecvBuf,sizeofBuffer),&\n!$OMP SHARED(dataSize,globalIDarray,crossComm), &\n!$OMP SHARED(totalReps,myMPIRank,crossCommRank)\n\n DO repIter = 1, totalReps !loop totalReps time\n\n IF (crossCommRank == pingNode) THEN\n \n !Calculate lower and upper bound of data array\n lBound = ((myThreadID-1)* dataSize) + 1\n uBound = (myThreadID * dataSize)\n\n !All threads write to its part of the pingBuf\n !array using a PARALLEL DO directive\n!$OMP DO SCHEDULE(STATIC,dataSize)\n DO i = 1, sizeofBuffer\n pingSendBuf(i) = globalIDarray(myThreadID)\n END DO\n!$OMP END DO NOWAIT\n!Implicit barrier at end of DO not needed for multiple\n\n !Each thread under Ping process sends dataSize items\n !to pongNode process in crossComm.\n !myThreadID is used as tag to ensure data goes to \n !correct place in buffer.\n CALL MPI_Send(pingSendBuf(lBound:uBound), dataSize,&\n MPI_INTEGER, pongNode, myThreadID, crossComm, ierr)\n\n !Thread then waits for a message from pongNode.\n CALL MPI_Recv(pongRecvBuf(lBound:uBound), dataSize,&\n MPI_INTEGER, pongNode, myThreadID, crossComm, &\n status, ierr)\n\n !Each thread reads its part of the received buffer,\n!$OMP DO SCHEDULE(STATIC,dataSize)\n DO i = 1, sizeofBuffer\n finalRecvBuf(i) = pongRecvBuf(i)\n END DO\n!$OMP END DO NOWAIT\n\n ELSEIF (crossCommRank == pongNode) THEN\n\n !Calculate lower and upper bound of data array\n lBound = ((myThreadID - 1) * dataSize) + 1\n uBound = (myThreadID * dataSize)\n\n !Each thread under pongRank receives a message\n !from the ping process\n CALL MPI_Recv(pingRecvBuf(lBound:uBound), dataSize, &\n MPI_INTEGER, pingNode, myThreadID, crossComm, &\n status, ierr)\n\n !Each thread now copies its part of the received\n !buffer to pongSendBuf\n!$OMP DO SCHEDULE(STATIC,dataSize)\n DO i = 1, sizeofBuffer\n pongSendBuf(i) = pingRecvBuf(i)\n END DO\n!$OMP END DO NOWAIT\n\n !Each thread now sends pongSendBuf to ping process.\n CALL MPI_Send(pongSendBuf(lBound:uBound), dataSize,&\n MPI_INTEGER, pingNode, myThreadID, &\n crossComm, ierr)\n\n END IF\n END DO !End repetitions loop\n!$OMP END PARALLEL\n\n END SUBROUTINE multipleMultiPingpong
  