Nature Language: multipleHaloexchange:Each process exchanges a message with its left and right neighbour. All threads take part in the inter-porcess communication.
  
CPP: int multipleHaloexchange(int totalReps, int dataSize){ \n    int repIter, i; \n    int lBound; \n    \n    #pragma omp parallel default(none) \\ \n    private(i,requestArray,statusArray,lBound,repIter) \\ \n    shared(dataSize,sizeofBuffer,leftSendBuf,rightSendBuf) \\ \n    shared(rightRecvBuf,leftRecvBuf,finalLeftBuf,finalRightBuf) \\ \n    shared(leftNeighbour,rightNeighbour,globalIDarray,commCart,totalReps) \n    { \n        for (repIter=0; repIter<totalReps; repIter++){ \n            lBound = (myThreadID * dataSize); \n            \n            #pragma omp for nowait schedule(static,dataSize) \n            for (i=0; i<sizeofBuffer; i++){ \n                leftSendBuf[i] = globalIDarray[myThreadID]; \n                rightSendBuf[i] = globalIDarray[myThreadID]; \n            } \n            \n            MPI_Isend(&leftSendBuf[lBound], dataSize, MPI_INT, leftNeighbour, \\\n            myThreadID, commCart, &requestArray[0]); \n            \n            MPI_Isend(&rightSendBuf[lBound], dataSize, MPI_INT, rightNeighbour, \\\n            myThreadID, commCart, &requestArray[1]); \n            \n            MPI_Irecv(&leftRecvBuf[lBound], dataSize, MPI_INT, leftNeighbour, \\\n            myThreadID, commCart, &requestArray[2]); \n            \n            MPI_Irecv(&rightRecvBuf[lBound], dataSize, MPI_INT, rightNeighbour, \\\n            myThreadID, commCart, &requestArray[3]); \n            \n            MPI_Waitall(4, requestArray, statusArray); \n            \n            #pragma omp for nowait schedule(static,dataSize) \n            for (i=0; i<sizeofBuffer; i++){ \n                finalLeftBuf[i] = leftRecvBuf[i]; \n                finalRightBuf[i] = rightRecvBuf[i]; \n            } \n        } \n    } \n    \n    return 0; \n}",
  
Fortran: SUBROUTINE multipleHaloexchange(totalReps, dataSize)\ninteger, intent(in) :: totalReps, dataSize\ninteger :: repIter, i\ninteger :: lBound, uBound\n\n !Open the parallel region\n!$OMP PARALLEL DEFAULT(NONE), &\n!$OMP PRIVATE(i,requestArray,statusArray,status,ierr), &\n!$OMP PRIVATE(lBound,uBound,repIter),&\n!$OMP SHARED(dataSize,sizeofBuffer,leftSendBuf,rightSendBuf), &\n!$OMP SHARED(rightRecvBuf,leftRecvBuf,finalLeftBuf), &\n!$OMP SHARED(finalRightBuf,leftNeighbour,rightNeighbour), &\n!$OMP SHARED(globalIDarray,comm,totalReps)\n\nDO repIter = 1, totalReps !repetition loop \n\n !Calculate lower and upper bound of each threads\n !portion of the data arrays.\n lBound = ((myThreadID-1) * dataSize) + 1\n uBound = (myThreadID * dataSize)\n \n !Each thread writes its globalID to rightSendBuf and\n !leftSendBuf.\n!$OMP DO SCHEDULE(STATIC,dataSize)\n DO i = 1, sizeofBuffer\n leftSendBuf(i) = globalIDarray(myThreadID)\n rightSendBuf(i) = globalIDarray(myThreadID)\n END DO\n!$OMP END DO NOWAIT\n!Implicit barrier not needed for multiple version\n\n !Each thread starts send of dataSize items to leftNeighbour\n !and to rightNeighbour\n CALL MPI_Isend(leftSendBuf(lBound:uBound), dataSize, &\n MPI_INTEGER, leftNeighbour, myThreadID, comm, &\n requestArray(1), ierr)\n CALL MPI_Isend(rightSendBuf(lBound:uBound), dataSize, &\n MPI_INTEGER, rightNeighbour, myThreadID, comm, &\n requestArray(2), ierr)\n \n !Each Thread then starts receive of messages from \n !leftNeighbour and rightNeighbour.\n !Receive leftRecvBuf from leftNeighbour...\n CALL MPI_IRecv(leftRecvBuf(lBound:uBound), dataSize, &\n MPI_INTEGER, leftNeighbour, myThreadID, comm, &\n requestArray(3), ierr)\n !Receive rightRecvBuf from rightNeighbour\n CALL MPI_IRecv(rightRecvBuf(lBound:uBound), dataSize, &\n MPI_INTEGER, rightNeighbour, myThreadID, comm, &\n requestArray(4), ierr)\n \n !Finish the sends with an MPI_Waitall on the requests\n CALL MPI_Waitall(4, requestArray, statusArray, ierr)\n\n !Each thread now reads its part of the left and right\n !received buffers.\n!$OMP DO SCHEDULE(STATIC,dataSize)\n DO i = 1, sizeofBuffer\n finalLeftBuf(i) = leftRecvBuf(i)\n finalRightBuf(i) = rightRecvBuf(i)\n END DO\n!$OMP END DO NOWAIT\n\nEND DO !End repetitions loop\n!$OMP END PARALLEL\nEND SUBROUTINE multipleHaloexchange
  