cudaFree blcoks
I have a piece of code LULESH benchmark https://computation.llnl.gov/casc/ShockHydro/LULESH-files/lulesh053111.cu

I am building a performance monitoring framework and hence wrapping some of the CUDA function calls. I insert events before and after each cudaLaunch. When I wrap cudaFree() I see that the cuda runtime never returns from it. The call stack below show how the code is blocked inside cudaFree. Any insights will be useful.


    #0  0x00000037f520a669 in pthread_mutex_unlock () from /lib64/libpthread.so.0
    #1  0x00007fd003f6582e in ?? ()	from /usr/lib64/libcuda.so.1
    #2  0x00007fd003f65a06 in ?? ()	from /usr/lib64/libcuda.so.1
    #3  0x00007fd003f34a29 in ?? ()	from /usr/lib64/libcuda.so.1
    #4  0x00007fd003f2a29a in ?? ()	from /usr/lib64/libcuda.so.1
    #5  0x00007fd003f22b23 in ?? ()	from /usr/lib64/libcuda.so.1
    #6  0x00007fd0039bac1e in ?? ()	from /usr/local/cuda/lib64/libcudart.so.4
    #7  0x00007fd0039e5a02 in cudaFree () from /usr/local/cuda/lib64/libcudart.so.4
    #8  0x00007fd004b18b20 in cudaFree (devPtr=0x2034e0000)	at ../../../../src/tool/hpcrun/sample-sources/itimer.c:1734
    #9  0x00000000004038aa in IntegrateStressForElems_gpu (numElem=91125, sigxx=0x2030e0000, sigyy=0x2031e0000, sigzz=0x2032e0000, determ=0x2033e0000, badvol=@0x7fff3b91c0bc) at lulesh.cu:1478
    #10 0x0000000000403e17 in IntegrateStressForElems (numElem=91125, sigxx=0x2030e0000, sigyy=0x2031e0000,	sigzz=0x2032e0000, determ=0x2033e0000, badvol=@0x7fff3b91c0bc, useCPU=0) at lulesh.cu:1552
    #11 0x0000000000408fe2 in CalcVolumeForceForElems_gpu () at lulesh.cu:2658
    #12 0x0000000000409283 in CalcForceForNodes_gpu	() at lulesh.cu:2716
    #13 0x000000000040931f in CalcForceForNodes (useCPU=0) at lulesh.cu:2747
    #14 0x000000000040a199 in LagrangeNodal	(useCPU=0) at lulesh.cu:2993
    #15 0x0000000000412302 in LagrangeLeapFrog (useCPU=0) at lulesh.cu:5423
    #16 0x00000000004130b2 in main (argc=1,	argv=0x7fff3b91c4d8) at lulesh.cu:5662
cudaFree blcoks
===

I have a piece of code LULESH benchmark https://computation.llnl.gov/casc/ShockHydro/LULESH-files/lulesh053111.cu

I am building a performance monitoring framework and hence wrapping some of the CUDA function calls. I insert events before and after each cudaLaunch. When I wrap cudaFree() I see that the cuda runtime never returns from it. The call stack below show how the code is blocked inside cudaFree. Any insights will be useful.


    #0  0x00000037f520a669 in pthread_mutex_unlock () from /lib64/libpthread.so.0
    #1  0x00007fd003f6582e in ?? ()	from /usr/lib64/libcuda.so.1
    #2  0x00007fd003f65a06 in ?? ()	from /usr/lib64/libcuda.so.1
    #3  0x00007fd003f34a29 in ?? ()	from /usr/lib64/libcuda.so.1
    #4  0x00007fd003f2a29a in ?? ()	from /usr/lib64/libcuda.so.1
    #5  0x00007fd003f22b23 in ?? ()	from /usr/lib64/libcuda.so.1
    #6  0x00007fd0039bac1e in ?? ()	from /usr/local/cuda/lib64/libcudart.so.4
    #7  0x00007fd0039e5a02 in cudaFree () from /usr/local/cuda/lib64/libcudart.so.4
    #8  0x00007fd004b18b20 in cudaFree (devPtr=0x2034e0000)	at ../../../../src/tool/hpcrun/sample-sources/itimer.c:1734
    #9  0x00000000004038aa in IntegrateStressForElems_gpu (numElem=91125, sigxx=0x2030e0000, sigyy=0x2031e0000, sigzz=0x2032e0000, determ=0x2033e0000, badvol=@0x7fff3b91c0bc) at lulesh.cu:1478
    #10 0x0000000000403e17 in IntegrateStressForElems (numElem=91125, sigxx=0x2030e0000, sigyy=0x2031e0000,	sigzz=0x2032e0000, determ=0x2033e0000, badvol=@0x7fff3b91c0bc, useCPU=0) at lulesh.cu:1552
    #11 0x0000000000408fe2 in CalcVolumeForceForElems_gpu () at lulesh.cu:2658
    #12 0x0000000000409283 in CalcForceForNodes_gpu	() at lulesh.cu:2716
    #13 0x000000000040931f in CalcForceForNodes (useCPU=0) at lulesh.cu:2747
    #14 0x000000000040a199 in LagrangeNodal	(useCPU=0) at lulesh.cu:2993
    #15 0x0000000000412302 in LagrangeLeapFrog (useCPU=0) at lulesh.cu:5423
    #16 0x00000000004130b2 in main (argc=1,	argv=0x7fff3b91c4d8) at lulesh.cu:5662