FinepointsBench/mpiBaseTest21.cc at master · lcebaman/FinepointsBench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
#include <omp.h>
#include <time.h>
#include <mpi.h>
#include <assert.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <mpiPart.h>
#include <cstdlib>
#include <random>
#include <ctime>

//#define VERIFY 1
double doTest21( int rank, int numIterations, char* sendBuf, char* recvBuf, int numThreads, size_t threadPart, double compTime, double noise )
{
	MPI_Request sendReq,recvReq;
	int other = (rank + 1) % 2;
	double start;
	int TAG = 0xdead;
	int rc;

#ifdef VERIFY
	for ( int i = 0; i < (threadPart * numThreads) / 8; i++ ) {
		((uint64_t*)sendBuf)[i] = i;
	}

	bzero( recvBuf, threadPart * numThreads );
#endif
    if ( rank == 0 ) {
		rc = MPI_Partitioned_Send_create( sendBuf, threadPart * numThreads, MPI_CHAR, numThreads, other, TAG,
                    MPI_COMM_WORLD, &sendReq );
		assert( rc == MPI_SUCCESS );

	} else {
		rc = MPI_Partitioned_Recv_create( recvBuf, threadPart * numThreads, MPI_CHAR, other, TAG,
                   	MPI_COMM_WORLD, &recvReq );
		assert( rc == MPI_SUCCESS );
    }

	start = MPI_Wtime();

	srand(time(NULL));

	long sleep = compTime * 1000000000;
	long sleepPlus = (compTime + ( compTime * noise)) * 1000000000 ;

#pragma omp parallel shared(rank,numIterations,sendBuf,recvBuf,threadPart,sendReq,recvReq,sleep,sleepPlus) num_threads(numThreads)
	{
	    int rc;
	    int tid = omp_get_thread_num();
	    int iteration;

    	struct timespec req,rem;
    	req.tv_sec = 0;
    	if ( numThreads > 1 && tid == numThreads - 1 ) {
        	req.tv_nsec = sleepPlus;
    	} else {
        	req.tv_nsec = sleep;
    	}

		for ( iteration = 0; iteration < numIterations; iteration++ ) {

#if 0
#pragma omp master
			MPI_Barrier( MPI_COMM_WORLD );
#endif

            if ( 0 == rank ) {

#pragma omp master
		      {
				rc = MPI_Start_part(&sendReq);
				assert( rc == MPI_SUCCESS );
			  }

#pragma omp barrier

				rc = clock_nanosleep(CLOCK_REALTIME,0,&req, &rem);
				if ( 0  !=  rc ) {
					printf("rc=%s rem %li\n",strerror(rc),rem.tv_nsec);
				}
				rc = MPI_Partitioned_Add_to_buffer( &sendReq, sendBuf + (threadPart * tid), threadPart, MPI_CHAR );
            	assert( rc  == MPI_SUCCESS );

#pragma omp barrier
			}

          	if ( 1 == rank ) {
#pragma omp master
			  {
				rc = MPI_Wait_part(&recvReq, MPI_STATUS_IGNORE );
				assert( rc == MPI_SUCCESS );
#ifdef VERIFY
				for ( int i = 0; i < (threadPart * numThreads) / 8; i++ ) {
					assert( ((uint64_t*)recvBuf)[i] == i );
				}
				bzero( recvBuf, threadPart * numThreads );
#endif
			  }
			}
		}
	}
	if ( 0 == rank ) {
		MPI_Partitioned_free( &sendReq );
	} else {
		MPI_Partitioned_free( &recvReq );
	}

	double duration = MPI_Wtime() - start;
	if ( numThreads > 1 ) {
		duration -=  sleepPlus / 1000000000.0 * numIterations;
	} else {
		duration -=  sleep / 1000000000.0 * numIterations;
	}
	return duration;
}