// Lab for Multicore topic on shared memory programming and message passing programming
//
// This program is designed to simulate the effect of non-uniform memory access (NUMA).
// Local memory access is simulated by a sharded counter variable between two threads.
// The access to the shared variable is guarded by a spin lock.
//
// Remote memory access is simulated by message paasing between the requesting process
// and the remote process. The requesting process sends a request to the remote process
// and the remote process responds with the value requested.
//
// Threads are implemented using POSIX Threads library and message passing is implemented
// uaing MPI.
//
// Complete the program by inserting codes at commented places. Specifically, prepare codes
// for spin lock checking and releasing at the beginning and the end of each critical 
// section; and prepare MPI routines for the requesting process (the process with rank 0)
// to send a request to each remote process and receive the requested values from each
// remote process.
//
// The syntax of Pthread mutex lock and unlock routines are:
//
// int pthread_mutex_lock(pthread_mutex_t *mutex); 
// int pthread_mutex_unlock(pthread_mutex_t *mutex); 
//
// The syntax of MPI_Send and MPI_Recv routines are:
//
// int MPI_Send(void* buffer /* in */,
// int count /* in */,
// MPI_Datatype datatype /* in */,
// int destination /* in */,
// int tag /* in */,
// MPI_Comm communicator /* in */)
//
// int MPI_Recv(void* buffer /* in */,
// int count /* in */,
// MPI_Datatype datatype /* in */,
// int source /* in */,
// int tag /* in */,
// MPI_Comm communicator /* in */,
// MPI_Status* status /* out */)
//
// To compile the program on Grid.uhd.edu, you can either use the web portal or SSH shell.
// If you use SSH shell, type:
//
// /opt/openmpi/bin/mpicc -lpthread -o TheNameOfYourExecutable YourSource.c
//
// To run your program in SSH shell:
//
// /opt/openmpi/bin/mpirun -np n ./TheNameOfYourExecutable
//
// where n is the number of processes. For this lab, it has to be greater or equal to 2.

#include
#include
#include
#include "mpi.h"

const int TIMES = 1000;

pthread_mutex_t mutex;
int rank, num_proc;

int local_count = 0, distributed_count = 0;
double local_total = 0.0, local_avg, distributed_total = 0.0, distributed_avg;

void *local_count_reader();
void *local_count_preparer();
void *distributed_count_reader();
void *distributed_count_preparer();

int main(int argc, char** argv)
{
pthread_t local_reader, local_preparer, distributed_reader, distributed_preparer;

MPI_Init(&argc, &argv); //initiate MPI
MPI_Comm_rank(MPI_COMM_WORLD, &rank); /* find rank */
MPI_Comm_size(MPI_COMM_WORLD, &num_proc);

if (rank == 0)
{
pthread_create(&local_reader, NULL, local_count_reader, NULL);
pthread_create(&local_preparer, NULL, local_count_preparer, NULL);
pthread_create(&distributed_reader, NULL, distributed_count_reader, NULL);
}
else
pthread_create(&distributed_preparer, NULL, distributed_count_preparer, NULL);

if (rank == 0)
{
pthread_join(local_reader, NULL);
pthread_join(local_preparer, NULL);
pthread_join(distributed_reader, NULL);
}
else
{
pthread_join(distributed_preparer, NULL);
}

if (rank == 0)
{ printf("The average time used in reading the local shared counter is: %f microseconds.n", local_avg);
printf("The average time used in reading the distributed counter is: %f microseconds.n", distributed_avg);
}
MPI_Finalize();

return(0);
}

void *local_count_reader()
{
int i, copy;
double start, end;

for (i = 0; i < TIMES; i++)
{
start = MPI_Wtime();

// Use lock to guard the critical section
// prepare code here

copy = local_count; 

// The end of critical section
// Insert code here

end = MPI_Wtime();
local_total += end - start;
}

local_avg = 1000000.0 * local_total / TIMES;
}

void *local_count_preparer()
{
int i;

for (i = 0; i < TIMES; i++)
{

// Use lock to guard the critical section
// prepare code here

local_count++; 

// The end of critical section
// Insert code here
}
}

void *distributed_count_reader()
{
int i, copy, req = 0, dest;
double start, end;
MPI_Status *status;

for (i = 0; i < TIMES; i++)
{
for (dest = 1; dest < num_proc; dest++)
{
start = MPI_Wtime();

// The master process (p0) send a request to the process with rank "dest"
// requesting the value of the distributed_count
// use MPI_Send to send the request.
// insert code here.

// The master process (p0) receive the requested value from each of the 
// process with rank "dest"
// use MPI_Recv to receive the value.
// insert code here.

end = MPI_Wtime();

distributed_total += end - start;
}
MPI_Barrier(MPI_COMM_WORLD);
}

distributed_avg = 1000000.0 * distributed_total / (TIMES * (num_proc-1));
pthread_exit(NULL);
}

void *distributed_count_preparer()
{
int i, req;
MPI_Status *status;

for (i = 0; i < TIMES; i++)
{
// The calling process (remote process) receives a request from the master process
// use MPI_Recv to receive the request
// insert code here

distributed_count++;

// The calling process (remote process) sends the value of variable "distributed_count"
// to the master process
// use MPI_Send to send the value
// insert code here

MPI_Barrier(MPI_COMM_WORLD);
}

pthread_exit(NULL);
return;
}