#include "moab/ProcConfig.hpp"
#include "moab/TupleList.hpp"
#include "moab/ProgOptions.hpp"
#include <time.h>
#include <iostream>
#include <sstream>

Functions
int	main (int argc, char **argv)
Variables
const char	BRIEF_DESC [] = "Example of gather scatter with tuple lists \n"
std::ostringstream	LONG_DESC

Function Documentation

int main	(	int	argc,
		char **	argv
	)

Definition at line 58 of file CrystalRouterExample.cpp.

{
  MPI_Init(&argc, &argv);

  ProcConfig pc(MPI_COMM_WORLD);
  int size = pc.proc_size();
  int rank = pc.proc_rank();

  // start copy
  LONG_DESC << "This program does a gather scatter with a list of tuples. \n"
          " It tries to see how much communication costs in terms of time and memory. \n"
          << "It starts with creating a list of tuples to be sent from each processor, \n to a list of other processors.\n" <<
          "The number of tuples and how many tasks to communicate to are controlled by input parameters.\n" <<
          "After communication, we verify locally if we received what we expected. \n";
  ProgOptions opts(LONG_DESC.str(), BRIEF_DESC);

  // how many procs communicate to current proc, on average (we will vary that too)
  int num_comms = 2;
  opts.addOpt<int>("num_comms,n",
       "each task will send to about num_comms other tasks some tuples (default 2)", &num_comms);

  int num_tuples = 4;
  opts.addOpt<int>("num_tuples,t",
        "each task will send to some task about num_tuples tuples (default 4)", &num_tuples);

  int reportrank = size+1;
  opts.addOpt<int>("reporting_rank,r",
      "this rank will report the tuples sent and the tuples received; it could be higher than num_procs, then no reporting"
      ,&reportrank);

  opts.parseCommandLine(argc, argv);




  if (rank==reportrank || (reportrank>=size && rank == 0))
  {
    std::cout << " There are " << size << " tasks in example.\n";
    std::cout<< " We will send groups of " << num_tuples << " from each task towards " <<
        num_comms << " other tasks.\n";
  }

  // send some data from proc i to i+n/2, also to i +n/2+1 modulo n, where n is num procs

  gs_data::crystal_data *cd = pc.crystal_router();

  long total_n_tuples = num_comms*num_tuples;

  // vary the number of tasks to send to, and the number of tuples to send
  if (rank<size/2)
    num_comms--;
  else
    num_comms++;

  if (rank<size/3)
    num_tuples*=2;
  else if (rank>size-size/3)
    num_tuples/=2;


  TupleList tl;
  // at most num_tuples* num_comms to send
  // we do a preallocate with this; some tuples on some processors might need more memory, to be able
  // to grow locally; Some tasks might receive more tuples though, and in the process, some might grow more than
  // others. By doing these logP sends/receives, we do not grow local memory too much.
  tl.initialize(1, 1, 0, 1, num_tuples*num_comms);
  tl.enableWriteAccess();
  // form num_tuples*num_comms tuples, send to various ranks
  unsigned int n = tl.get_n();
  for (int i=0; i<num_comms; i++)
  {
    int sendTo = rank+i*size/2+1;// spread out the send to, for a stress-like test
    sendTo = sendTo%size;//
    long intToSend = 1000*rank + 100000*sendTo;
    for (int j=0; j<num_tuples; j++)
    {
      n = tl.get_n();
      tl.vi_wr[n]= sendTo;
      tl.vl_wr[n]= intToSend+j;
      tl.vr_wr[n]= 10000.*rank+j;
      tl.inc_n();
    }
  }

  if (rank==reportrank)
  {
    std::cout << "rank " << rank << "\n";
    tl.print(" before sending");
  }

  clock_t tt = clock();
  // all communication happens here; no mpi calls for the user
  ErrorCode rval = cd->gs_transfer(1,tl,0);

  if (MB_SUCCESS!= rval)
  {
    std::cout << "error in tuple transfer\n";
  }

  double secs=0;
  if (rank==reportrank || (reportrank>=size && rank == 0))
  {
    secs = (clock() - tt) / (double) CLOCKS_PER_SEC;
  }
  if (rank==reportrank)
  {
    std::cout << "rank " << rank << "\n";
    tl.print(" after transfer");
  }
  // check that all tuples received have the form 10000* rank + 100*from
  unsigned int received = tl.get_n();
  for (int i=0; i<(int)received; i++)
  {
    int from = tl.vi_rd[i];
    long valrec = tl.vl_rd[i];
    int remainder = valrec -100000*rank -1000*from;
    if (remainder < 0 || remainder >= num_tuples*4)
      std::cout << " error: tuple " << i << " received at proc rank " << rank << " from proc " << from << " has value " <<
         valrec << " remainder " <<  remainder << "\n";
  }

  if (rank==reportrank || (reportrank>=size && rank == 0))
  {
    std::cout << "communication of about "<<  total_n_tuples << " tuples/per proc took "
        << secs  << " seconds" << std::endl;
        tt = clock();
  }
  MPI_Finalize();

  return 0;
}

Variable Documentation

const char BRIEF_DESC[] = "Example of gather scatter with tuple lists \n"

Definition at line 51 of file CrystalRouterExample.cpp.

std::ostringstream LONG_DESC

Definition at line 53 of file CrystalRouterExample.cpp.

Functions

Variables

Function Documentation

Variable Documentation