| /*========================================================================= |
| |
| Copyright (c) 2007, Los Alamos National Security, LLC |
| |
| All rights reserved. |
| |
| Copyright 2007. Los Alamos National Security, LLC. |
| This software was produced under U.S. Government contract DE-AC52-06NA25396 |
| for Los Alamos National Laboratory (LANL), which is operated by |
| Los Alamos National Security, LLC for the U.S. Department of Energy. |
| The U.S. Government has rights to use, reproduce, and distribute this software. |
| NEITHER THE GOVERNMENT NOR LOS ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, |
| EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. |
| If software is modified to produce derivative works, such modified software |
| should be clearly marked, so as not to confuse it with the version available |
| from LANL. |
| |
| Additionally, redistribution and use in source and binary forms, with or |
| without modification, are permitted provided that the following conditions |
| are met: |
| - Redistributions of source code must retain the above copyright notice, |
| this list of conditions and the following disclaimer. |
| - Redistributions in binary form must reproduce the above copyright notice, |
| this list of conditions and the following disclaimer in the documentation |
| and/or other materials provided with the distribution. |
| - Neither the name of Los Alamos National Security, LLC, Los Alamos National |
| Laboratory, LANL, the U.S. Government, nor the names of its contributors |
| may be used to endorse or promote products derived from this software |
| without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, |
| THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR |
| CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; |
| OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, |
| WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR |
| OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
| ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| =========================================================================*/ |
| |
| #include <iostream> |
| |
| #include "Partition.h" |
| #include "dims.h" |
| |
| ///////////////////////////////////////////////////////////////////////// |
| // |
| // Static class to control MPI and the partitioning of processors in |
| // a Cartesian grid across the problem space. |
| // |
| ///////////////////////////////////////////////////////////////////////// |
| |
| #ifndef USE_SERIAL_COSMO |
| MPI_Comm Partition::cartComm; |
| #endif |
| |
| int Partition::numProc = 0; |
| int Partition::myProc = -1; |
| int Partition::decompSize[DIMENSION]; |
| int Partition::myPosition[DIMENSION]; |
| int Partition::neighbor[NUM_OF_NEIGHBORS]; |
| int Partition::initialized = 0; |
| |
| Partition::Partition() |
| { |
| } |
| |
| Partition::~Partition() |
| { |
| } |
| |
| ///////////////////////////////////////////////////////////////////////// |
| // |
| // Initialize MPI, allocate the processors across a Cartesian grid of |
| // DIMENSION size and record this processors position, id and neighbor ids |
| // |
| ///////////////////////////////////////////////////////////////////////// |
| |
| //void Partition::initialize(int& argc, char** argv) |
| void Partition::initialize |
| #ifndef USE_SERIAL_COSMO |
| (MPI_Comm comm) |
| #else |
| () |
| #endif |
| { |
| if(!initialized) |
| { |
| #ifndef USE_SERIAL_COSMO |
| fprintf(stdout, "Using MPI Initialize\n"); // TEMP: DEBUG ONLY |
| #ifdef USE_VTK_COSMO |
| // this is for when it is compiled against MPI but single processor |
| // on ParaView (client only, it won't MPI_Init itself) |
| int temp; |
| MPI_Initialized(&temp); |
| if(!temp) |
| { |
| temp = 0; |
| MPI_Init(&temp, 0); |
| } |
| #endif |
| |
| // Start up MPI |
| //MPI_Init(&argc, &argv); |
| MPI_Comm_rank(comm, &myProc); |
| MPI_Comm_size(comm, &numProc); |
| #endif |
| |
| for (int dim = 0; dim < DIMENSION; dim++) |
| decompSize[dim] = 0; |
| |
| #ifdef USE_SERIAL_COSMO |
| myProc = 0; |
| numProc = 1; |
| |
| fprintf(stdout, "Using serial initalize\n"); // ** TEMP: DEBUG ONLY |
| for(int dim = 0; dim < DIMENSION; dim = dim + 1) |
| { |
| decompSize[dim] = 1; |
| myPosition[dim] = 0; |
| } |
| #else |
| int periodic[] = {1, 1, 1}; |
| int reorder = 0; |
| |
| // Compute the number of processors in each dimension |
| //MPI_Dims_create(numProc, DIMENSION, decompSize); |
| MY_Dims_create_3D(numProc, DIMENSION, decompSize); |
| |
| // Create the Cartesion communicator |
| MPI_Cart_create(comm, |
| DIMENSION, decompSize, periodic, reorder, &cartComm); |
| |
| // Reset my rank if it changed |
| MPI_Comm_rank(cartComm, &myProc); |
| |
| // Get this processor's position in the Cartesian topology |
| MPI_Cart_coords(cartComm, myProc, DIMENSION, myPosition); |
| #endif |
| |
| // Set all my neighbor processor ids for communication |
| setNeighbors(); |
| |
| #ifndef USE_VTK_COSMO |
| if (myProc == 0) |
| cout << "Partition 3D: [" << decompSize[0] << ":" |
| << decompSize[1] << ":" << decompSize[2] << "]" << endl; |
| #endif |
| |
| initialized = 1; |
| } |
| } |
| |
| ///////////////////////////////////////////////////////////////////////// |
| // |
| // Return the decomposition size of this problem |
| // |
| ///////////////////////////////////////////////////////////////////////// |
| |
| void Partition::getDecompSize(int size[]) |
| { |
| for (int dim = 0; dim < DIMENSION; dim++) |
| size[dim] = decompSize[dim]; |
| } |
| |
| ///////////////////////////////////////////////////////////////////////// |
| // |
| // Return the position of this processor within the Cartesian grid |
| // |
| ///////////////////////////////////////////////////////////////////////// |
| |
| void Partition::getMyPosition(int pos[]) |
| { |
| for (int dim = 0; dim < DIMENSION; dim++) |
| pos[dim] = myPosition[dim]; |
| } |
| |
| ///////////////////////////////////////////////////////////////////////// |
| // |
| // Return the ranks of the neighbors of this processor using the |
| // description in Definition.h |
| // |
| ///////////////////////////////////////////////////////////////////////// |
| |
| void Partition::getNeighbors(int neigh[]) |
| { |
| for (int n = 0; n < NUM_OF_NEIGHBORS; n++) |
| neigh[n] = neighbor[n]; |
| } |
| |
| ///////////////////////////////////////////////////////////////////////// |
| // |
| // Get the id of a particular processor given its position in the topology |
| // |
| ///////////////////////////////////////////////////////////////////////// |
| |
| int Partition::getNeighbor |
| #ifdef USE_SERIAL_COSMO |
| (int , int , int ) |
| #else |
| (int xpos, int ypos, int zpos) |
| #endif |
| { |
| #ifdef USE_SERIAL_COSMO |
| return 0; |
| #else |
| static int pos[DIMENSION]; |
| pos[0] = xpos; |
| pos[1] = ypos; |
| pos[2] = zpos; |
| |
| int neighborProc; |
| MPI_Cart_rank(cartComm, pos, &neighborProc); |
| return neighborProc; |
| #endif |
| } |
| |
| ///////////////////////////////////////////////////////////////////////// |
| // |
| // Every processor will have 26 neighbors because the cosmology structure |
| // is a 3D torus. Each will have 6 face neighbors, 12 edge neighbors and |
| // 8 corner neighbors. |
| // |
| ///////////////////////////////////////////////////////////////////////// |
| |
| void Partition::setNeighbors() |
| { |
| // Where is this processor in the decomposition |
| int xpos = myPosition[0]; |
| int ypos = myPosition[1]; |
| int zpos = myPosition[2]; |
| |
| // Face neighbors |
| neighbor[X0] = Partition::getNeighbor(xpos-1, ypos, zpos); |
| neighbor[X1] = Partition::getNeighbor(xpos+1, ypos, zpos); |
| neighbor[Y0] = Partition::getNeighbor(xpos, ypos-1, zpos); |
| neighbor[Y1] = Partition::getNeighbor(xpos, ypos+1, zpos); |
| neighbor[Z0] = Partition::getNeighbor(xpos, ypos, zpos-1); |
| neighbor[Z1] = Partition::getNeighbor(xpos, ypos, zpos+1); |
| |
| // Edge neighbors |
| neighbor[X0_Y0] = Partition::getNeighbor(xpos-1, ypos-1, zpos); |
| neighbor[X0_Y1] = Partition::getNeighbor(xpos-1, ypos+1, zpos); |
| neighbor[X1_Y0] = Partition::getNeighbor(xpos+1, ypos-1, zpos); |
| neighbor[X1_Y1] = Partition::getNeighbor(xpos+1, ypos+1, zpos); |
| |
| neighbor[Y0_Z0] = Partition::getNeighbor(xpos, ypos-1, zpos-1); |
| neighbor[Y0_Z1] = Partition::getNeighbor(xpos, ypos-1, zpos+1); |
| neighbor[Y1_Z0] = Partition::getNeighbor(xpos, ypos+1, zpos-1); |
| neighbor[Y1_Z1] = Partition::getNeighbor(xpos, ypos+1, zpos+1); |
| |
| neighbor[Z0_X0] = Partition::getNeighbor(xpos-1, ypos, zpos-1); |
| neighbor[Z0_X1] = Partition::getNeighbor(xpos+1, ypos, zpos-1); |
| neighbor[Z1_X0] = Partition::getNeighbor(xpos-1, ypos, zpos+1); |
| neighbor[Z1_X1] = Partition::getNeighbor(xpos+1, ypos, zpos+1); |
| |
| // Corner neighbors |
| neighbor[X0_Y0_Z0] = Partition::getNeighbor(xpos-1, ypos-1, zpos-1); |
| neighbor[X1_Y0_Z0] = Partition::getNeighbor(xpos+1, ypos-1, zpos-1); |
| neighbor[X0_Y1_Z0] = Partition::getNeighbor(xpos-1, ypos+1, zpos-1); |
| neighbor[X1_Y1_Z0] = Partition::getNeighbor(xpos+1, ypos+1, zpos-1); |
| neighbor[X0_Y0_Z1] = Partition::getNeighbor(xpos-1, ypos-1, zpos+1); |
| neighbor[X1_Y0_Z1] = Partition::getNeighbor(xpos+1, ypos-1, zpos+1); |
| neighbor[X0_Y1_Z1] = Partition::getNeighbor(xpos-1, ypos+1, zpos+1); |
| neighbor[X1_Y1_Z1] = Partition::getNeighbor(xpos+1, ypos+1, zpos+1); |
| } |
| |
| ///////////////////////////////////////////////////////////////////////// |
| // |
| // Shut down MPI |
| // |
| ///////////////////////////////////////////////////////////////////////// |
| |
| void Partition::finalize() |
| { |
| numProc = 0; |
| myProc = -1; |
| |
| //MPI_Finalize(); |
| } |