/*************************************************************************/
/*                                                                       */
/*  Copyright (c) 1994 Stanford University                               */
/*                                                                       */
/*  All rights reserved.                                                 */
/*                                                                       */
/*  Permission is given to use, copy, and modify this software for any   */
/*  non-commercial purpose as long as this copyright notice is not       */
/*  removed.  All other uses, including redistribution in whole or in    */
/*  part, are forbidden without prior written permission.                */
/*                                                                       */
/*  This software is provided with absolutely no warranty and no         */
/*  support.                                                             */
/*                                                                       */
/*************************************************************************/

/*    ****************
      subroutine slave
      ****************  */
     
#include <stdio.h>
#include <math.h>
#include <time.h>
#include "decs.h"

void slave() 

{
   int i;
   int j;
   int nstep;
   int iindex;
   int iday;
   double ysca1;
   double y;
   double factor;
   double sintemp;
   double curlt;
   double ressqr;
   int istart; 
   int iend; 
   int jstart; 
   int jend;
   int ist; 
   int ien; 
   int jst; 
   int jen;
   double fac;
   int dayflag=0;
   int dhourflag=0;
   int endflag=0;
   double ttime;
   double dhour;
   double day;
   int firstrow;
   int lastrow;
   int numrows;
   int firstcol;
   int lastcol;
   int numcols;
   int psiindex;
   double psibipriv;
   int psinum;
   int procid;
   unsigned long t1;

   ressqr = lev_res[numlev-1] * lev_res[numlev-1];

   LOCK(locks->idlock)
     procid = global->id;
     global->id = global->id+1;
   UNLOCK(locks->idlock)

/* POSSIBLE ENHANCEMENT:  Here is where one might pin processes to
   processors to avoid migration. */

/* POSSIBLE ENHANCEMENT:  Here is where one might distribute
   data structures across physically distributed memories in
   a round-robin fashion. */

   firstcol = gp[procid].rel_start_x[numlev-1];
   lastcol = firstcol + gp[procid].rel_num_x[numlev-1] - 1;
   firstrow = gp[procid].rel_start_y[numlev-1];
   lastrow = firstrow + gp[procid].rel_num_y[numlev-1] - 1;
   numcols = gp[procid].rel_num_x[numlev-1];
   numrows = gp[procid].rel_num_y[numlev-1];

   if (procid > nprocs/2) {
      psinum = 2;
   } else {
      psinum = 1;
   }

/* every process gets its own copy of the timing variables to avoid
   contention at shared memory locations.  here, these variables
   are initialized.  */

   ttime = 0.0;
   dhour = 0.0;
   nstep = 0 ;
   day = 0.0;
 
   ysca1 = 0.5*ysca;
   if (procid == MASTER) {
     for(iindex = 0;iindex<=jm-1;iindex++) {
       y = ((double) iindex)*res;
       wrk2->f[iindex] = f0+beta*(y-ysca1);
     }
   }

   if (procid == MASTER) {
     fields2->psium[0][0]=0.0;
   }
   if (procid == nprocs-xprocs) {
     fields2->psium[im-1][0]=0.0;
   }
   if (procid == xprocs-1) {
     fields2->psium[0][jm-1]=0.0;
   }
   if (procid == nprocs-1) {
     fields2->psium[im-1][jm-1]=0.0;
   }
   if (firstrow == 1) {
     for(j=firstcol;j<=lastcol;j++) {
       fields2->psium[0][j] = 0.0;
     }
   }
   if ((firstrow+numrows) == im-1) {
     for(j=firstcol;j<=lastcol;j++) {
       fields2->psium[im-1][j] = 0.0;
     }
   }
   if (firstcol == 1) {
     for(j=firstrow;j<=lastrow;j++) {
       fields2->psium[j][0] = 0.0;
     }
   }
   if ((firstcol+numcols) == jm-1) {
     for(j=firstrow;j<=lastrow;j++) {
       fields2->psium[j][jm-1] = 0.0;
     }
   }
   
   for(i=firstrow;i<=lastrow;i++) {
     for(iindex=firstcol;iindex<=lastcol;iindex++) {
       fields2->psium[i][iindex] = 0.0;
     }
   }
   if (procid == MASTER) {
     fields2->psilm[0][0]=0.0;
   }
   if (procid == nprocs-xprocs) {
     fields2->psilm[im-1][0]=0.0;
   }
   if (procid == xprocs-1) {
     fields2->psilm[0][jm-1]=0.0;
   }
   if (procid == nprocs-1) {
     fields2->psilm[im-1][jm-1]=0.0;
   }
   if (firstrow == 1) {
     for(j=firstcol;j<=lastcol;j++) {
       fields2->psilm[0][j] = 0.0;
     }
   }
   if ((firstrow+numrows) == im-1) {
     for(j=firstcol;j<=lastcol;j++) {
       fields2->psilm[im-1][j] = 0.0;
     }
   }
   if (firstcol == 1) {
     for(j=firstrow;j<=lastrow;j++) {
       fields2->psilm[j][0] = 0.0;
     }
   }
   if ((firstcol+numcols) == jm-1) {
     for(j=firstrow;j<=lastrow;j++) {
       fields2->psilm[j][jm-1] = 0.0;
     }
   }
   for(i=firstrow;i<=lastrow;i++) {
     for(iindex=firstcol;iindex<=lastcol;iindex++) {
       fields2->psilm[i][iindex] = 0.0;
     }
   }

   if (procid == MASTER) {
     wrk1->psib[0][0]=1.0;
   }
   if (procid == xprocs-1) {
     wrk1->psib[0][jm-1]=1.0;
   }
   if (procid == nprocs-xprocs) {
     wrk1->psib[im-1][0]=1.0;
   }
   if (procid == nprocs-1) {
     wrk1->psib[im-1][jm-1]=1.0;
   }
   if (firstrow == 1) {
     for(j=firstcol;j<=lastcol;j++) {
       wrk1->psib[0][j] = 1.0;
     }
   }
   if ((firstrow+numrows) == im-1) {
     for(j=firstcol;j<=lastcol;j++) {
       wrk1->psib[im-1][j] = 1.0;
     }
   }
   if (firstcol == 1) {
     for(j=firstrow;j<=lastrow;j++) {
       wrk1->psib[j][0] = 1.0;
     }
   }
   if ((firstcol+numcols) == jm-1) {
     for(j=firstrow;j<=lastrow;j++) {
       wrk1->psib[j][jm-1] = 1.0;
     }
   }
   for(i=firstrow;i<=lastrow;i++) {
     for(iindex=firstcol;iindex<=lastcol;iindex++) {
       wrk1->psib[i][iindex] = 0.0;
     }
   }

/* wait until all processes have completed the above initialization  */

BARRIER(bars->sl_prini,nprocs)

   istart = gp[procid].rel_start_y[numlev-1];
   iend = istart + gp[procid].rel_num_y[numlev-1] - 1;
   jstart = gp[procid].rel_start_x[numlev-1];
   jend = jstart + gp[procid].rel_num_x[numlev-1] - 1;
   ist = istart;
   ien = iend;
   jst = jstart;
   jen = jend;
   if (istart == 1) {
     istart = 0;
   }
   if (jstart == 1) {
     jstart = 0;
   }
   if (iend == im-2) {
     iend = im-1;
   }
   if (jend == jm-2) {
     jend = jm-1;
   }
   for(i=istart;i<=iend;i++) {
     for(j=jstart;j<=jend;j++) {
       multi->rhs_multi[numlev-1][i][j] = wrk1->psib[i][j] * ressqr;
     }
   }
   if (istart == 0) {
     for(j=jstart;j<=jend;j++) {
       multi->q_multi[numlev-1][0][j] = wrk1->psib[0][j];
     }
   }
   if (iend == im-1) {
     for(j=jstart;j<=jend;j++) {
       multi->q_multi[numlev-1][im-1][j] = wrk1->psib[im-1][j];
     }
   }
   if (jstart == 0) {
     for(i=istart;i<=iend;i++) {
       multi->q_multi[numlev-1][i][0] = wrk1->psib[i][0];
     }
   }
   if (jend == jm-1) {
     for(i=istart;i<=iend;i++) {
       multi->q_multi[numlev-1][i][jm-1] = wrk1->psib[i][jm-1];
     }
   }
   
   fac = 1.0 / (4.0 - ressqr*eig2);
   for(i=ist;i<=ien;i++) {
     for(j=jst;j<=jen;j++) {
       multi->q_multi[numlev-1][i][j] = fac * (wrk1->psib[i+1][j] + 
           wrk1->psib[i-1][j] + wrk1->psib[i][j+1] + wrk1->psib[i][j-1] - 
           ressqr*wrk1->psib[i][j]);
     }
   }
   BARRIER(bars->sl_prini,nprocs)
   
   multig(procid);

   for(i=istart;i<=iend;i++) {
     for(j=jstart;j<=jend;j++) {
       wrk1->psib[i][j] = multi->q_multi[numlev-1][i][j];
     }
   }

   BARRIER(bars->sl_psini,nprocs)

/* update the local running sum psibipriv by summing all the resulting
   values in that process's share of the psib matrix   */

   psibipriv=0.0;
   if (procid == MASTER) {
     psibipriv = psibipriv + 0.25*(wrk1->psib[0][0]);
   }
   if (procid == xprocs-1){
     psibipriv = psibipriv + 0.25*(wrk1->psib[0][jm-1]);
   }
   if (procid == nprocs - xprocs) {
     psibipriv=psibipriv+0.25*(wrk1->psib[im-1][0]);
   }
   if (procid == nprocs-1) {
     psibipriv=psibipriv+0.25*(wrk1->psib[im-1][jm-1]);
   }
   if (firstrow == 1) {
     for(j=firstcol;j<=lastcol;j++) {
       psibipriv = psibipriv + 0.5*wrk1->psib[0][j];
     }
   }
   if ((firstrow+numrows) == im-1) {
     for(j=firstcol;j<=lastcol;j++) {
       psibipriv = psibipriv + 0.5*wrk1->psib[im-1][j];
     }
   }
   if (firstcol == 1) {
     for(j=firstrow;j<=lastrow;j++) {
       psibipriv = psibipriv + 0.5*wrk1->psib[j][0];
     }
   }
   if ((firstcol+numcols) == jm-1) {
     for(j=firstrow;j<=lastrow;j++) {
       psibipriv = psibipriv + 0.5*wrk1->psib[j][jm-1];
     }
   }
     for(iindex=firstcol;iindex<=lastcol;iindex++) {
   for(i=firstrow;i<=lastrow;i++) {
       psibipriv = psibipriv + wrk1->psib[i][iindex];
     }
   } 

/* update the shared variable psibi by summing all the psibiprivs
   of the individual processes into it.  note that this combined
   private and shared sum method avoids accessing the shared 
   variable psibi once for every element of the matrix.  */

   LOCK(locks->psibilock)
   global->psibi = global->psibi + psibipriv;
   UNLOCK(locks->psibilock)

   for(psiindex=0;psiindex<=1;psiindex++) {
     if (procid == MASTER) {
       fields->psim[psiindex][0][0] = 0.0;
     }
     if (procid == nprocs-xprocs) {
       fields->psim[psiindex][im-1][0] = 0.0;
     }
     if (procid == xprocs-1) {
       fields->psim[psiindex][0][jm-1] = 0.0;
     }
     if (procid == nprocs-1) {
       fields->psim[psiindex][im-1][jm-1] = 0.0;
     }
     if (firstrow == 1) {
       for(j=firstcol;j<=lastcol;j++) {
         fields->psim[psiindex][0][j] = 0.0;
       }
     }
     if ((firstrow+numrows) == im-1) {
       for(j=firstcol;j<=lastcol;j++) {
         fields->psim[psiindex][im-1][j] = 0.0;
       }
     }
     if (firstcol == 1) {
       for(j=firstrow;j<=lastrow;j++) {
         fields->psim[psiindex][j][0] = 0.0;
       }
     }
     if ((firstcol+numcols) == jm-1) {
       for(j=firstrow;j<=lastrow;j++) {
         fields->psim[psiindex][j][jm-1] = 0.0;
       }
     }
     for(i=firstrow;i<=lastrow;i++) {
       for(iindex=firstcol;iindex<=lastcol;iindex++) {
           fields->psim[psiindex][i][iindex] = 0.0;
       }
     }
   }
 
/* initialize psi matrices the same way  */

   for(psiindex=0;psiindex<=1;psiindex++) {
     if (procid == MASTER) {
       fields->psi[psiindex][0][0] = 0.0;
     }
     if (procid == xprocs-1) {
       fields->psi[psiindex][0][jm-1] = 0.0;
     }
     if (procid == nprocs-xprocs) {
       fields->psi[psiindex][im-1][0] = 0.0;
     }
     if (procid == nprocs-1) {
       fields->psi[psiindex][im-1][jm-1] = 0.0;
     }
     if (firstrow == 1) {
       for(j=firstcol;j<=lastcol;j++) {
         fields->psi[psiindex][0][j] = 0.0;
       }
     }
     if ((firstrow+numrows) == im-1) {
       for(j=firstcol;j<=lastcol;j++) {
         fields->psi[psiindex][im-1][j] = 0.0;
       }
     }
     if (firstcol == 1) {
       for(j=firstrow;j<=lastrow;j++) {
         fields->psi[psiindex][j][0] = 0.0;
       }
     }
     if ((firstcol+numcols) == jm-1) {
       for(j=firstrow;j<=lastrow;j++) {
         fields->psi[psiindex][j][jm-1] = 0.0;
       }
     }
     for(i=firstrow;i<=lastrow;i++) {
       for(iindex=firstcol;iindex<=lastcol;iindex++) {
         fields->psi[psiindex][i][iindex] = 0.0;
       }
     }
   }

/* compute input curl of wind stress */

   ysca1 = .5*ysca;
   factor= -t0*pi/ysca1;
   if (procid == MASTER) {
     frcng->tauz[0][0] = 0.0;
   }
   if (procid == nprocs-xprocs) {
     frcng->tauz[im-1][0] = 0.0;
   }
   if (procid == xprocs-1) {
     sintemp = pi*((double) jmm1)*res/ysca1;
     sintemp = sin(sintemp);
     frcng->tauz[0][jm-1] = factor*sintemp;
   }
   if (procid == nprocs-1) {
     sintemp = pi*((double) jmm1)*res/ysca1;
     sintemp = sin(sintemp);
     frcng->tauz[im-1][jm-1] = frcng->tauz[0][jm-1];
   }
   if (firstrow == 1) {
     for(j=firstcol;j<=lastcol;j++) {
       sintemp = pi*((double) j)*res/ysca1;
       sintemp = sin(sintemp);
       curlt = factor*sintemp;
       frcng->tauz[0][j] = curlt;
     }
   }
   if ((firstrow+numrows) == im-1) {
     for(j=firstcol;j<=lastcol;j++) {
       sintemp = pi*((double) j)*res/ysca1;
       sintemp = sin(sintemp);
       curlt = factor*sintemp;
       frcng->tauz[im-1][j] = curlt;
     }
   }
   if (firstcol == 1) {
     for(j=firstrow;j<=lastrow;j++) {
       frcng->tauz[j][0] = 0.0;
     }
   }
   if ((firstcol+numcols) == jm-1) {
     sintemp = pi*((double) jmm1)*res/ysca1;
     sintemp = sin(sintemp);
     curlt = factor*sintemp;
     for(j=firstrow;j<=lastrow;j++) {
       frcng->tauz[j][jm-1] = curlt;
     }
   }
   for(iindex=firstcol;iindex<=lastcol;iindex++) {
     sintemp = pi*((double) iindex)*res/ysca1;
     sintemp = sin(sintemp);
     curlt = factor*sintemp;
     for(i=firstrow;i<=lastrow;i++) {
       frcng->tauz[i][iindex] = curlt;
     }
   }
   
   BARRIER(bars->sl_onetime,nprocs)
   
   
/***************************************************************
 one-time stuff over at this point
 ***************************************************************/

   while (!endflag) {
     while ((!dayflag) || (!dhourflag)) {
       dayflag = 0;
       dhourflag = 0;
       if (nstep == 1) {
         if (procid == MASTER) {
            CLOCK(global->trackstart)
         }
         if ((procid == MASTER) || (do_stats)) {
           CLOCK(t1);
           gp[procid].total_time = t1;
           gp[procid].multi_time = 0;
         }
/* POSSIBLE ENHANCEMENT:  Here is where one might reset the
   statistics that one is measuring about the parallel execution */
       }

       slave2(procid,firstrow,lastrow,numrows,firstcol,lastcol,numcols);

/* update time and step number
   note that these time and step variables are private i.e. every
   process has its own copy and keeps track of its own time  */

       ttime = ttime + dtau;
       nstep = nstep + 1;
       day = ttime/86400.0;
       if (day > ((double) outday0)) {
         dayflag = 1;
         iday = (int) day;
         dhour = dhour+dtau;
         if (dhour >= 86400.0) {
	   dhourflag = 1;
         }
       }
     }
     dhour = 0.0;

/* update values of psium array to psium + psim{1}  */

     if (procid == MASTER) {
       fields2->psium[0][0] = fields2->psium[0][0]+fields->psim[0][0][0];
     }
     if (procid == nprocs-xprocs) {
       fields2->psium[im-1][0] = fields2->psium[im-1][0]+fields->psim[0][im-1][0];
     }
     if (procid == xprocs-1) {
       fields2->psium[0][jm-1] = fields2->psium[0][jm-1]+fields->psim[0][0][jm-1];
     }
     if (procid == nprocs-1) {
       fields2->psium[im-1][jm-1] = fields2->psium[im-1][jm-1]+fields->psim[0][im-1][jm-1];
     }
     if (firstrow == 1) {
       for(j=firstcol;j<=lastcol;j++) {
         fields2->psium[0][j] = fields2->psium[0][j]+fields->psim[0][0][j];
       }
     }
     if ((firstrow+numrows) == im-1) {
       for(j=firstcol;j<=lastcol;j++) {
         fields2->psium[im-1][j] = fields2->psium[im-1][j]+fields->psim[0][im-1][j];
       }
     }
     if (firstcol == 1) {
       for(j=firstrow;j<=lastrow;j++) {
         fields2->psium[j][0] = fields2->psium[j][0]+fields->psim[0][j][0];
       }
     }
     if ((firstcol+numcols) == jm-1) {
       for(j=firstrow;j<=lastrow;j++) {
         fields2->psium[j][jm-1] = fields2->psium[j][jm-1]+fields->psim[0][j][jm-1];
       }
     }
     for(i=firstrow;i<=lastrow;i++) {
       for(iindex=firstcol;iindex<=lastcol;iindex++) {
         fields2->psium[i][iindex] = fields2->psium[i][iindex]+fields->psim[0][i][iindex];
       }
     }
     
/* update values of psilm array to psilm + psim[2]  */
     
     if (procid == MASTER) {
       fields2->psilm[0][0] = fields2->psilm[0][0]+fields->psim[1][0][0];
     }
     if (procid == nprocs-xprocs) {
       fields2->psilm[im-1][0] = fields2->psilm[im-1][0]+fields->psim[1][im-1][0];
     }
     if (procid == xprocs-1) {
       fields2->psilm[0][jm-1] = fields2->psilm[0][jm-1]+fields->psim[1][0][jm-1];
     }
     if (procid == nprocs-1) {
       fields2->psilm[im-1][jm-1] = fields2->psilm[im-1][jm-1]+fields->psim[1][im-1][jm-1];
     }
     if (firstrow == 1) {
       for(j=firstcol;j<=lastcol;j++) {
         fields2->psilm[0][j] = fields2->psilm[0][j]+fields->psim[1][0][j];
       }
     }
     if ((firstrow+numrows) == im-1) {
       for(j=firstcol;j<=lastcol;j++) {
         fields2->psilm[im-1][j] = fields2->psilm[im-1][j]+fields->psim[1][im-1][j];
       }
     }
     if (firstcol == 1) {
       for(j=firstrow;j<=lastrow;j++) {
         fields2->psilm[j][0] = fields2->psilm[j][0]+fields->psim[1][j][0];
       }
     }
     if ((firstcol+numcols) == jm-1) {
       for(j=firstrow;j<=lastrow;j++) {
         fields2->psilm[j][jm-1] = fields2->psilm[j][jm-1]+fields->psim[1][j][jm-1];
       }
     }
     for(i=firstrow;i<=lastrow;i++) {
       for(iindex=firstcol;iindex<=lastcol;iindex++) {
         fields2->psilm[i][iindex] = fields2->psilm[i][iindex]+fields->psim[1][i][iindex];
       }
     }
     if (iday >= (int) outday3) {
       endflag = 1;
     }
  }
  if ((procid == MASTER) || (do_stats)) {
    CLOCK(t1);
    gp[procid].total_time = t1-gp[procid].total_time;
  }
}