Actual source code: matis.c

petsc-3.7.4 2016-10-02
Report Typos and Errors
  2: /*
  3:     Creates a matrix class for using the Neumann-Neumann type preconditioners.
  4:    This stores the matrices in globally unassembled form. Each processor
  5:    assembles only its local Neumann problem and the parallel matrix vector
  6:    product is handled "implicitly".

  8:      We provide:
  9:          MatMult()

 11:     Currently this allows for only one subdomain per processor.

 13: */

 15: #include <../src/mat/impls/is/matis.h>      /*I "petscmat.h" I*/

 17: static PetscErrorCode MatISComputeSF_Private(Mat);

 21: static PetscErrorCode MatISComputeSF_Private(Mat B)
 22: {
 23:   Mat_IS         *matis = (Mat_IS*)(B->data);
 24:   const PetscInt *gidxs;

 28:   MatGetSize(matis->A,&matis->sf_nleaves,NULL);
 29:   MatGetLocalSize(B,&matis->sf_nroots,NULL);
 30:   PetscSFCreate(PetscObjectComm((PetscObject)B),&matis->sf);
 31:   ISLocalToGlobalMappingGetIndices(B->rmap->mapping,&gidxs);
 32:   /* PETSC_OWN_POINTER refers to ilocal which is NULL */
 33:   PetscSFSetGraphLayout(matis->sf,B->rmap,matis->sf_nleaves,NULL,PETSC_OWN_POINTER,gidxs);
 34:   ISLocalToGlobalMappingRestoreIndices(B->rmap->mapping,&gidxs);
 35:   PetscMalloc2(matis->sf_nroots,&matis->sf_rootdata,matis->sf_nleaves,&matis->sf_leafdata);
 36:   return(0);
 37: }

 41: /*@
 42:    MatISSetPreallocation - Preallocates memory for a MATIS parallel matrix.

 44:    Collective on MPI_Comm

 46:    Input Parameters:
 47: +  B - the matrix
 48: .  d_nz  - number of nonzeros per row in DIAGONAL portion of local submatrix
 49:            (same value is used for all local rows)
 50: .  d_nnz - array containing the number of nonzeros in the various rows of the
 51:            DIAGONAL portion of the local submatrix (possibly different for each row)
 52:            or NULL, if d_nz is used to specify the nonzero structure.
 53:            The size of this array is equal to the number of local rows, i.e 'm'.
 54:            For matrices that will be factored, you must leave room for (and set)
 55:            the diagonal entry even if it is zero.
 56: .  o_nz  - number of nonzeros per row in the OFF-DIAGONAL portion of local
 57:            submatrix (same value is used for all local rows).
 58: -  o_nnz - array containing the number of nonzeros in the various rows of the
 59:            OFF-DIAGONAL portion of the local submatrix (possibly different for
 60:            each row) or NULL, if o_nz is used to specify the nonzero
 61:            structure. The size of this array is equal to the number
 62:            of local rows, i.e 'm'.

 64:    If the *_nnz parameter is given then the *_nz parameter is ignored

 66:    Level: intermediate

 68:    Notes: This function has the same interface as the MPIAIJ preallocation routine in order to simplify the transition
 69:           from the asssembled format to the unassembled one. It overestimates the preallocation of MATIS local
 70:           matrices; for exact preallocation, the user should set the preallocation directly on local matrix objects.

 72: .keywords: matrix

 74: .seealso: MatCreate(), MatCreateIS(), MatMPIAIJSetPreallocation(), MatISGetLocalMat()
 75: @*/
 76: PetscErrorCode  MatISSetPreallocation(Mat B,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
 77: {

 83:   PetscTryMethod(B,"MatISSetPreallocation_C",(Mat,PetscInt,const PetscInt[],PetscInt,const PetscInt[]),(B,d_nz,d_nnz,o_nz,o_nnz));
 84:   return(0);
 85: }

 89: PetscErrorCode  MatISSetPreallocation_IS(Mat B,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
 90: {
 91:   Mat_IS         *matis = (Mat_IS*)(B->data);
 92:   PetscInt       bs,i,nlocalcols;

 96:   if (!matis->A) SETERRQ(PetscObjectComm((PetscObject)B),PETSC_ERR_SUP,"You should first call MatSetLocalToGlobalMapping");
 97:   if (!matis->sf) { /* setup SF if not yet created and allocate rootdata and leafdata */
 98:     MatISComputeSF_Private(B);
 99:   }
100:   if (!d_nnz) {
101:     for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] = d_nz;
102:   } else {
103:     for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] = d_nnz[i];
104:   }
105:   if (!o_nnz) {
106:     for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] += o_nz;
107:   } else {
108:     for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] += o_nnz[i];
109:   }
110:   PetscSFBcastBegin(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);
111:   MatGetSize(matis->A,NULL,&nlocalcols);
112:   MatGetBlockSize(matis->A,&bs);
113:   PetscSFBcastEnd(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);
114:   for (i=0;i<matis->sf_nleaves;i++) {
115:     matis->sf_leafdata[i] = PetscMin(matis->sf_leafdata[i],nlocalcols);
116:   }
117:   MatSeqAIJSetPreallocation(matis->A,0,matis->sf_leafdata);
118:   for (i=0;i<matis->sf_nleaves/bs;i++) {
119:     matis->sf_leafdata[i] = matis->sf_leafdata[i*bs]/bs;
120:   }
121:   MatSeqBAIJSetPreallocation(matis->A,bs,0,matis->sf_leafdata);
122:   for (i=0;i<matis->sf_nleaves/bs;i++) {
123:     matis->sf_leafdata[i] = matis->sf_leafdata[i]-i;
124:   }
125:   MatSeqSBAIJSetPreallocation(matis->A,bs,0,matis->sf_leafdata);
126:   return(0);
127: }

131: PETSC_EXTERN PetscErrorCode MatISSetMPIXAIJPreallocation_Private(Mat A, Mat B, PetscBool maxreduce)
132: {
133:   Mat_IS          *matis = (Mat_IS*)(A->data);
134:   PetscInt        *my_dnz,*my_onz,*dnz,*onz,*mat_ranges,*row_ownership;
135:   const PetscInt  *global_indices_r,*global_indices_c;
136:   PetscInt        i,j,bs,rows,cols;
137:   PetscInt        lrows,lcols;
138:   PetscInt        local_rows,local_cols;
139:   PetscMPIInt     nsubdomains;
140:   PetscBool       isdense,issbaij;
141:   PetscErrorCode  ierr;

144:   MPI_Comm_size(PetscObjectComm((PetscObject)A),&nsubdomains);
145:   MatGetSize(A,&rows,&cols);
146:   MatGetBlockSize(A,&bs);
147:   MatGetSize(matis->A,&local_rows,&local_cols);
148:   PetscObjectTypeCompare((PetscObject)matis->A,MATSEQDENSE,&isdense);
149:   PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);
150:   ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_r);
151:   if (A->rmap->mapping != A->cmap->mapping) {
152:     ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_c);
153:   } else {
154:     global_indices_c = global_indices_r;
155:   }

157:   if (issbaij) {
158:     MatGetRowUpperTriangular(matis->A);
159:   }
160:   /*
161:      An SF reduce is needed to sum up properly on shared rows.
162:      Note that generally preallocation is not exact, since it overestimates nonzeros
163:   */
164:   if (!matis->sf) { /* setup SF if not yet created and allocate rootdata and leafdata */
165:     MatISComputeSF_Private(A);
166:   }
167:   MatGetLocalSize(A,&lrows,&lcols);
168:   MatPreallocateInitialize(PetscObjectComm((PetscObject)A),lrows,lcols,dnz,onz);
169:   /* All processes need to compute entire row ownership */
170:   PetscMalloc1(rows,&row_ownership);
171:   MatGetOwnershipRanges(A,(const PetscInt**)&mat_ranges);
172:   for (i=0;i<nsubdomains;i++) {
173:     for (j=mat_ranges[i];j<mat_ranges[i+1];j++) {
174:       row_ownership[j] = i;
175:     }
176:   }

178:   /*
179:      my_dnz and my_onz contains exact contribution to preallocation from each local mat
180:      then, they will be summed up properly. This way, preallocation is always sufficient
181:   */
182:   PetscCalloc2(local_rows,&my_dnz,local_rows,&my_onz);
183:   /* preallocation as a MATAIJ */
184:   if (isdense) { /* special case for dense local matrices */
185:     for (i=0;i<local_rows;i++) {
186:       PetscInt index_row = global_indices_r[i];
187:       for (j=i;j<local_rows;j++) {
188:         PetscInt owner = row_ownership[index_row];
189:         PetscInt index_col = global_indices_c[j];
190:         if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
191:           my_dnz[i] += 1;
192:         } else { /* offdiag block */
193:           my_onz[i] += 1;
194:         }
195:         /* same as before, interchanging rows and cols */
196:         if (i != j) {
197:           owner = row_ownership[index_col];
198:           if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
199:             my_dnz[j] += 1;
200:           } else {
201:             my_onz[j] += 1;
202:           }
203:         }
204:       }
205:     }
206:   } else { /* TODO: this could be optimized using MatGetRowIJ */
207:     for (i=0;i<local_rows;i++) {
208:       const PetscInt *cols;
209:       PetscInt       ncols,index_row = global_indices_r[i];
210:       MatGetRow(matis->A,i,&ncols,&cols,NULL);
211:       for (j=0;j<ncols;j++) {
212:         PetscInt owner = row_ownership[index_row];
213:         PetscInt index_col = global_indices_c[cols[j]];
214:         if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
215:           my_dnz[i] += 1;
216:         } else { /* offdiag block */
217:           my_onz[i] += 1;
218:         }
219:         /* same as before, interchanging rows and cols */
220:         if (issbaij && index_col != index_row) {
221:           owner = row_ownership[index_col];
222:           if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
223:             my_dnz[cols[j]] += 1;
224:           } else {
225:             my_onz[cols[j]] += 1;
226:           }
227:         }
228:       }
229:       MatRestoreRow(matis->A,i,&ncols,&cols,NULL);
230:     }
231:   }
232:   ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_r);
233:   if (global_indices_c != global_indices_r) {
234:     ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_c);
235:   }
236:   PetscFree(row_ownership);

238:   /* Reduce my_dnz and my_onz */
239:   if (maxreduce) {
240:     PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);
241:     PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);
242:     PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);
243:     PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);
244:   } else {
245:     PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);
246:     PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);
247:     PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);
248:     PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);
249:   }
250:   PetscFree2(my_dnz,my_onz);

252:   /* Resize preallocation if overestimated */
253:   for (i=0;i<lrows;i++) {
254:     dnz[i] = PetscMin(dnz[i],lcols);
255:     onz[i] = PetscMin(onz[i],cols-lcols);
256:   }
257:   /* set preallocation */
258:   MatMPIAIJSetPreallocation(B,0,dnz,0,onz);
259:   for (i=0;i<lrows/bs;i++) {
260:     dnz[i] = dnz[i*bs]/bs;
261:     onz[i] = onz[i*bs]/bs;
262:   }
263:   MatMPIBAIJSetPreallocation(B,bs,0,dnz,0,onz);
264:   MatMPISBAIJSetPreallocation(B,bs,0,dnz,0,onz);
265:   MatPreallocateFinalize(dnz,onz);
266:   if (issbaij) {
267:     MatRestoreRowUpperTriangular(matis->A);
268:   }
269:   return(0);
270: }

274: PetscErrorCode MatISGetMPIXAIJ_IS(Mat mat, MatReuse reuse, Mat *M)
275: {
276:   Mat_IS         *matis = (Mat_IS*)(mat->data);
277:   Mat            local_mat;
278:   /* info on mat */
279:   PetscInt       bs,rows,cols,lrows,lcols;
280:   PetscInt       local_rows,local_cols;
281:   PetscBool      isdense,issbaij,isseqaij;
282:   PetscMPIInt    nsubdomains;
283:   /* values insertion */
284:   PetscScalar    *array;
285:   /* work */

289:   /* get info from mat */
290:   MPI_Comm_size(PetscObjectComm((PetscObject)mat),&nsubdomains);
291:   if (nsubdomains == 1) {
292:     if (reuse == MAT_INITIAL_MATRIX) {
293:       MatDuplicate(matis->A,MAT_COPY_VALUES,&(*M));
294:     } else {
295:       MatCopy(matis->A,*M,SAME_NONZERO_PATTERN);
296:     }
297:     return(0);
298:   }
299:   MatGetSize(mat,&rows,&cols);
300:   MatGetBlockSize(mat,&bs);
301:   MatGetLocalSize(mat,&lrows,&lcols);
302:   MatGetSize(matis->A,&local_rows,&local_cols);
303:   PetscObjectTypeCompare((PetscObject)matis->A,MATSEQDENSE,&isdense);
304:   PetscObjectTypeCompare((PetscObject)matis->A,MATSEQAIJ,&isseqaij);
305:   PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);

307:   if (reuse == MAT_INITIAL_MATRIX) {
308:     MatType     new_mat_type;
309:     PetscBool   issbaij_red;

311:     /* determining new matrix type */
312:     MPIU_Allreduce(&issbaij,&issbaij_red,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)mat));
313:     if (issbaij_red) {
314:       new_mat_type = MATSBAIJ;
315:     } else {
316:       if (bs>1) {
317:         new_mat_type = MATBAIJ;
318:       } else {
319:         new_mat_type = MATAIJ;
320:       }
321:     }

323:     MatCreate(PetscObjectComm((PetscObject)mat),M);
324:     MatSetSizes(*M,lrows,lcols,rows,cols);
325:     MatSetBlockSize(*M,bs);
326:     MatSetType(*M,new_mat_type);
327:     MatISSetMPIXAIJPreallocation_Private(mat,*M,PETSC_FALSE);
328:   } else {
329:     PetscInt mbs,mrows,mcols,mlrows,mlcols;
330:     /* some checks */
331:     MatGetBlockSize(*M,&mbs);
332:     MatGetSize(*M,&mrows,&mcols);
333:     MatGetLocalSize(*M,&mlrows,&mlcols);
334:     if (mrows != rows) SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong number of rows (%d != %d)",rows,mrows);
335:     if (mcols != cols) SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong number of cols (%d != %d)",cols,mcols);
336:     if (mlrows != lrows) SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong number of local rows (%d != %d)",lrows,mlrows);
337:     if (mlcols != lcols) SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong number of local cols (%d != %d)",lcols,mlcols);
338:     if (mbs != bs) SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong block size (%d != %d)",bs,mbs);
339:     MatZeroEntries(*M);
340:   }

342:   if (issbaij) {
343:     MatConvert(matis->A,MATSEQBAIJ,MAT_INITIAL_MATRIX,&local_mat);
344:   } else {
345:     PetscObjectReference((PetscObject)matis->A);
346:     local_mat = matis->A;
347:   }

349:   /* Set values */
350:   MatSetLocalToGlobalMapping(*M,mat->rmap->mapping,mat->cmap->mapping);
351:   if (isdense) { /* special case for dense local matrices */
352:     PetscInt i,*dummy_rows,*dummy_cols;

354:     if (local_rows != local_cols) {
355:       PetscMalloc2(local_rows,&dummy_rows,local_cols,&dummy_cols);
356:       for (i=0;i<local_rows;i++) dummy_rows[i] = i;
357:       for (i=0;i<local_cols;i++) dummy_cols[i] = i;
358:     } else {
359:       PetscMalloc1(local_rows,&dummy_rows);
360:       for (i=0;i<local_rows;i++) dummy_rows[i] = i;
361:       dummy_cols = dummy_rows;
362:     }
363:     MatSetOption(*M,MAT_ROW_ORIENTED,PETSC_FALSE);
364:     MatDenseGetArray(local_mat,&array);
365:     MatSetValuesLocal(*M,local_rows,dummy_rows,local_cols,dummy_cols,array,ADD_VALUES);
366:     MatDenseRestoreArray(local_mat,&array);
367:     if (dummy_rows != dummy_cols) {
368:       PetscFree2(dummy_rows,dummy_cols);
369:     } else {
370:       PetscFree(dummy_rows);
371:     }
372:   } else if (isseqaij) {
373:     PetscInt  i,nvtxs,*xadj,*adjncy;
374:     PetscBool done;

376:     MatGetRowIJ(local_mat,0,PETSC_FALSE,PETSC_FALSE,&nvtxs,(const PetscInt**)&xadj,(const PetscInt**)&adjncy,&done);
377:     if (!done) SETERRQ1(PetscObjectComm((PetscObject)local_mat),PETSC_ERR_PLIB,"Error in MatRestoreRowIJ called in %s\n",__FUNCT__);
378:     MatSeqAIJGetArray(local_mat,&array);
379:     for (i=0;i<nvtxs;i++) {
380:       MatSetValuesLocal(*M,1,&i,xadj[i+1]-xadj[i],adjncy+xadj[i],array+xadj[i],ADD_VALUES);
381:     }
382:     MatRestoreRowIJ(local_mat,0,PETSC_FALSE,PETSC_FALSE,&nvtxs,(const PetscInt**)&xadj,(const PetscInt**)&adjncy,&done);
383:     if (!done) SETERRQ1(PetscObjectComm((PetscObject)local_mat),PETSC_ERR_PLIB,"Error in MatRestoreRowIJ called in %s\n",__FUNCT__);
384:     MatSeqAIJRestoreArray(local_mat,&array);
385:   } else { /* very basic values insertion for all other matrix types */
386:     PetscInt i;

388:     for (i=0;i<local_rows;i++) {
389:       PetscInt       j;
390:       const PetscInt *local_indices_cols;

392:       MatGetRow(local_mat,i,&j,&local_indices_cols,(const PetscScalar**)&array);
393:       MatSetValuesLocal(*M,1,&i,j,local_indices_cols,array,ADD_VALUES);
394:       MatRestoreRow(local_mat,i,&j,&local_indices_cols,(const PetscScalar**)&array);
395:     }
396:   }
397:   MatAssemblyBegin(*M,MAT_FINAL_ASSEMBLY);
398:   MatDestroy(&local_mat);
399:   MatAssemblyEnd(*M,MAT_FINAL_ASSEMBLY);
400:   if (isdense) {
401:     MatSetOption(*M,MAT_ROW_ORIENTED,PETSC_TRUE);
402:   }
403:   return(0);
404: }

408: /*@
409:     MatISGetMPIXAIJ - Converts MATIS matrix into a parallel AIJ format

411:   Input Parameter:
412: .  mat - the matrix (should be of type MATIS)
413: .  reuse - either MAT_INITIAL_MATRIX or MAT_REUSE_MATRIX

415:   Output Parameter:
416: .  newmat - the matrix in AIJ format

418:   Level: developer

420:   Notes: mat and *newmat cannot be the same object when MAT_REUSE_MATRIX is requested.

422: .seealso: MATIS
423: @*/
424: PetscErrorCode MatISGetMPIXAIJ(Mat mat, MatReuse reuse, Mat *newmat)
425: {

432:   if (reuse != MAT_INITIAL_MATRIX) {
435:     if (mat == *newmat) SETERRQ(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse the same matrix");
436:   }
437:   PetscUseMethod(mat,"MatISGetMPIXAIJ_C",(Mat,MatReuse,Mat*),(mat,reuse,newmat));
438:   return(0);
439: }

443: PetscErrorCode MatDuplicate_IS(Mat mat,MatDuplicateOption op,Mat *newmat)
444: {
446:   Mat_IS         *matis = (Mat_IS*)(mat->data);
447:   PetscInt       bs,m,n,M,N;
448:   Mat            B,localmat;

451:   MatGetBlockSize(mat,&bs);
452:   MatGetSize(mat,&M,&N);
453:   MatGetLocalSize(mat,&m,&n);
454:   MatCreateIS(PetscObjectComm((PetscObject)mat),bs,m,n,M,N,mat->rmap->mapping,mat->cmap->mapping,&B);
455:   MatDuplicate(matis->A,op,&localmat);
456:   MatISSetLocalMat(B,localmat);
457:   MatDestroy(&localmat);
458:   MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);
459:   MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);
460:   *newmat = B;
461:   return(0);
462: }

466: PetscErrorCode MatIsHermitian_IS(Mat A,PetscReal tol,PetscBool  *flg)
467: {
469:   Mat_IS         *matis = (Mat_IS*)A->data;
470:   PetscBool      local_sym;

473:   MatIsHermitian(matis->A,tol,&local_sym);
474:   MPIU_Allreduce(&local_sym,flg,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)A));
475:   return(0);
476: }

480: PetscErrorCode MatIsSymmetric_IS(Mat A,PetscReal tol,PetscBool  *flg)
481: {
483:   Mat_IS         *matis = (Mat_IS*)A->data;
484:   PetscBool      local_sym;

487:   MatIsSymmetric(matis->A,tol,&local_sym);
488:   MPIU_Allreduce(&local_sym,flg,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)A));
489:   return(0);
490: }

494: PetscErrorCode MatDestroy_IS(Mat A)
495: {
497:   Mat_IS         *b = (Mat_IS*)A->data;

500:   MatDestroy(&b->A);
501:   VecScatterDestroy(&b->cctx);
502:   VecScatterDestroy(&b->rctx);
503:   VecDestroy(&b->x);
504:   VecDestroy(&b->y);
505:   PetscSFDestroy(&b->sf);
506:   PetscFree2(b->sf_rootdata,b->sf_leafdata);
507:   PetscFree(A->data);
508:   PetscObjectChangeTypeName((PetscObject)A,0);
509:   PetscObjectComposeFunction((PetscObject)A,"MatISGetLocalMat_C",NULL);
510:   PetscObjectComposeFunction((PetscObject)A,"MatISSetLocalMat_C",NULL);
511:   PetscObjectComposeFunction((PetscObject)A,"MatISGetMPIXAIJ_C",NULL);
512:   PetscObjectComposeFunction((PetscObject)A,"MatISSetPreallocation_C",NULL);
513:   return(0);
514: }

518: PetscErrorCode MatMult_IS(Mat A,Vec x,Vec y)
519: {
521:   Mat_IS         *is  = (Mat_IS*)A->data;
522:   PetscScalar    zero = 0.0;

525:   /*  scatter the global vector x into the local work vector */
526:   VecScatterBegin(is->cctx,x,is->x,INSERT_VALUES,SCATTER_FORWARD);
527:   VecScatterEnd(is->cctx,x,is->x,INSERT_VALUES,SCATTER_FORWARD);

529:   /* multiply the local matrix */
530:   MatMult(is->A,is->x,is->y);

532:   /* scatter product back into global memory */
533:   VecSet(y,zero);
534:   VecScatterBegin(is->rctx,is->y,y,ADD_VALUES,SCATTER_REVERSE);
535:   VecScatterEnd(is->rctx,is->y,y,ADD_VALUES,SCATTER_REVERSE);
536:   return(0);
537: }

541: PetscErrorCode MatMultAdd_IS(Mat A,Vec v1,Vec v2,Vec v3)
542: {
543:   Vec            temp_vec;

547:   if (v3 != v2) {
548:     MatMult(A,v1,v3);
549:     VecAXPY(v3,1.0,v2);
550:   } else {
551:     VecDuplicate(v2,&temp_vec);
552:     MatMult(A,v1,temp_vec);
553:     VecAXPY(temp_vec,1.0,v2);
554:     VecCopy(temp_vec,v3);
555:     VecDestroy(&temp_vec);
556:   }
557:   return(0);
558: }

562: PetscErrorCode MatMultTranspose_IS(Mat A,Vec y,Vec x)
563: {
564:   Mat_IS         *is = (Mat_IS*)A->data;

568:   /*  scatter the global vector x into the local work vector */
569:   VecScatterBegin(is->rctx,y,is->y,INSERT_VALUES,SCATTER_FORWARD);
570:   VecScatterEnd(is->rctx,y,is->y,INSERT_VALUES,SCATTER_FORWARD);

572:   /* multiply the local matrix */
573:   MatMultTranspose(is->A,is->y,is->x);

575:   /* scatter product back into global vector */
576:   VecSet(x,0);
577:   VecScatterBegin(is->cctx,is->x,x,ADD_VALUES,SCATTER_REVERSE);
578:   VecScatterEnd(is->cctx,is->x,x,ADD_VALUES,SCATTER_REVERSE);
579:   return(0);
580: }

584: PetscErrorCode MatMultTransposeAdd_IS(Mat A,Vec v1,Vec v2,Vec v3)
585: {
586:   Vec            temp_vec;

590:   if (v3 != v2) {
591:     MatMultTranspose(A,v1,v3);
592:     VecAXPY(v3,1.0,v2);
593:   } else {
594:     VecDuplicate(v2,&temp_vec);
595:     MatMultTranspose(A,v1,temp_vec);
596:     VecAXPY(temp_vec,1.0,v2);
597:     VecCopy(temp_vec,v3);
598:     VecDestroy(&temp_vec);
599:   }
600:   return(0);
601: }

605: PetscErrorCode MatView_IS(Mat A,PetscViewer viewer)
606: {
607:   Mat_IS         *a = (Mat_IS*)A->data;
609:   PetscViewer    sviewer;

612:   PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&sviewer);
613:   MatView(a->A,sviewer);
614:   PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&sviewer);
615:   return(0);
616: }

620: PetscErrorCode MatSetLocalToGlobalMapping_IS(Mat A,ISLocalToGlobalMapping rmapping,ISLocalToGlobalMapping cmapping)
621: {
623:   PetscInt       nr,rbs,nc,cbs;
624:   Mat_IS         *is = (Mat_IS*)A->data;
625:   IS             from,to;
626:   Vec            cglobal,rglobal;

631:   /* Destroy any previous data */
632:   VecDestroy(&is->x);
633:   VecDestroy(&is->y);
634:   VecScatterDestroy(&is->rctx);
635:   VecScatterDestroy(&is->cctx);
636:   MatDestroy(&is->A);
637:   PetscSFDestroy(&is->sf);
638:   PetscFree2(is->sf_rootdata,is->sf_leafdata);

640:   /* Setup Layout and set local to global maps */
641:   PetscLayoutSetUp(A->rmap);
642:   PetscLayoutSetUp(A->cmap);
643:   PetscLayoutSetISLocalToGlobalMapping(A->rmap,rmapping);
644:   PetscLayoutSetISLocalToGlobalMapping(A->cmap,cmapping);

646:   /* Create the local matrix A */
647:   ISLocalToGlobalMappingGetSize(rmapping,&nr);
648:   ISLocalToGlobalMappingGetBlockSize(rmapping,&rbs);
649:   ISLocalToGlobalMappingGetSize(cmapping,&nc);
650:   ISLocalToGlobalMappingGetBlockSize(cmapping,&cbs);
651:   MatCreate(PETSC_COMM_SELF,&is->A);
652:   MatSetType(is->A,MATAIJ);
653:   MatSetSizes(is->A,nr,nc,nr,nc);
654:   MatSetBlockSizes(is->A,rbs,cbs);
655:   MatSetOptionsPrefix(is->A,((PetscObject)A)->prefix);
656:   MatAppendOptionsPrefix(is->A,"is_");
657:   MatSetFromOptions(is->A);

659:   /* Create the local work vectors */
660:   MatCreateVecs(is->A,&is->x,&is->y);

662:   /* setup the global to local scatters */
663:   MatCreateVecs(A,&cglobal,&rglobal);
664:   ISCreateStride(PETSC_COMM_SELF,nr,0,1,&to);
665:   ISLocalToGlobalMappingApplyIS(rmapping,to,&from);
666:   VecScatterCreate(rglobal,from,is->y,to,&is->rctx);
667:   if (rmapping != cmapping) {
668:     ISDestroy(&to);
669:     ISDestroy(&from);
670:     ISCreateStride(PETSC_COMM_SELF,nc,0,1,&to);
671:     ISLocalToGlobalMappingApplyIS(cmapping,to,&from);
672:     VecScatterCreate(cglobal,from,is->x,to,&is->cctx);
673:   } else {
674:     PetscObjectReference((PetscObject)is->rctx);
675:     is->cctx = is->rctx;
676:   }
677:   VecDestroy(&rglobal);
678:   VecDestroy(&cglobal);
679:   ISDestroy(&to);
680:   ISDestroy(&from);
681:   return(0);
682: }

686: PetscErrorCode MatSetValues_IS(Mat mat, PetscInt m,const PetscInt *rows, PetscInt n,const PetscInt *cols, const PetscScalar *values, InsertMode addv)
687: {
688:   Mat_IS         *is = (Mat_IS*)mat->data;
689:   PetscInt       rows_l[2048],cols_l[2048];

693: #if defined(PETSC_USE_DEBUG)
694:   if (m > 2048 || n > 2048) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of row/column indices must be <= 2048: they are %D %D",m,n);
695: #endif
696:   ISG2LMapApply(mat->rmap->mapping,m,rows,rows_l);
697:   ISG2LMapApply(mat->cmap->mapping,n,cols,cols_l);
698:   MatSetValues(is->A,m,rows_l,n,cols_l,values,addv);
699:   return(0);
700: }

704: PetscErrorCode MatSetValuesLocal_IS(Mat A,PetscInt m,const PetscInt *rows, PetscInt n,const PetscInt *cols,const PetscScalar *values,InsertMode addv)
705: {
707:   Mat_IS         *is = (Mat_IS*)A->data;

710:   MatSetValues(is->A,m,rows,n,cols,values,addv);
711:   return(0);
712: }

716: PetscErrorCode MatSetValuesBlockedLocal_IS(Mat A,PetscInt m,const PetscInt *rows, PetscInt n,const PetscInt *cols,const PetscScalar *values,InsertMode addv)
717: {
719:   Mat_IS         *is = (Mat_IS*)A->data;

722:   MatSetValuesBlocked(is->A,m,rows,n,cols,values,addv);
723:   return(0);
724: }

728: PetscErrorCode MatZeroRows_IS(Mat A,PetscInt n,const PetscInt rows[],PetscScalar diag,Vec x,Vec b)
729: {
730:   PetscInt       n_l = 0, *rows_l = NULL;

734:   if (x && b) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_SUP,"No support");
735:   if (n) {
736:     PetscMalloc1(n,&rows_l);
737:     ISGlobalToLocalMappingApply(A->rmap->mapping,IS_GTOLM_DROP,n,rows,&n_l,rows_l);
738:   }
739:   MatZeroRowsLocal(A,n_l,rows_l,diag,x,b);
740:   PetscFree(rows_l);
741:   return(0);
742: }

746: PetscErrorCode MatZeroRowsLocal_IS(Mat A,PetscInt n,const PetscInt rows[],PetscScalar diag,Vec x,Vec b)
747: {
748:   Mat_IS         *is = (Mat_IS*)A->data;
750:   PetscInt       i;
751:   PetscScalar    *array;

754:   if (x && b) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_SUP,"No support");
755:   {
756:     /*
757:        Set up is->x as a "counting vector". This is in order to MatMult_IS
758:        work properly in the interface nodes.
759:     */
760:     Vec counter;
761:     MatCreateVecs(A,NULL,&counter);
762:     VecSet(counter,0.);
763:     VecSet(is->y,1.);
764:     VecScatterBegin(is->rctx,is->y,counter,ADD_VALUES,SCATTER_REVERSE);
765:     VecScatterEnd(is->rctx,is->y,counter,ADD_VALUES,SCATTER_REVERSE);
766:     VecScatterBegin(is->rctx,counter,is->y,INSERT_VALUES,SCATTER_FORWARD);
767:     VecScatterEnd(is->rctx,counter,is->y,INSERT_VALUES,SCATTER_FORWARD);
768:     VecDestroy(&counter);
769:   }
770:   if (!n) {
771:     is->pure_neumann = PETSC_TRUE;
772:   } else {
773:     is->pure_neumann = PETSC_FALSE;

775:     VecGetArray(is->y,&array);
776:     MatZeroRows(is->A,n,rows,diag,0,0);
777:     for (i=0; i<n; i++) {
778:       MatSetValue(is->A,rows[i],rows[i],diag/(array[rows[i]]),INSERT_VALUES);
779:     }
780:     MatAssemblyBegin(is->A,MAT_FINAL_ASSEMBLY);
781:     MatAssemblyEnd(is->A,MAT_FINAL_ASSEMBLY);
782:     VecRestoreArray(is->y,&array);
783:   }
784:   return(0);
785: }

789: PetscErrorCode MatAssemblyBegin_IS(Mat A,MatAssemblyType type)
790: {
791:   Mat_IS         *is = (Mat_IS*)A->data;

795:   MatAssemblyBegin(is->A,type);
796:   return(0);
797: }

801: PetscErrorCode MatAssemblyEnd_IS(Mat A,MatAssemblyType type)
802: {
803:   Mat_IS         *is = (Mat_IS*)A->data;

807:   MatAssemblyEnd(is->A,type);
808:   return(0);
809: }

813: PetscErrorCode MatISGetLocalMat_IS(Mat mat,Mat *local)
814: {
815:   Mat_IS *is = (Mat_IS*)mat->data;

818:   *local = is->A;
819:   return(0);
820: }

824: /*@
825:     MatISGetLocalMat - Gets the local matrix stored inside a MATIS matrix.

827:   Input Parameter:
828: .  mat - the matrix

830:   Output Parameter:
831: .  local - the local matrix

833:   Level: advanced

835:   Notes:
836:     This can be called if you have precomputed the nonzero structure of the
837:   matrix and want to provide it to the inner matrix object to improve the performance
838:   of the MatSetValues() operation.

840: .seealso: MATIS
841: @*/
842: PetscErrorCode MatISGetLocalMat(Mat mat,Mat *local)
843: {

849:   PetscUseMethod(mat,"MatISGetLocalMat_C",(Mat,Mat*),(mat,local));
850:   return(0);
851: }

855: PetscErrorCode MatISSetLocalMat_IS(Mat mat,Mat local)
856: {
857:   Mat_IS         *is = (Mat_IS*)mat->data;
858:   PetscInt       nrows,ncols,orows,ocols;

862:   if (is->A) {
863:     MatGetSize(is->A,&orows,&ocols);
864:     MatGetSize(local,&nrows,&ncols);
865:     if (orows != nrows || ocols != ncols) SETERRQ4(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Local MATIS matrix should be of size %dx%d (you passed a %dx%d matrix)\n",orows,ocols,nrows,ncols);
866:   }
867:   PetscObjectReference((PetscObject)local);
868:   MatDestroy(&is->A);
869:   is->A = local;
870:   return(0);
871: }

875: /*@
876:     MatISSetLocalMat - Replace the local matrix stored inside a MATIS object.

878:   Input Parameter:
879: .  mat - the matrix
880: .  local - the local matrix

882:   Output Parameter:

884:   Level: advanced

886:   Notes:
887:     This can be called if you have precomputed the local matrix and
888:   want to provide it to the matrix object MATIS.

890: .seealso: MATIS
891: @*/
892: PetscErrorCode MatISSetLocalMat(Mat mat,Mat local)
893: {

899:   PetscUseMethod(mat,"MatISSetLocalMat_C",(Mat,Mat),(mat,local));
900:   return(0);
901: }

905: PetscErrorCode MatZeroEntries_IS(Mat A)
906: {
907:   Mat_IS         *a = (Mat_IS*)A->data;

911:   MatZeroEntries(a->A);
912:   return(0);
913: }

917: PetscErrorCode MatScale_IS(Mat A,PetscScalar a)
918: {
919:   Mat_IS         *is = (Mat_IS*)A->data;

923:   MatScale(is->A,a);
924:   return(0);
925: }

929: PetscErrorCode MatGetDiagonal_IS(Mat A, Vec v)
930: {
931:   Mat_IS         *is = (Mat_IS*)A->data;

935:   /* get diagonal of the local matrix */
936:   MatGetDiagonal(is->A,is->y);

938:   /* scatter diagonal back into global vector */
939:   VecSet(v,0);
940:   VecScatterBegin(is->rctx,is->y,v,ADD_VALUES,SCATTER_REVERSE);
941:   VecScatterEnd(is->rctx,is->y,v,ADD_VALUES,SCATTER_REVERSE);
942:   return(0);
943: }

947: PetscErrorCode MatSetOption_IS(Mat A,MatOption op,PetscBool flg)
948: {
949:   Mat_IS         *a = (Mat_IS*)A->data;

953:   MatSetOption(a->A,op,flg);
954:   return(0);
955: }

959: /*@
960:     MatCreateIS - Creates a "process" unassmembled matrix, it is assembled on each
961:        process but not across processes.

963:    Input Parameters:
964: +     comm    - MPI communicator that will share the matrix
965: .     bs      - block size of the matrix
966: .     m,n,M,N - local and/or global sizes of the left and right vector used in matrix vector products
967: .     rmap    - local to global map for rows
968: -     cmap    - local to global map for cols

970:    Output Parameter:
971: .    A - the resulting matrix

973:    Level: advanced

975:    Notes: See MATIS for more details
976:           m and n are NOT related to the size of the map, they are the size of the part of the vector owned
977:           by that process. The sizes of rmap and cmap define the size of the local matrices.
978:           If either rmap or cmap are NULL, than the matrix is assumed to be square

980: .seealso: MATIS, MatSetLocalToGlobalMapping()
981: @*/
982: PetscErrorCode  MatCreateIS(MPI_Comm comm,PetscInt bs,PetscInt m,PetscInt n,PetscInt M,PetscInt N,ISLocalToGlobalMapping rmap,ISLocalToGlobalMapping cmap,Mat *A)
983: {

987:   if (!rmap && !cmap) SETERRQ(comm,PETSC_ERR_USER,"You need to provide at least one of the mapping");
988:   MatCreate(comm,A);
989:   MatSetBlockSize(*A,bs);
990:   MatSetSizes(*A,m,n,M,N);
991:   MatSetType(*A,MATIS);
992:   MatSetUp(*A);
993:   if (rmap && cmap) {
994:     MatSetLocalToGlobalMapping(*A,rmap,cmap);
995:   } else if (!rmap) {
996:     MatSetLocalToGlobalMapping(*A,cmap,cmap);
997:   } else {
998:     MatSetLocalToGlobalMapping(*A,rmap,rmap);
999:   }
1000:   return(0);
1001: }

1003: /*MC
1004:    MATIS - MATIS = "is" - A matrix type to be used for using the non-overlapping domain decomposition type preconditioners (e.g. PCBDDC).
1005:    This stores the matrices in globally unassembled form. Each processor
1006:    assembles only its local Neumann problem and the parallel matrix vector
1007:    product is handled "implicitly".

1009:    Operations Provided:
1010: +  MatMult()
1011: .  MatMultAdd()
1012: .  MatMultTranspose()
1013: .  MatMultTransposeAdd()
1014: .  MatZeroEntries()
1015: .  MatSetOption()
1016: .  MatZeroRows()
1017: .  MatZeroRowsLocal()
1018: .  MatSetValues()
1019: .  MatSetValuesLocal()
1020: .  MatScale()
1021: .  MatGetDiagonal()
1022: -  MatSetLocalToGlobalMapping()

1024:    Options Database Keys:
1025: . -mat_type is - sets the matrix type to "is" during a call to MatSetFromOptions()

1027:    Notes: Options prefix for the inner matrix are given by -is_mat_xxx

1029:           You must call MatSetLocalToGlobalMapping() before using this matrix type.

1031:           You can do matrix preallocation on the local matrix after you obtain it with
1032:           MatISGetLocalMat(); otherwise, you could use MatISSetPreallocation()

1034:   Level: advanced

1036: .seealso: Mat, MatISGetLocalMat(), MatSetLocalToGlobalMapping(), MatISSetPreallocation(), PCBDDC

1038: M*/

1042: PETSC_EXTERN PetscErrorCode MatCreate_IS(Mat A)
1043: {
1045:   Mat_IS         *b;

1048:   PetscNewLog(A,&b);
1049:   A->data = (void*)b;

1051:   /* matrix ops */
1052:   PetscMemzero(A->ops,sizeof(struct _MatOps));
1053:   A->ops->mult                    = MatMult_IS;
1054:   A->ops->multadd                 = MatMultAdd_IS;
1055:   A->ops->multtranspose           = MatMultTranspose_IS;
1056:   A->ops->multtransposeadd        = MatMultTransposeAdd_IS;
1057:   A->ops->destroy                 = MatDestroy_IS;
1058:   A->ops->setlocaltoglobalmapping = MatSetLocalToGlobalMapping_IS;
1059:   A->ops->setvalues               = MatSetValues_IS;
1060:   A->ops->setvalueslocal          = MatSetValuesLocal_IS;
1061:   A->ops->setvaluesblockedlocal   = MatSetValuesBlockedLocal_IS;
1062:   A->ops->zerorows                = MatZeroRows_IS;
1063:   A->ops->zerorowslocal           = MatZeroRowsLocal_IS;
1064:   A->ops->assemblybegin           = MatAssemblyBegin_IS;
1065:   A->ops->assemblyend             = MatAssemblyEnd_IS;
1066:   A->ops->view                    = MatView_IS;
1067:   A->ops->zeroentries             = MatZeroEntries_IS;
1068:   A->ops->scale                   = MatScale_IS;
1069:   A->ops->getdiagonal             = MatGetDiagonal_IS;
1070:   A->ops->setoption               = MatSetOption_IS;
1071:   A->ops->ishermitian             = MatIsHermitian_IS;
1072:   A->ops->issymmetric             = MatIsSymmetric_IS;
1073:   A->ops->duplicate               = MatDuplicate_IS;

1075:   /* special MATIS functions */
1076:   PetscObjectComposeFunction((PetscObject)A,"MatISGetLocalMat_C",MatISGetLocalMat_IS);
1077:   PetscObjectComposeFunction((PetscObject)A,"MatISSetLocalMat_C",MatISSetLocalMat_IS);
1078:   PetscObjectComposeFunction((PetscObject)A,"MatISGetMPIXAIJ_C",MatISGetMPIXAIJ_IS);
1079:   PetscObjectComposeFunction((PetscObject)A,"MatISSetPreallocation_C",MatISSetPreallocation_IS);
1080:   PetscObjectChangeTypeName((PetscObject)A,MATIS);
1081:   return(0);
1082: }