Actual source code: tools.c
petsc-3.3-p7 2013-05-11
1: /*
2: GAMG geometric-algebric multigrid PC - Mark Adams 2011
3: */
4: #include "petsc-private/matimpl.h" /*I "petscmat.h" I*/
5: #include <../src/ksp/pc/impls/gamg/gamg.h> /*I "petscpc.h" I*/
6: #include <petsc-private/kspimpl.h>
8: /* -------------------------------------------------------------------------- */
9: /*
10: PCGAMGCreateGraph - create simple scaled scalar graph from matrix
11:
12: Input Parameter:
13: . Amat - matrix
14: Output Parameter:
15: . a_Gmaat - eoutput scalar graph (symmetric?)
16: */
19: PetscErrorCode PCGAMGCreateGraph( const Mat Amat, Mat *a_Gmat )
20: {
22: PetscInt Istart,Iend,Ii,jj,ncols,nloc,NN,MM,bs;
23: PetscMPIInt mype, npe;
24: MPI_Comm wcomm = ((PetscObject)Amat)->comm;
25: Mat Gmat;
28: MPI_Comm_rank(wcomm,&mype);
29: MPI_Comm_size(wcomm,&npe);
30: MatGetOwnershipRange( Amat, &Istart, &Iend );
31: MatGetSize( Amat, &MM, &NN );
32: MatGetBlockSize( Amat, &bs );
33: nloc = (Iend-Istart)/bs;
34:
35: #if defined PETSC_GAMG_USE_LOG
36: PetscLogEventBegin(petsc_gamg_setup_events[GRAPH],0,0,0,0);
37: #endif
38: if( bs > 1 ) {
39: const PetscScalar *vals;
40: const PetscInt *idx;
41: PetscInt *d_nnz, *o_nnz;
42: /* count nnz, there is sparcity in here so this might not be enough */
43: PetscMalloc( nloc*sizeof(PetscInt), &d_nnz );
44: PetscMalloc( nloc*sizeof(PetscInt), &o_nnz );
45: for ( Ii = Istart, jj = 0 ; Ii < Iend ; Ii += bs, jj++ ) {
46: MatGetRow(Amat,Ii,&ncols,0,0);
47: d_nnz[jj] = ncols; /* very pessimistic */
48: o_nnz[jj] = ncols;
49: if( d_nnz[jj] > nloc ) d_nnz[jj] = nloc;
50: if( o_nnz[jj] > (NN/bs-nloc) ) o_nnz[jj] = NN/bs-nloc;
51: MatRestoreRow(Amat,Ii,&ncols,0,0);
52: }
54: /* get scalar copy (norms) of matrix -- AIJ specific!!! */
55: MatCreateAIJ( wcomm, nloc, nloc,
56: PETSC_DETERMINE, PETSC_DETERMINE,
57: 0, d_nnz, 0, o_nnz, &Gmat );
59: PetscFree( d_nnz );
60: PetscFree( o_nnz );
62: for( Ii = Istart; Ii < Iend ; Ii++ ) {
63: PetscInt dest_row = Ii/bs;
64: MatGetRow(Amat,Ii,&ncols,&idx,&vals);
65: for(jj=0;jj<ncols;jj++){
66: PetscInt dest_col = idx[jj]/bs;
67: PetscScalar sv = PetscAbs(PetscRealPart(vals[jj]));
68: MatSetValues(Gmat,1,&dest_row,1,&dest_col,&sv,ADD_VALUES);
69: }
70: MatRestoreRow(Amat,Ii,&ncols,&idx,&vals);
71: }
72: MatAssemblyBegin(Gmat,MAT_FINAL_ASSEMBLY);
73: MatAssemblyEnd(Gmat,MAT_FINAL_ASSEMBLY);
74: }
75: else {
76: /* just copy scalar matrix - abs() not taken here but scaled later */
77: MatDuplicate( Amat, MAT_COPY_VALUES, &Gmat );
78: }
80: #if defined PETSC_GAMG_USE_LOG
81: PetscLogEventEnd(petsc_gamg_setup_events[GRAPH],0,0,0,0);
82: #endif
84: *a_Gmat = Gmat;
86: return(0);
87: }
89: /* -------------------------------------------------------------------------- */
90: /*
91: PCGAMGFilterGraph - filter graph and symetrize if needed
92:
93: Input Parameter:
94: . vfilter - threshold paramter [0,1)
95: . symm - symetrize?
96: In/Output Parameter:
97: . a_Gmat - original graph
98: */
101: PetscErrorCode PCGAMGFilterGraph( Mat *a_Gmat, const PetscReal vfilter, const PetscBool symm, const PetscInt verbose )
102: {
104: PetscInt Istart,Iend,Ii,jj,ncols,nnz0,nnz1, NN, MM, nloc;
105: PetscMPIInt mype, npe;
106: Mat Gmat = *a_Gmat, tGmat, matTrans;
107: MPI_Comm wcomm = ((PetscObject)Gmat)->comm;
108: const PetscScalar *vals;
109: const PetscInt *idx;
110: PetscInt *d_nnz, *o_nnz;
111: Vec diag;
114: MPI_Comm_rank(wcomm,&mype);
115: MPI_Comm_size(wcomm,&npe);
116: MatGetOwnershipRange( Gmat, &Istart, &Iend );
117: nloc = Iend - Istart;
118: MatGetSize( Gmat, &MM, &NN );
119: #if defined PETSC_GAMG_USE_LOG
120: PetscLogEventBegin(petsc_gamg_setup_events[GRAPH],0,0,0,0);
121: #endif
122: /* scale Gmat so filter works */
123: MatGetVecs( Gmat, &diag, 0 );
124: MatGetDiagonal( Gmat, diag );
125: VecReciprocal( diag );
126: VecSqrtAbs( diag );
127: MatDiagonalScale( Gmat, diag, diag );
128: VecDestroy( &diag );
130: if( symm ) {
131: MatTranspose( Gmat, MAT_INITIAL_MATRIX, &matTrans );
132: }
134: /* filter - dup zeros out matrix */
135: PetscMalloc( nloc*sizeof(PetscInt), &d_nnz );
136: PetscMalloc( nloc*sizeof(PetscInt), &o_nnz );
137: for( Ii = Istart, jj = 0 ; Ii < Iend; Ii++, jj++ ){
138: MatGetRow(Gmat,Ii,&ncols,PETSC_NULL,PETSC_NULL);
139: d_nnz[jj] = ncols;
140: o_nnz[jj] = ncols;
141: MatRestoreRow(Gmat,Ii,&ncols,PETSC_NULL,PETSC_NULL);
142: if( symm ) {
143: MatGetRow(matTrans,Ii,&ncols,PETSC_NULL,PETSC_NULL);
144: d_nnz[jj] += ncols;
145: o_nnz[jj] += ncols;
146: MatRestoreRow(matTrans,Ii,&ncols,PETSC_NULL,PETSC_NULL);
147: }
148: if( d_nnz[jj] > nloc ) d_nnz[jj] = nloc;
149: if( o_nnz[jj] > (MM-nloc) ) o_nnz[jj] = MM - nloc;
150: }
151: MatCreateAIJ( wcomm, nloc, nloc, MM, MM, 0, d_nnz, 0, o_nnz, &tGmat );
152:
153: PetscFree( d_nnz );
154: PetscFree( o_nnz );
155: if( symm ) {
156: MatDestroy( &matTrans );
157: }
159: for( Ii = Istart, nnz0 = nnz1 = 0 ; Ii < Iend; Ii++ ){
160: MatGetRow(Gmat,Ii,&ncols,&idx,&vals);
161: for(jj=0;jj<ncols;jj++,nnz0++){
162: PetscScalar sv = PetscAbs(PetscRealPart(vals[jj]));
163: if( PetscRealPart(sv) > vfilter ) {
164: nnz1++;
165: if( symm ) {
166: sv *= 0.5;
167: MatSetValues(tGmat,1,&Ii,1,&idx[jj],&sv,ADD_VALUES);
168: MatSetValues(tGmat,1,&idx[jj],1,&Ii,&sv,ADD_VALUES);
169: }
170: else {
171: MatSetValues(tGmat,1,&Ii,1,&idx[jj],&sv,ADD_VALUES);
172: }
173: }
174: }
175: MatRestoreRow(Gmat,Ii,&ncols,&idx,&vals);
176: }
177: MatAssemblyBegin(tGmat,MAT_FINAL_ASSEMBLY);
178: MatAssemblyEnd(tGmat,MAT_FINAL_ASSEMBLY);
180: #if defined PETSC_GAMG_USE_LOG
181: PetscLogEventEnd(petsc_gamg_setup_events[GRAPH],0,0,0,0);
182: #endif
184: if( verbose ) {
185: if( verbose == 1 ) {
186: PetscPrintf(wcomm,"\t[%d]%s %g%% nnz after filtering, with threshold %g, %g nnz ave. (N=%d)\n",mype,__FUNCT__,
187: 100.*(double)nnz1/(double)nnz0,vfilter,(double)nnz0/(double)nloc,MM);
188: }
189: else {
190: PetscInt nnz[2],out[2];
191: nnz[0] = nnz0;
192: nnz[1] = nnz1;
193: MPI_Allreduce( nnz, out, 2, MPIU_INT, MPI_SUM, wcomm );
194: PetscPrintf(wcomm,"\t[%d]%s %g%% nnz after filtering, with threshold %g, %g nnz ave. (N=%d)\n",mype,__FUNCT__,
195: 100.*(double)out[1]/(double)out[0],vfilter,(double)out[0]/(double)MM,MM);
196: }
197: }
198:
199: MatDestroy( &Gmat );
201: *a_Gmat = tGmat;
203: return(0);
204: }
206: /* -------------------------------------------------------------------------- */
207: /*
208: PCGAMGGetDataWithGhosts - hacks into Mat MPIAIJ so this must have > 1 pe
210: Input Parameter:
211: . Gmat - MPIAIJ matrix for scattters
212: . data_sz - number of data terms per node (# cols in output)
213: . data_in[nloc*data_sz] - column oriented data
214: Output Parameter:
215: . a_stride - numbrt of rows of output
216: . a_data_out[stride*data_sz] - output data with ghosts
217: */
220: PetscErrorCode PCGAMGGetDataWithGhosts( const Mat Gmat,
221: const PetscInt data_sz,
222: const PetscReal data_in[],
223: PetscInt *a_stride,
224: PetscReal **a_data_out
225: )
226: {
228: PetscMPIInt mype,npe;
229: MPI_Comm wcomm = ((PetscObject)Gmat)->comm;
230: Vec tmp_crds;
231: Mat_MPIAIJ *mpimat = (Mat_MPIAIJ*)Gmat->data;
232: PetscInt nnodes,num_ghosts,dir,kk,jj,my0,Iend,nloc;
233: PetscScalar *data_arr;
234: PetscReal *datas;
235: PetscBool isMPIAIJ;
238: PetscObjectTypeCompare( (PetscObject)Gmat, MATMPIAIJ, &isMPIAIJ );
239: MPI_Comm_rank(wcomm,&mype);
240: MPI_Comm_size(wcomm,&npe);
241: MatGetOwnershipRange( Gmat, &my0, &Iend );
242: nloc = Iend - my0;
243: VecGetLocalSize( mpimat->lvec, &num_ghosts );
244: nnodes = num_ghosts + nloc;
245: *a_stride = nnodes;
246: MatGetVecs( Gmat, &tmp_crds, 0 );
248: PetscMalloc( data_sz*nnodes*sizeof(PetscReal), &datas);
249: for(dir=0; dir<data_sz; dir++) {
250: /* set local, and global */
251: for(kk=0; kk<nloc; kk++) {
252: PetscInt gid = my0 + kk;
253: PetscScalar crd = (PetscScalar)data_in[dir*nloc + kk]; /* col oriented */
254: datas[dir*nnodes + kk] = PetscRealPart(crd);
255: VecSetValues(tmp_crds, 1, &gid, &crd, INSERT_VALUES );
256: }
257: VecAssemblyBegin( tmp_crds );
258: VecAssemblyEnd( tmp_crds );
259: /* get ghost datas */
260: VecScatterBegin(mpimat->Mvctx,tmp_crds,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
261:
262: VecScatterEnd(mpimat->Mvctx,tmp_crds,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
263:
264: VecGetArray( mpimat->lvec, &data_arr );
265: for(kk=nloc,jj=0;jj<num_ghosts;kk++,jj++){
266: datas[dir*nnodes + kk] = PetscRealPart(data_arr[jj]);
267: }
268: VecRestoreArray( mpimat->lvec, &data_arr );
269: }
270: VecDestroy(&tmp_crds);
272: *a_data_out = datas;
274: return(0);
275: }
278: /* hash table stuff - simple, not dymanic, key >= 0, has table
279: *
280: * GAMGTableCreate
281: */
282: /* avoid overflow */
283: #define GAMG_HASH(key) ((7*key)%a_tab->size)
284: PetscErrorCode GAMGTableCreate( PetscInt a_size, GAMGHashTable *a_tab )
285: {
287: PetscInt kk;
288: a_tab->size = a_size;
289: PetscMalloc(a_size*sizeof(PetscInt), &a_tab->table );
290: PetscMalloc(a_size*sizeof(PetscInt), &a_tab->data );
291: for(kk=0;kk<a_size;kk++) a_tab->table[kk] = -1;
292: return 0;
293: }
295: PetscErrorCode GAMGTableDestroy( GAMGHashTable *a_tab )
296: {
298: PetscFree( a_tab->table );
299: PetscFree( a_tab->data );
300: return 0;
301: }
303: PetscErrorCode GAMGTableAdd( GAMGHashTable *a_tab, PetscInt a_key, PetscInt a_data )
304: {
305: PetscInt kk,idx;
306: if(a_key<0)SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"Table size %d too small.",a_tab->size);
307: for( kk = 0, idx = GAMG_HASH(a_key) ; kk < a_tab->size ; kk++, idx = (idx==(a_tab->size-1)) ? 0 : idx + 1 ){
308: if( a_tab->table[idx] == a_key ) {
309: /* exists */
310: assert(0); /* not used this way now */
311: a_tab->data[idx] = a_data;
312: break;
313: }
314: else if( a_tab->table[idx] == -1 ) {
315: /* add */
316: a_tab->table[idx] = a_key;
317: a_tab->data[idx] = a_data;
318: break;
319: }
320: }
321: if(kk==a_tab->size) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"Table size %d too small.",a_tab->size);
322: return 0;
323: }
325: PetscErrorCode GAMGTableFind( GAMGHashTable *a_tab, PetscInt a_key, PetscInt *a_data )
326: {
327: PetscInt kk,idx;
328: if(a_key<0)SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"Table size %d too small.",a_tab->size);
329: for( kk = 0, idx = GAMG_HASH(a_key) ; kk < a_tab->size ; kk++, idx = (idx==(a_tab->size-1)) ? 0 : idx + 1 ){
330: if( a_tab->table[idx] == a_key ) {
331: *a_data = a_tab->data[idx];
332: break;
333: }
334: else if( a_tab->table[idx] == -1 ) {
335: /* not here */
336: *a_data = -1;
337: break;
338: }
339: }
340: if(kk==a_tab->size) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"Table size %d too small.",a_tab->size);
341: return 0;
342: }