Actual source code: swarm.c

petsc-master 2020-09-18
Report Typos and Errors
  1: #define PETSCDM_DLL
  2: #include <petsc/private/dmswarmimpl.h>
  3: #include <petsc/private/hashsetij.h>
  4: #include <petsc/private/petscfeimpl.h>
  5: #include <petscviewer.h>
  6: #include <petscdraw.h>
  7: #include <petscdmplex.h>
  8: #include <petscblaslapack.h>
  9: #include "../src/dm/impls/swarm/data_bucket.h"

 11: PetscLogEvent DMSWARM_Migrate, DMSWARM_SetSizes, DMSWARM_AddPoints, DMSWARM_RemovePoints, DMSWARM_Sort;
 12: PetscLogEvent DMSWARM_DataExchangerTopologySetup, DMSWARM_DataExchangerBegin, DMSWARM_DataExchangerEnd;
 13: PetscLogEvent DMSWARM_DataExchangerSendCount, DMSWARM_DataExchangerPack;

 15: const char* DMSwarmTypeNames[] = { "basic", "pic", NULL };
 16: const char* DMSwarmMigrateTypeNames[] = { "basic", "dmcellnscatter", "dmcellexact", "user", NULL };
 17: const char* DMSwarmCollectTypeNames[] = { "basic", "boundingbox", "general", "user", NULL  };
 18: const char* DMSwarmPICLayoutTypeNames[] = { "regular", "gauss", "subdivision", NULL  };

 20: const char DMSwarmField_pid[] = "DMSwarm_pid";
 21: const char DMSwarmField_rank[] = "DMSwarm_rank";
 22: const char DMSwarmPICField_coor[] = "DMSwarmPIC_coor";
 23: const char DMSwarmPICField_cellid[] = "DMSwarm_cellid";

 25: PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
 26: PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);

 28: #if defined(PETSC_HAVE_HDF5)
 29: #include <petscviewerhdf5.h>

 31: PetscErrorCode VecView_Swarm_HDF5_Internal(Vec v, PetscViewer viewer)
 32: {
 33:   DM             dm;
 34:   PetscReal      seqval;
 35:   PetscInt       seqnum, bs;
 36:   PetscBool      isseq;

 40:   VecGetDM(v, &dm);
 41:   VecGetBlockSize(v, &bs);
 42:   PetscViewerHDF5PushGroup(viewer, "/particle_fields");
 43:   PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);
 44:   DMGetOutputSequenceNumber(dm, &seqnum, &seqval);
 45:   PetscViewerHDF5SetTimestep(viewer, seqnum);
 46:   /* DMSequenceView_HDF5(dm, "time", seqnum, (PetscScalar) seqval, viewer); */
 47:   if (isseq) {VecView_Seq(v, viewer);}
 48:   else       {VecView_MPI(v, viewer);}
 49:   PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject) v, "Nc", PETSC_INT, (void *) &bs);
 50:   PetscViewerHDF5PopGroup(viewer);
 51:   return(0);
 52: }

 54: PetscErrorCode DMSwarmView_HDF5(DM dm, PetscViewer viewer)
 55: {
 56:   Vec            coordinates;
 57:   PetscInt       Np;
 58:   PetscBool      isseq;

 62:   DMSwarmGetSize(dm, &Np);
 63:   DMSwarmCreateGlobalVectorFromField(dm, DMSwarmPICField_coor, &coordinates);
 64:   PetscObjectSetName((PetscObject) coordinates, "coordinates");
 65:   PetscViewerHDF5PushGroup(viewer, "/particles");
 66:   PetscObjectTypeCompare((PetscObject) coordinates, VECSEQ, &isseq);
 67:   if (isseq) {VecView_Seq(coordinates, viewer);}
 68:   else       {VecView_MPI(coordinates, viewer);}
 69:   PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject) coordinates, "Np", PETSC_INT, (void *) &Np);
 70:   PetscViewerHDF5PopGroup(viewer);
 71:   DMSwarmDestroyGlobalVectorFromField(dm, DMSwarmPICField_coor, &coordinates);
 72:   return(0);
 73: }
 74: #endif

 76: PetscErrorCode VecView_Swarm(Vec v, PetscViewer viewer)
 77: {
 78:   DM             dm;
 79:   PetscBool      ishdf5;

 83:   VecGetDM(v, &dm);
 84:   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
 85:   PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);
 86:   if (ishdf5) {
 87: #if defined(PETSC_HAVE_HDF5)
 88:     VecView_Swarm_HDF5_Internal(v, viewer);
 89: #else
 90:     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
 91: #endif
 92:   } else {
 93:     PetscBool isseq;

 95:     PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);
 96:     if (isseq) {VecView_Seq(v, viewer);}
 97:     else       {VecView_MPI(v, viewer);}
 98:   }
 99:   return(0);
100: }

102: /*@C
103:    DMSwarmVectorDefineField - Sets the field from which to define a Vec object
104:                              when DMCreateLocalVector(), or DMCreateGlobalVector() is called

106:    Collective on dm

108:    Input parameters:
109: +  dm - a DMSwarm
110: -  fieldname - the textual name given to a registered field

112:    Level: beginner

114:    Notes:

116:    The field with name fieldname must be defined as having a data type of PetscScalar.

118:    This function must be called prior to calling DMCreateLocalVector(), DMCreateGlobalVector().
119:    Mutiple calls to DMSwarmVectorDefineField() are permitted.

121: .seealso: DMSwarmRegisterPetscDatatypeField(), DMCreateGlobalVector(), DMCreateLocalVector()
122: @*/
123: PetscErrorCode DMSwarmVectorDefineField(DM dm,const char fieldname[])
124: {
125:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;
127:   PetscInt       bs,n;
128:   PetscScalar    *array;
129:   PetscDataType  type;

132:   if (!swarm->issetup) { DMSetUp(dm); }
133:   DMSwarmDataBucketGetSizes(swarm->db,&n,NULL,NULL);
134:   DMSwarmGetField(dm,fieldname,&bs,&type,(void**)&array);

136:   /* Check all fields are of type PETSC_REAL or PETSC_SCALAR */
137:   if (type != PETSC_REAL) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Only valid for PETSC_REAL");
138:   PetscSNPrintf(swarm->vec_field_name,PETSC_MAX_PATH_LEN-1,"%s",fieldname);
139:   swarm->vec_field_set = PETSC_TRUE;
140:   swarm->vec_field_bs = bs;
141:   swarm->vec_field_nlocal = n;
142:   DMSwarmRestoreField(dm,fieldname,&bs,&type,(void**)&array);
143:   return(0);
144: }

146: /* requires DMSwarmDefineFieldVector has been called */
147: PetscErrorCode DMCreateGlobalVector_Swarm(DM dm,Vec *vec)
148: {
149:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;
151:   Vec            x;
152:   char           name[PETSC_MAX_PATH_LEN];

155:   if (!swarm->issetup) { DMSetUp(dm); }
156:   if (!swarm->vec_field_set) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"Must call DMSwarmVectorDefineField first");
157:   if (swarm->vec_field_nlocal != swarm->db->L) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"DMSwarm sizes have changed since last call to VectorDefineField first"); /* Stale data */

159:   PetscSNPrintf(name,PETSC_MAX_PATH_LEN-1,"DMSwarmField_%s",swarm->vec_field_name);
160:   VecCreate(PetscObjectComm((PetscObject)dm),&x);
161:   PetscObjectSetName((PetscObject)x,name);
162:   VecSetSizes(x,swarm->db->L*swarm->vec_field_bs,PETSC_DETERMINE);
163:   VecSetBlockSize(x,swarm->vec_field_bs);
164:   VecSetDM(x,dm);
165:   VecSetFromOptions(x);
166:   VecSetDM(x, dm);
167:   VecSetOperation(x, VECOP_VIEW, (void (*)(void)) VecView_Swarm);
168:   *vec = x;
169:   return(0);
170: }

172: /* requires DMSwarmDefineFieldVector has been called */
173: PetscErrorCode DMCreateLocalVector_Swarm(DM dm,Vec *vec)
174: {
175:   DM_Swarm *swarm = (DM_Swarm*)dm->data;
177:   Vec x;
178:   char name[PETSC_MAX_PATH_LEN];

181:   if (!swarm->issetup) { DMSetUp(dm); }
182:   if (!swarm->vec_field_set) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"Must call DMSwarmVectorDefineField first");
183:   if (swarm->vec_field_nlocal != swarm->db->L) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"DMSwarm sizes have changed since last call to VectorDefineField first"); /* Stale data */

185:   PetscSNPrintf(name,PETSC_MAX_PATH_LEN-1,"DMSwarmField_%s",swarm->vec_field_name);
186:   VecCreate(PETSC_COMM_SELF,&x);
187:   PetscObjectSetName((PetscObject)x,name);
188:   VecSetSizes(x,swarm->db->L*swarm->vec_field_bs,PETSC_DETERMINE);
189:   VecSetBlockSize(x,swarm->vec_field_bs);
190:   VecSetDM(x,dm);
191:   VecSetFromOptions(x);
192:   *vec = x;
193:   return(0);
194: }

196: static PetscErrorCode DMSwarmDestroyVectorFromField_Private(DM dm, const char fieldname[], Vec *vec)
197: {
198:   DM_Swarm      *swarm = (DM_Swarm *) dm->data;
199:   DMSwarmDataField      gfield;
200:   void         (*fptr)(void);
201:   PetscInt       bs, nlocal;
202:   char           name[PETSC_MAX_PATH_LEN];

206:   VecGetLocalSize(*vec, &nlocal);
207:   VecGetBlockSize(*vec, &bs);
208:   if (nlocal/bs != swarm->db->L) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"DMSwarm sizes have changed since vector was created - cannot ensure pointers are valid"); /* Stale data */
209:   DMSwarmDataBucketGetDMSwarmDataFieldByName(swarm->db, fieldname, &gfield);
210:   /* check vector is an inplace array */
211:   PetscSNPrintf(name, PETSC_MAX_PATH_LEN-1, "DMSwarm_VecFieldInPlace_%s", fieldname);
212:   PetscObjectQueryFunction((PetscObject) *vec, name, &fptr);
213:   if (!fptr) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_USER, "Vector being destroyed was not created from DMSwarm field(%s)", fieldname);
214:   DMSwarmDataFieldRestoreAccess(gfield);
215:   VecDestroy(vec);
216:   return(0);
217: }

219: static PetscErrorCode DMSwarmCreateVectorFromField_Private(DM dm, const char fieldname[], MPI_Comm comm, Vec *vec)
220: {
221:   DM_Swarm      *swarm = (DM_Swarm *) dm->data;
222:   PetscDataType  type;
223:   PetscScalar   *array;
224:   PetscInt       bs, n;
225:   char           name[PETSC_MAX_PATH_LEN];
226:   PetscMPIInt    size;

230:   if (!swarm->issetup) {DMSetUp(dm);}
231:   DMSwarmDataBucketGetSizes(swarm->db, &n, NULL, NULL);
232:   DMSwarmGetField(dm, fieldname, &bs, &type, (void **) &array);
233:   if (type != PETSC_REAL) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Only valid for PETSC_REAL");

235:   MPI_Comm_size(comm, &size);
236:   if (size == 1) {
237:     VecCreateSeqWithArray(comm, bs, n*bs, array, vec);
238:   } else {
239:     VecCreateMPIWithArray(comm, bs, n*bs, PETSC_DETERMINE, array, vec);
240:   }
241:   PetscSNPrintf(name, PETSC_MAX_PATH_LEN-1, "DMSwarmSharedField_%s", fieldname);
242:   PetscObjectSetName((PetscObject) *vec, name);

244:   /* Set guard */
245:   PetscSNPrintf(name, PETSC_MAX_PATH_LEN-1, "DMSwarm_VecFieldInPlace_%s", fieldname);
246:   PetscObjectComposeFunction((PetscObject) *vec, name, DMSwarmDestroyVectorFromField_Private);

248:   VecSetDM(*vec, dm);
249:   VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Swarm);
250:   return(0);
251: }

253: /* This creates a "mass matrix" between a finite element and particle space. If a finite element interpolant is given by

255:      \hat f = \sum_i f_i \phi_i

257:    and a particle function is given by

259:      f = \sum_i w_i \delta(x - x_i)

261:    then we want to require that

263:      M \hat f = M_p f

265:    where the particle mass matrix is given by

267:      (M_p)_{ij} = \int \phi_i \delta(x - x_j)

269:    The way Dave May does particles, they amount to quadratue weights rather than delta functions, so he has |J| is in
270:    his integral. We allow this with the boolean flag.
271: */
272: static PetscErrorCode DMSwarmComputeMassMatrix_Private(DM dmc, DM dmf, Mat mass, PetscBool useDeltaFunction, void *ctx)
273: {
274:   const char    *name = "Mass Matrix";
275:   MPI_Comm       comm;
276:   PetscDS        prob;
277:   PetscSection   fsection, globalFSection;
278:   PetscHSetIJ    ht;
279:   PetscLayout    rLayout, colLayout;
280:   PetscInt      *dnz, *onz;
281:   PetscInt       locRows, locCols, rStart, colStart, colEnd, *rowIDXs;
282:   PetscReal     *xi, *v0, *J, *invJ, detJ = 1.0, v0ref[3] = {-1.0, -1.0, -1.0};
283:   PetscScalar   *elemMat;
284:   PetscInt       dim, Nf, field, cStart, cEnd, cell, totDim, maxC = 0;

288:   PetscObjectGetComm((PetscObject) mass, &comm);
289:   DMGetCoordinateDim(dmf, &dim);
290:   DMGetDS(dmf, &prob);
291:   PetscDSGetNumFields(prob, &Nf);
292:   PetscDSGetTotalDimension(prob, &totDim);
293:   PetscMalloc3(dim, &v0, dim*dim, &J, dim*dim,&invJ);
294:   DMGetLocalSection(dmf, &fsection);
295:   DMGetGlobalSection(dmf, &globalFSection);
296:   DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd);
297:   MatGetLocalSize(mass, &locRows, &locCols);

299:   PetscLayoutCreate(comm, &colLayout);
300:   PetscLayoutSetLocalSize(colLayout, locCols);
301:   PetscLayoutSetBlockSize(colLayout, 1);
302:   PetscLayoutSetUp(colLayout);
303:   PetscLayoutGetRange(colLayout, &colStart, &colEnd);
304:   PetscLayoutDestroy(&colLayout);

306:   PetscLayoutCreate(comm, &rLayout);
307:   PetscLayoutSetLocalSize(rLayout, locRows);
308:   PetscLayoutSetBlockSize(rLayout, 1);
309:   PetscLayoutSetUp(rLayout);
310:   PetscLayoutGetRange(rLayout, &rStart, NULL);
311:   PetscLayoutDestroy(&rLayout);

313:   PetscCalloc2(locRows, &dnz, locRows, &onz);
314:   PetscHSetIJCreate(&ht);

316:   PetscSynchronizedFlush(comm, NULL);
317:   /* count non-zeros */
318:   DMSwarmSortGetAccess(dmc);
319:   for (field = 0; field < Nf; ++field) {
320:     for (cell = cStart; cell < cEnd; ++cell) {
321:       PetscInt  c, i;
322:       PetscInt *findices,   *cindices; /* fine is vertices, coarse is particles */
323:       PetscInt  numFIndices, numCIndices;

325:       DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);
326:       DMSwarmSortGetPointsPerCell(dmc, cell, &numCIndices, &cindices);
327:       maxC = PetscMax(maxC, numCIndices);
328:       {
329:         PetscHashIJKey key;
330:         PetscBool      missing;
331:         for (i = 0; i < numFIndices; ++i) {
332:           key.j = findices[i]; /* global column (from Plex) */
333:           if (key.j >= 0) {
334:             /* Get indices for coarse elements */
335:             for (c = 0; c < numCIndices; ++c) {
336:               key.i = cindices[c] + rStart; /* global cols (from Swarm) */
337:               if (key.i < 0) continue;
338:               PetscHSetIJQueryAdd(ht, key, &missing);
339:               if (missing) {
340:                 if ((key.j >= colStart) && (key.j < colEnd)) ++dnz[key.i - rStart];
341:                 else                                         ++onz[key.i - rStart];
342:               } else SETERRQ2(PetscObjectComm((PetscObject) dmf), PETSC_ERR_SUP, "Set new value at %D,%D", key.i, key.j);
343:             }
344:           }
345:         }
346:         PetscFree(cindices);
347:       }
348:       DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);
349:     }
350:   }
351:   PetscHSetIJDestroy(&ht);
352:   MatXAIJSetPreallocation(mass, 1, dnz, onz, NULL, NULL);
353:   MatSetOption(mass, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE);
354:   PetscFree2(dnz, onz);
355:   PetscMalloc3(maxC*totDim, &elemMat, maxC, &rowIDXs, maxC*dim, &xi);
356:   for (field = 0; field < Nf; ++field) {
357:     PetscTabulation Tcoarse;
358:     PetscObject       obj;
359:     PetscReal        *coords;
360:     PetscInt          Nc, i;

362:     PetscDSGetDiscretization(prob, field, &obj);
363:     PetscFEGetNumComponents((PetscFE) obj, &Nc);
364:     if (Nc != 1) SETERRQ1(PetscObjectComm((PetscObject) dmf), PETSC_ERR_SUP, "Can only interpolate a scalar field from particles, Nc = %D", Nc);
365:     DMSwarmGetField(dmc, DMSwarmPICField_coor, NULL, NULL, (void **) &coords);
366:     for (cell = cStart; cell < cEnd; ++cell) {
367:       PetscInt *findices  , *cindices;
368:       PetscInt  numFIndices, numCIndices;
369:       PetscInt  p, c;

371:       /* TODO: Use DMField instead of assuming affine */
372:       DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ);
373:       DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);
374:       DMSwarmSortGetPointsPerCell(dmc, cell, &numCIndices, &cindices);
375:       for (p = 0; p < numCIndices; ++p) {
376:         CoordinatesRealToRef(dim, dim, v0ref, v0, invJ, &coords[cindices[p]*dim], &xi[p*dim]);
377:       }
378:       PetscFECreateTabulation((PetscFE) obj, 1, numCIndices, xi, 0, &Tcoarse);
379:       /* Get elemMat entries by multiplying by weight */
380:       PetscArrayzero(elemMat, numCIndices*totDim);
381:       for (i = 0; i < numFIndices; ++i) {
382:         for (p = 0; p < numCIndices; ++p) {
383:           for (c = 0; c < Nc; ++c) {
384:             /* B[(p*pdim + i)*Nc + c] is the value at point p for basis function i and component c */
385:             elemMat[p*numFIndices+i] += Tcoarse->T[0][(p*numFIndices + i)*Nc + c]*(useDeltaFunction ? 1.0 : detJ);
386:           }
387:         }
388:       }
389:       for (p = 0; p < numCIndices; ++p) rowIDXs[p] = cindices[p] + rStart;
390:       if (0) {DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat);}
391:       MatSetValues(mass, numCIndices, rowIDXs, numFIndices, findices, elemMat, ADD_VALUES);
392:       PetscFree(cindices);
393:       DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);
394:       PetscTabulationDestroy(&Tcoarse);
395:     }
396:     DMSwarmRestoreField(dmc, DMSwarmPICField_coor, NULL, NULL, (void **) &coords);
397:   }
398:   PetscFree3(elemMat, rowIDXs, xi);
399:   DMSwarmSortRestoreAccess(dmc);
400:   PetscFree3(v0, J, invJ);
401:   MatAssemblyBegin(mass, MAT_FINAL_ASSEMBLY);
402:   MatAssemblyEnd(mass, MAT_FINAL_ASSEMBLY);
403:   return(0);
404: }

406: /* FEM cols, Particle rows */
407: static PetscErrorCode DMCreateMassMatrix_Swarm(DM dmCoarse, DM dmFine, Mat *mass)
408: {
409:   PetscSection   gsf;
410:   PetscInt       m, n;
411:   void          *ctx;

415:   DMGetGlobalSection(dmFine, &gsf);
416:   PetscSectionGetConstrainedStorageSize(gsf, &m);
417:   DMSwarmGetLocalSize(dmCoarse, &n);
418:   MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);
419:   MatSetSizes(*mass, n, m, PETSC_DETERMINE, PETSC_DETERMINE);
420:   MatSetType(*mass, dmCoarse->mattype);
421:   DMGetApplicationContext(dmFine, &ctx);

423:   DMSwarmComputeMassMatrix_Private(dmCoarse, dmFine, *mass, PETSC_TRUE, ctx);
424:   MatViewFromOptions(*mass, NULL, "-mass_mat_view");
425:   return(0);
426: }

428: static PetscErrorCode DMSwarmComputeMassMatrixSquare_Private(DM dmc, DM dmf, Mat mass, PetscBool useDeltaFunction, void *ctx)
429: {
430:   const char    *name = "Mass Matrix Square";
431:   MPI_Comm       comm;
432:   PetscDS        prob;
433:   PetscSection   fsection, globalFSection;
434:   PetscHSetIJ    ht;
435:   PetscLayout    rLayout, colLayout;
436:   PetscInt      *dnz, *onz, *adj, depth, maxConeSize, maxSupportSize, maxAdjSize;
437:   PetscInt       locRows, locCols, rStart, colStart, colEnd, *rowIDXs;
438:   PetscReal     *xi, *v0, *J, *invJ, detJ = 1.0, v0ref[3] = {-1.0, -1.0, -1.0};
439:   PetscScalar   *elemMat, *elemMatSq;
440:   PetscInt       cdim, Nf, field, cStart, cEnd, cell, totDim, maxC = 0;

444:   PetscObjectGetComm((PetscObject) mass, &comm);
445:   DMGetCoordinateDim(dmf, &cdim);
446:   DMGetDS(dmf, &prob);
447:   PetscDSGetNumFields(prob, &Nf);
448:   PetscDSGetTotalDimension(prob, &totDim);
449:   PetscMalloc3(cdim, &v0, cdim*cdim, &J, cdim*cdim,&invJ);
450:   DMGetLocalSection(dmf, &fsection);
451:   DMGetGlobalSection(dmf, &globalFSection);
452:   DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd);
453:   MatGetLocalSize(mass, &locRows, &locCols);

455:   PetscLayoutCreate(comm, &colLayout);
456:   PetscLayoutSetLocalSize(colLayout, locCols);
457:   PetscLayoutSetBlockSize(colLayout, 1);
458:   PetscLayoutSetUp(colLayout);
459:   PetscLayoutGetRange(colLayout, &colStart, &colEnd);
460:   PetscLayoutDestroy(&colLayout);

462:   PetscLayoutCreate(comm, &rLayout);
463:   PetscLayoutSetLocalSize(rLayout, locRows);
464:   PetscLayoutSetBlockSize(rLayout, 1);
465:   PetscLayoutSetUp(rLayout);
466:   PetscLayoutGetRange(rLayout, &rStart, NULL);
467:   PetscLayoutDestroy(&rLayout);

469:   DMPlexGetDepth(dmf, &depth);
470:   DMPlexGetMaxSizes(dmf, &maxConeSize, &maxSupportSize);
471:   maxAdjSize = PetscPowInt(maxConeSize*maxSupportSize, depth);
472:   PetscMalloc1(maxAdjSize, &adj);

474:   PetscCalloc2(locRows, &dnz, locRows, &onz);
475:   PetscHSetIJCreate(&ht);
476:   /* Count nonzeros
477:        This is just FVM++, but we cannot use the Plex P0 allocation since unknowns in a cell will not be contiguous
478:   */
479:   DMSwarmSortGetAccess(dmc);
480:   for (cell = cStart; cell < cEnd; ++cell) {
481:     PetscInt  i;
482:     PetscInt *cindices;
483:     PetscInt  numCIndices;
484:   #if 0
485:     PetscInt  adjSize = maxAdjSize, a, j;
486:   #endif

488:     DMSwarmSortGetPointsPerCell(dmc, cell, &numCIndices, &cindices);
489:     maxC = PetscMax(maxC, numCIndices);
490:     /* Diagonal block */
491:     for (i = 0; i < numCIndices; ++i) {dnz[cindices[i]] += numCIndices;}
492: #if 0
493:     /* Off-diagonal blocks */
494:     DMPlexGetAdjacency(dmf, cell, &adjSize, &adj);
495:     for (a = 0; a < adjSize; ++a) {
496:       if (adj[a] >= cStart && adj[a] < cEnd && adj[a] != cell) {
497:         const PetscInt ncell = adj[a];
498:         PetscInt      *ncindices;
499:         PetscInt       numNCIndices;

501:         DMSwarmSortGetPointsPerCell(dmc, ncell, &numNCIndices, &ncindices);
502:         {
503:           PetscHashIJKey key;
504:           PetscBool      missing;

506:           for (i = 0; i < numCIndices; ++i) {
507:             key.i = cindices[i] + rStart; /* global rows (from Swarm) */
508:             if (key.i < 0) continue;
509:             for (j = 0; j < numNCIndices; ++j) {
510:               key.j = ncindices[j] + rStart; /* global column (from Swarm) */
511:               if (key.j < 0) continue;
512:               PetscHSetIJQueryAdd(ht, key, &missing);
513:               if (missing) {
514:                 if ((key.j >= colStart) && (key.j < colEnd)) ++dnz[key.i - rStart];
515:                 else                                         ++onz[key.i - rStart];
516:               }
517:             }
518:           }
519:         }
520:         PetscFree(ncindices);
521:       }
522:     }
523: #endif
524:     PetscFree(cindices);
525:   }
526:   PetscHSetIJDestroy(&ht);
527:   MatXAIJSetPreallocation(mass, 1, dnz, onz, NULL, NULL);
528:   MatSetOption(mass, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE);
529:   PetscFree2(dnz, onz);
530:   PetscMalloc4(maxC*totDim, &elemMat, maxC*maxC, &elemMatSq, maxC, &rowIDXs, maxC*cdim, &xi);
531:   /* Fill in values
532:        Each entry is a sum of terms \phi_i(x_p) \phi_i(x_q)
533:        Start just by producing block diagonal
534:        Could loop over adjacent cells
535:          Produce neighboring element matrix
536:          TODO Determine which columns and rows correspond to shared dual vector
537:          Do MatMatMult with rectangular matrices
538:          Insert block
539:   */
540:   for (field = 0; field < Nf; ++field) {
541:     PetscTabulation Tcoarse;
542:     PetscObject       obj;
543:     PetscReal        *coords;
544:     PetscInt          Nc, i;

546:     PetscDSGetDiscretization(prob, field, &obj);
547:     PetscFEGetNumComponents((PetscFE) obj, &Nc);
548:     if (Nc != 1) SETERRQ1(PetscObjectComm((PetscObject) dmf), PETSC_ERR_SUP, "Can only interpolate a scalar field from particles, Nc = %D", Nc);
549:     DMSwarmGetField(dmc, DMSwarmPICField_coor, NULL, NULL, (void **) &coords);
550:     for (cell = cStart; cell < cEnd; ++cell) {
551:       PetscInt *findices  , *cindices;
552:       PetscInt  numFIndices, numCIndices;
553:       PetscInt  p, c;

555:       /* TODO: Use DMField instead of assuming affine */
556:       DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ);
557:       DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);
558:       DMSwarmSortGetPointsPerCell(dmc, cell, &numCIndices, &cindices);
559:       for (p = 0; p < numCIndices; ++p) {
560:         CoordinatesRealToRef(cdim, cdim, v0ref, v0, invJ, &coords[cindices[p]*cdim], &xi[p*cdim]);
561:       }
562:       PetscFECreateTabulation((PetscFE) obj, 1, numCIndices, xi, 0, &Tcoarse);
563:       /* Get elemMat entries by multiplying by weight */
564:       PetscArrayzero(elemMat, numCIndices*totDim);
565:       for (i = 0; i < numFIndices; ++i) {
566:         for (p = 0; p < numCIndices; ++p) {
567:           for (c = 0; c < Nc; ++c) {
568:             /* B[(p*pdim + i)*Nc + c] is the value at point p for basis function i and component c */
569:             elemMat[p*numFIndices+i] += Tcoarse->T[0][(p*numFIndices + i)*Nc + c]*(useDeltaFunction ? 1.0 : detJ);
570:           }
571:         }
572:       }
573:       PetscTabulationDestroy(&Tcoarse);
574:       for (p = 0; p < numCIndices; ++p) rowIDXs[p] = cindices[p] + rStart;
575:       if (0) {DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat);}
576:       /* Block diagonal */
577:       {
578:         PetscBLASInt blasn, blask;
579:         PetscScalar  one = 1.0, zero = 0.0;

581:         PetscBLASIntCast(numCIndices, &blasn);
582:         PetscBLASIntCast(numFIndices, &blask);
583:         PetscStackCallBLAS("BLASgemm",BLASgemm_("T","N",&blasn,&blasn,&blask,&one,elemMat,&blask,elemMat,&blask,&zero,elemMatSq,&blasn));
584:       }
585:       MatSetValues(mass, numCIndices, rowIDXs, numCIndices, rowIDXs, elemMatSq, ADD_VALUES);
586:       /* TODO Off-diagonal */
587:       PetscFree(cindices);
588:       DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);
589:     }
590:     DMSwarmRestoreField(dmc, DMSwarmPICField_coor, NULL, NULL, (void **) &coords);
591:   }
592:   PetscFree4(elemMat, elemMatSq, rowIDXs, xi);
593:   PetscFree(adj);
594:   DMSwarmSortRestoreAccess(dmc);
595:   PetscFree3(v0, J, invJ);
596:   MatAssemblyBegin(mass, MAT_FINAL_ASSEMBLY);
597:   MatAssemblyEnd(mass, MAT_FINAL_ASSEMBLY);
598:   return(0);
599: }

601: /*@C
602:   DMSwarmCreateMassMatrixSquare - Creates the block-diagonal of the square, M^T_p M_p, of the particle mass matrix M_p

604:   Collective on dmCoarse

606:   Input parameters:
607: + dmCoarse - a DMSwarm
608: - dmFine   - a DMPlex

610:   Output parameter:
611: . mass     - the square of the particle mass matrix

613:   Level: advanced

615:   Notes:
616:   We only compute the block diagonal since this provides a good preconditioner and is completely local. It would be possible in the
617:   future to compute the full normal equations.

619: .seealso: DMCreateMassMatrix()
620: @*/
621: PetscErrorCode DMSwarmCreateMassMatrixSquare(DM dmCoarse, DM dmFine, Mat *mass)
622: {
623:   PetscInt       n;
624:   void          *ctx;

628:   DMSwarmGetLocalSize(dmCoarse, &n);
629:   MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);
630:   MatSetSizes(*mass, n, n, PETSC_DETERMINE, PETSC_DETERMINE);
631:   MatSetType(*mass, dmCoarse->mattype);
632:   DMGetApplicationContext(dmFine, &ctx);

634:   DMSwarmComputeMassMatrixSquare_Private(dmCoarse, dmFine, *mass, PETSC_TRUE, ctx);
635:   MatViewFromOptions(*mass, NULL, "-mass_sq_mat_view");
636:   return(0);
637: }


640: /*@C
641:    DMSwarmCreateGlobalVectorFromField - Creates a Vec object sharing the array associated with a given field

643:    Collective on dm

645:    Input parameters:
646: +  dm - a DMSwarm
647: -  fieldname - the textual name given to a registered field

649:    Output parameter:
650: .  vec - the vector

652:    Level: beginner

654:    Notes:
655:    The vector must be returned using a matching call to DMSwarmDestroyGlobalVectorFromField().

657: .seealso: DMSwarmRegisterPetscDatatypeField(), DMSwarmDestroyGlobalVectorFromField()
658: @*/
659: PetscErrorCode DMSwarmCreateGlobalVectorFromField(DM dm,const char fieldname[],Vec *vec)
660: {
661:   MPI_Comm       comm = PetscObjectComm((PetscObject) dm);

665:   DMSwarmCreateVectorFromField_Private(dm, fieldname, comm, vec);
666:   return(0);
667: }

669: /*@C
670:    DMSwarmDestroyGlobalVectorFromField - Destroys the Vec object which share the array associated with a given field

672:    Collective on dm

674:    Input parameters:
675: +  dm - a DMSwarm
676: -  fieldname - the textual name given to a registered field

678:    Output parameter:
679: .  vec - the vector

681:    Level: beginner

683: .seealso: DMSwarmRegisterPetscDatatypeField(), DMSwarmCreateGlobalVectorFromField()
684: @*/
685: PetscErrorCode DMSwarmDestroyGlobalVectorFromField(DM dm,const char fieldname[],Vec *vec)
686: {

690:   DMSwarmDestroyVectorFromField_Private(dm, fieldname, vec);
691:   return(0);
692: }

694: /*@C
695:    DMSwarmCreateLocalVectorFromField - Creates a Vec object sharing the array associated with a given field

697:    Collective on dm

699:    Input parameters:
700: +  dm - a DMSwarm
701: -  fieldname - the textual name given to a registered field

703:    Output parameter:
704: .  vec - the vector

706:    Level: beginner

708:    Notes:
709:    The vector must be returned using a matching call to DMSwarmDestroyLocalVectorFromField().

711: .seealso: DMSwarmRegisterPetscDatatypeField(), DMSwarmDestroyLocalVectorFromField()
712: @*/
713: PetscErrorCode DMSwarmCreateLocalVectorFromField(DM dm,const char fieldname[],Vec *vec)
714: {
715:   MPI_Comm       comm = PETSC_COMM_SELF;

719:   DMSwarmCreateVectorFromField_Private(dm, fieldname, comm, vec);
720:   return(0);
721: }

723: /*@C
724:    DMSwarmDestroyLocalVectorFromField - Destroys the Vec object which share the array associated with a given field

726:    Collective on dm

728:    Input parameters:
729: +  dm - a DMSwarm
730: -  fieldname - the textual name given to a registered field

732:    Output parameter:
733: .  vec - the vector

735:    Level: beginner

737: .seealso: DMSwarmRegisterPetscDatatypeField(), DMSwarmCreateLocalVectorFromField()
738: @*/
739: PetscErrorCode DMSwarmDestroyLocalVectorFromField(DM dm,const char fieldname[],Vec *vec)
740: {

744:   DMSwarmDestroyVectorFromField_Private(dm, fieldname, vec);
745:   return(0);
746: }

748: /*
749: PetscErrorCode DMSwarmCreateGlobalVectorFromFields(DM dm,const PetscInt nf,const char *fieldnames[],Vec *vec)
750: {
751:   return(0);
752: }

754: PetscErrorCode DMSwarmRestoreGlobalVectorFromFields(DM dm,Vec *vec)
755: {
756:   return(0);
757: }
758: */

760: /*@C
761:    DMSwarmInitializeFieldRegister - Initiates the registration of fields to a DMSwarm

763:    Collective on dm

765:    Input parameter:
766: .  dm - a DMSwarm

768:    Level: beginner

770:    Notes:
771:    After all fields have been registered, you must call DMSwarmFinalizeFieldRegister().

773: .seealso: DMSwarmFinalizeFieldRegister(), DMSwarmRegisterPetscDatatypeField(),
774:  DMSwarmRegisterUserStructField(), DMSwarmRegisterUserDatatypeField()
775: @*/
776: PetscErrorCode DMSwarmInitializeFieldRegister(DM dm)
777: {
778:   DM_Swarm      *swarm = (DM_Swarm *) dm->data;

782:   if (!swarm->field_registration_initialized) {
783:     swarm->field_registration_initialized = PETSC_TRUE;
784:     DMSwarmRegisterPetscDatatypeField(dm,DMSwarmField_pid,1,PETSC_INT64); /* unique identifer */
785:     DMSwarmRegisterPetscDatatypeField(dm,DMSwarmField_rank,1,PETSC_INT); /* used for communication */
786:   }
787:   return(0);
788: }

790: /*@
791:    DMSwarmFinalizeFieldRegister - Finalizes the registration of fields to a DMSwarm

793:    Collective on dm

795:    Input parameter:
796: .  dm - a DMSwarm

798:    Level: beginner

800:    Notes:
801:    After DMSwarmFinalizeFieldRegister() has been called, no new fields can be defined on the DMSwarm.

803: .seealso: DMSwarmInitializeFieldRegister(), DMSwarmRegisterPetscDatatypeField(),
804:  DMSwarmRegisterUserStructField(), DMSwarmRegisterUserDatatypeField()
805: @*/
806: PetscErrorCode DMSwarmFinalizeFieldRegister(DM dm)
807: {
808:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

812:   if (!swarm->field_registration_finalized) {
813:     DMSwarmDataBucketFinalize(swarm->db);
814:   }
815:   swarm->field_registration_finalized = PETSC_TRUE;
816:   return(0);
817: }

819: /*@
820:    DMSwarmSetLocalSizes - Sets the length of all registered fields on the DMSwarm

822:    Not collective

824:    Input parameters:
825: +  dm - a DMSwarm
826: .  nlocal - the length of each registered field
827: -  buffer - the length of the buffer used to efficient dynamic re-sizing

829:    Level: beginner

831: .seealso: DMSwarmGetLocalSize()
832: @*/
833: PetscErrorCode DMSwarmSetLocalSizes(DM dm,PetscInt nlocal,PetscInt buffer)
834: {
835:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

839:   PetscLogEventBegin(DMSWARM_SetSizes,0,0,0,0);
840:   DMSwarmDataBucketSetSizes(swarm->db,nlocal,buffer);
841:   PetscLogEventEnd(DMSWARM_SetSizes,0,0,0,0);
842:   return(0);
843: }

845: /*@
846:    DMSwarmSetCellDM - Attachs a DM to a DMSwarm

848:    Collective on dm

850:    Input parameters:
851: +  dm - a DMSwarm
852: -  dmcell - the DM to attach to the DMSwarm

854:    Level: beginner

856:    Notes:
857:    The attached DM (dmcell) will be queried for point location and
858:    neighbor MPI-rank information if DMSwarmMigrate() is called.

860: .seealso: DMSwarmSetType(), DMSwarmGetCellDM(), DMSwarmMigrate()
861: @*/
862: PetscErrorCode DMSwarmSetCellDM(DM dm,DM dmcell)
863: {
864:   DM_Swarm *swarm = (DM_Swarm*)dm->data;

867:   swarm->dmcell = dmcell;
868:   return(0);
869: }

871: /*@
872:    DMSwarmGetCellDM - Fetches the attached cell DM

874:    Collective on dm

876:    Input parameter:
877: .  dm - a DMSwarm

879:    Output parameter:
880: .  dmcell - the DM which was attached to the DMSwarm

882:    Level: beginner

884: .seealso: DMSwarmSetCellDM()
885: @*/
886: PetscErrorCode DMSwarmGetCellDM(DM dm,DM *dmcell)
887: {
888:   DM_Swarm *swarm = (DM_Swarm*)dm->data;

891:   *dmcell = swarm->dmcell;
892:   return(0);
893: }

895: /*@
896:    DMSwarmGetLocalSize - Retrives the local length of fields registered

898:    Not collective

900:    Input parameter:
901: .  dm - a DMSwarm

903:    Output parameter:
904: .  nlocal - the length of each registered field

906:    Level: beginner

908: .seealso: DMSwarmGetSize(), DMSwarmSetLocalSizes()
909: @*/
910: PetscErrorCode DMSwarmGetLocalSize(DM dm,PetscInt *nlocal)
911: {
912:   DM_Swarm *swarm = (DM_Swarm*)dm->data;

916:   if (nlocal) {DMSwarmDataBucketGetSizes(swarm->db,nlocal,NULL,NULL);}
917:   return(0);
918: }

920: /*@
921:    DMSwarmGetSize - Retrives the total length of fields registered

923:    Collective on dm

925:    Input parameter:
926: .  dm - a DMSwarm

928:    Output parameter:
929: .  n - the total length of each registered field

931:    Level: beginner

933:    Note:
934:    This calls MPI_Allreduce upon each call (inefficient but safe)

936: .seealso: DMSwarmGetLocalSize(), DMSwarmSetLocalSizes()
937: @*/
938: PetscErrorCode DMSwarmGetSize(DM dm,PetscInt *n)
939: {
940:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;
942:   PetscInt       nlocal,ng;

945:   DMSwarmDataBucketGetSizes(swarm->db,&nlocal,NULL,NULL);
946:   MPI_Allreduce(&nlocal,&ng,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)dm));
947:   if (n) { *n = ng; }
948:   return(0);
949: }

951: /*@C
952:    DMSwarmRegisterPetscDatatypeField - Register a field to a DMSwarm with a native PETSc data type

954:    Collective on dm

956:    Input parameters:
957: +  dm - a DMSwarm
958: .  fieldname - the textual name to identify this field
959: .  blocksize - the number of each data type
960: -  type - a valid PETSc data type (PETSC_CHAR, PETSC_SHORT, PETSC_INT, PETSC_FLOAT, PETSC_REAL, PETSC_LONG)

962:    Level: beginner

964:    Notes:
965:    The textual name for each registered field must be unique.

967: .seealso: DMSwarmRegisterUserStructField(), DMSwarmRegisterUserDatatypeField()
968: @*/
969: PetscErrorCode DMSwarmRegisterPetscDatatypeField(DM dm,const char fieldname[],PetscInt blocksize,PetscDataType type)
970: {
972:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;
973:   size_t         size;

976:   if (!swarm->field_registration_initialized) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"Must call DMSwarmInitializeFieldRegister() first");
977:   if (swarm->field_registration_finalized) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"Cannot register additional fields after calling DMSwarmFinalizeFieldRegister() first");

979:   if (type == PETSC_OBJECT) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Valid for {char,short,int,long,float,double}");
980:   if (type == PETSC_FUNCTION) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Valid for {char,short,int,long,float,double}");
981:   if (type == PETSC_STRING) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Valid for {char,short,int,long,float,double}");
982:   if (type == PETSC_STRUCT) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Valid for {char,short,int,long,float,double}");
983:   if (type == PETSC_DATATYPE_UNKNOWN) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Valid for {char,short,int,long,float,double}");

985:   PetscDataTypeGetSize(type, &size);
986:   /* Load a specific data type into data bucket, specifying textual name and its size in bytes */
987:   DMSwarmDataBucketRegisterField(swarm->db,"DMSwarmRegisterPetscDatatypeField",fieldname,blocksize*size,NULL);
988:   {
989:     DMSwarmDataField gfield;

991:     DMSwarmDataBucketGetDMSwarmDataFieldByName(swarm->db,fieldname,&gfield);
992:     DMSwarmDataFieldSetBlockSize(gfield,blocksize);
993:   }
994:   swarm->db->field[swarm->db->nfields-1]->petsc_type = type;
995:   return(0);
996: }

998: /*@C
999:    DMSwarmRegisterUserStructField - Register a user defined struct to a DMSwarm

1001:    Collective on dm

1003:    Input parameters:
1004: +  dm - a DMSwarm
1005: .  fieldname - the textual name to identify this field
1006: -  size - the size in bytes of the user struct of each data type

1008:    Level: beginner

1010:    Notes:
1011:    The textual name for each registered field must be unique.

1013: .seealso: DMSwarmRegisterPetscDatatypeField(), DMSwarmRegisterUserDatatypeField()
1014: @*/
1015: PetscErrorCode DMSwarmRegisterUserStructField(DM dm,const char fieldname[],size_t size)
1016: {
1018:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

1021:   DMSwarmDataBucketRegisterField(swarm->db,"DMSwarmRegisterUserStructField",fieldname,size,NULL);
1022:   swarm->db->field[swarm->db->nfields-1]->petsc_type = PETSC_STRUCT ;
1023:   return(0);
1024: }

1026: /*@C
1027:    DMSwarmRegisterUserDatatypeField - Register a user defined data type to a DMSwarm

1029:    Collective on dm

1031:    Input parameters:
1032: +  dm - a DMSwarm
1033: .  fieldname - the textual name to identify this field
1034: .  size - the size in bytes of the user data type
1035: -  blocksize - the number of each data type

1037:    Level: beginner

1039:    Notes:
1040:    The textual name for each registered field must be unique.

1042: .seealso: DMSwarmRegisterPetscDatatypeField(), DMSwarmRegisterUserStructField(), DMSwarmRegisterUserDatatypeField()
1043: @*/
1044: PetscErrorCode DMSwarmRegisterUserDatatypeField(DM dm,const char fieldname[],size_t size,PetscInt blocksize)
1045: {
1046:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

1050:   DMSwarmDataBucketRegisterField(swarm->db,"DMSwarmRegisterUserDatatypeField",fieldname,blocksize*size,NULL);
1051:   {
1052:     DMSwarmDataField gfield;

1054:     DMSwarmDataBucketGetDMSwarmDataFieldByName(swarm->db,fieldname,&gfield);
1055:     DMSwarmDataFieldSetBlockSize(gfield,blocksize);
1056:   }
1057:   swarm->db->field[swarm->db->nfields-1]->petsc_type = PETSC_DATATYPE_UNKNOWN;
1058:   return(0);
1059: }

1061: /*@C
1062:    DMSwarmGetField - Get access to the underlying array storing all entries associated with a registered field

1064:    Not collective

1066:    Input parameters:
1067: +  dm - a DMSwarm
1068: -  fieldname - the textual name to identify this field

1070:    Output parameters:
1071: +  blocksize - the number of each data type
1072: .  type - the data type
1073: -  data - pointer to raw array

1075:    Level: beginner

1077:    Notes:
1078:    The array must be returned using a matching call to DMSwarmRestoreField().

1080: .seealso: DMSwarmRestoreField()
1081: @*/
1082: PetscErrorCode DMSwarmGetField(DM dm,const char fieldname[],PetscInt *blocksize,PetscDataType *type,void **data)
1083: {
1084:   DM_Swarm         *swarm = (DM_Swarm*)dm->data;
1085:   DMSwarmDataField gfield;

1089:   if (!swarm->issetup) { DMSetUp(dm); }
1090:   DMSwarmDataBucketGetDMSwarmDataFieldByName(swarm->db,fieldname,&gfield);
1091:   DMSwarmDataFieldGetAccess(gfield);
1092:   DMSwarmDataFieldGetEntries(gfield,data);
1093:   if (blocksize) {*blocksize = gfield->bs; }
1094:   if (type) { *type = gfield->petsc_type; }
1095:   return(0);
1096: }

1098: /*@C
1099:    DMSwarmRestoreField - Restore access to the underlying array storing all entries associated with a registered field

1101:    Not collective

1103:    Input parameters:
1104: +  dm - a DMSwarm
1105: -  fieldname - the textual name to identify this field

1107:    Output parameters:
1108: +  blocksize - the number of each data type
1109: .  type - the data type
1110: -  data - pointer to raw array

1112:    Level: beginner

1114:    Notes:
1115:    The user must call DMSwarmGetField() prior to calling DMSwarmRestoreField().

1117: .seealso: DMSwarmGetField()
1118: @*/
1119: PetscErrorCode DMSwarmRestoreField(DM dm,const char fieldname[],PetscInt *blocksize,PetscDataType *type,void **data)
1120: {
1121:   DM_Swarm         *swarm = (DM_Swarm*)dm->data;
1122:   DMSwarmDataField gfield;

1126:   DMSwarmDataBucketGetDMSwarmDataFieldByName(swarm->db,fieldname,&gfield);
1127:   DMSwarmDataFieldRestoreAccess(gfield);
1128:   if (data) *data = NULL;
1129:   return(0);
1130: }

1132: /*@
1133:    DMSwarmAddPoint - Add space for one new point in the DMSwarm

1135:    Not collective

1137:    Input parameter:
1138: .  dm - a DMSwarm

1140:    Level: beginner

1142:    Notes:
1143:    The new point will have all fields initialized to zero.

1145: .seealso: DMSwarmAddNPoints()
1146: @*/
1147: PetscErrorCode DMSwarmAddPoint(DM dm)
1148: {
1149:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

1153:   if (!swarm->issetup) {DMSetUp(dm);}
1154:   PetscLogEventBegin(DMSWARM_AddPoints,0,0,0,0);
1155:   DMSwarmDataBucketAddPoint(swarm->db);
1156:   PetscLogEventEnd(DMSWARM_AddPoints,0,0,0,0);
1157:   return(0);
1158: }

1160: /*@
1161:    DMSwarmAddNPoints - Add space for a number of new points in the DMSwarm

1163:    Not collective

1165:    Input parameters:
1166: +  dm - a DMSwarm
1167: -  npoints - the number of new points to add

1169:    Level: beginner

1171:    Notes:
1172:    The new point will have all fields initialized to zero.

1174: .seealso: DMSwarmAddPoint()
1175: @*/
1176: PetscErrorCode DMSwarmAddNPoints(DM dm,PetscInt npoints)
1177: {
1178:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;
1180:   PetscInt       nlocal;

1183:   PetscLogEventBegin(DMSWARM_AddPoints,0,0,0,0);
1184:   DMSwarmDataBucketGetSizes(swarm->db,&nlocal,NULL,NULL);
1185:   nlocal = nlocal + npoints;
1186:   DMSwarmDataBucketSetSizes(swarm->db,nlocal,DMSWARM_DATA_BUCKET_BUFFER_DEFAULT);
1187:   PetscLogEventEnd(DMSWARM_AddPoints,0,0,0,0);
1188:   return(0);
1189: }

1191: /*@
1192:    DMSwarmRemovePoint - Remove the last point from the DMSwarm

1194:    Not collective

1196:    Input parameter:
1197: .  dm - a DMSwarm

1199:    Level: beginner

1201: .seealso: DMSwarmRemovePointAtIndex()
1202: @*/
1203: PetscErrorCode DMSwarmRemovePoint(DM dm)
1204: {
1205:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

1209:   PetscLogEventBegin(DMSWARM_RemovePoints,0,0,0,0);
1210:   DMSwarmDataBucketRemovePoint(swarm->db);
1211:   PetscLogEventEnd(DMSWARM_RemovePoints,0,0,0,0);
1212:   return(0);
1213: }

1215: /*@
1216:    DMSwarmRemovePointAtIndex - Removes a specific point from the DMSwarm

1218:    Not collective

1220:    Input parameters:
1221: +  dm - a DMSwarm
1222: -  idx - index of point to remove

1224:    Level: beginner

1226: .seealso: DMSwarmRemovePoint()
1227: @*/
1228: PetscErrorCode DMSwarmRemovePointAtIndex(DM dm,PetscInt idx)
1229: {
1230:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

1234:   PetscLogEventBegin(DMSWARM_RemovePoints,0,0,0,0);
1235:   DMSwarmDataBucketRemovePointAtIndex(swarm->db,idx);
1236:   PetscLogEventEnd(DMSWARM_RemovePoints,0,0,0,0);
1237:   return(0);
1238: }

1240: /*@
1241:    DMSwarmCopyPoint - Copy point pj to point pi in the DMSwarm

1243:    Not collective

1245:    Input parameters:
1246: +  dm - a DMSwarm
1247: .  pi - the index of the point to copy
1248: -  pj - the point index where the copy should be located

1250:  Level: beginner

1252: .seealso: DMSwarmRemovePoint()
1253: @*/
1254: PetscErrorCode DMSwarmCopyPoint(DM dm,PetscInt pi,PetscInt pj)
1255: {
1256:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

1260:   if (!swarm->issetup) {DMSetUp(dm);}
1261:   DMSwarmDataBucketCopyPoint(swarm->db,pi,swarm->db,pj);
1262:   return(0);
1263: }

1265: PetscErrorCode DMSwarmMigrate_Basic(DM dm,PetscBool remove_sent_points)
1266: {

1270:   DMSwarmMigrate_Push_Basic(dm,remove_sent_points);
1271:   return(0);
1272: }

1274: /*@
1275:    DMSwarmMigrate - Relocates points defined in the DMSwarm to other MPI-ranks

1277:    Collective on dm

1279:    Input parameters:
1280: +  dm - the DMSwarm
1281: -  remove_sent_points - flag indicating if sent points should be removed from the current MPI-rank

1283:    Notes:
1284:    The DM will be modified to accomodate received points.
1285:    If remove_sent_points = PETSC_TRUE, any points that were sent will be removed from the DM.
1286:    Different styles of migration are supported. See DMSwarmSetMigrateType().

1288:    Level: advanced

1290: .seealso: DMSwarmSetMigrateType()
1291: @*/
1292: PetscErrorCode DMSwarmMigrate(DM dm,PetscBool remove_sent_points)
1293: {
1294:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

1298:   PetscLogEventBegin(DMSWARM_Migrate,0,0,0,0);
1299:   switch (swarm->migrate_type) {
1300:     case DMSWARM_MIGRATE_BASIC:
1301:       DMSwarmMigrate_Basic(dm,remove_sent_points);
1302:       break;
1303:     case DMSWARM_MIGRATE_DMCELLNSCATTER:
1304:       DMSwarmMigrate_CellDMScatter(dm,remove_sent_points);
1305:       break;
1306:     case DMSWARM_MIGRATE_DMCELLEXACT:
1307:       SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"DMSWARM_MIGRATE_DMCELLEXACT not implemented");
1308:       break;
1309:     case DMSWARM_MIGRATE_USER:
1310:       SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"DMSWARM_MIGRATE_USER not implemented");
1311:       break;
1312:     default:
1313:       SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"DMSWARM_MIGRATE type unknown");
1314:       break;
1315:   }
1316:   PetscLogEventEnd(DMSWARM_Migrate,0,0,0,0);
1317:   return(0);
1318: }

1320: PetscErrorCode DMSwarmMigrate_GlobalToLocal_Basic(DM dm,PetscInt *globalsize);

1322: /*
1323:  DMSwarmCollectViewCreate

1325:  * Applies a collection method and gathers point neighbour points into dm

1327:  Notes:
1328:  Users should call DMSwarmCollectViewDestroy() after
1329:  they have finished computations associated with the collected points
1330: */

1332: /*@
1333:    DMSwarmCollectViewCreate - Applies a collection method and gathers points
1334:    in neighbour MPI-ranks into the DMSwarm

1336:    Collective on dm

1338:    Input parameter:
1339: .  dm - the DMSwarm

1341:    Notes:
1342:    Users should call DMSwarmCollectViewDestroy() after
1343:    they have finished computations associated with the collected points
1344:    Different collect methods are supported. See DMSwarmSetCollectType().

1346:    Level: advanced

1348: .seealso: DMSwarmCollectViewDestroy(), DMSwarmSetCollectType()
1349: @*/
1350: PetscErrorCode DMSwarmCollectViewCreate(DM dm)
1351: {
1353:   DM_Swarm *swarm = (DM_Swarm*)dm->data;
1354:   PetscInt ng;

1357:   if (swarm->collect_view_active) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"CollectView currently active");
1358:   DMSwarmGetLocalSize(dm,&ng);
1359:   switch (swarm->collect_type) {

1361:     case DMSWARM_COLLECT_BASIC:
1362:       DMSwarmMigrate_GlobalToLocal_Basic(dm,&ng);
1363:       break;
1364:     case DMSWARM_COLLECT_DMDABOUNDINGBOX:
1365:       SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"DMSWARM_COLLECT_DMDABOUNDINGBOX not implemented");
1366:       break;
1367:     case DMSWARM_COLLECT_GENERAL:
1368:       SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"DMSWARM_COLLECT_GENERAL not implemented");
1369:       break;
1370:     default:
1371:       SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"DMSWARM_COLLECT type unknown");
1372:       break;
1373:   }
1374:   swarm->collect_view_active = PETSC_TRUE;
1375:   swarm->collect_view_reset_nlocal = ng;
1376:   return(0);
1377: }

1379: /*@
1380:    DMSwarmCollectViewDestroy - Resets the DMSwarm to the size prior to calling DMSwarmCollectViewCreate()

1382:    Collective on dm

1384:    Input parameters:
1385: .  dm - the DMSwarm

1387:    Notes:
1388:    Users should call DMSwarmCollectViewCreate() before this function is called.

1390:    Level: advanced

1392: .seealso: DMSwarmCollectViewCreate(), DMSwarmSetCollectType()
1393: @*/
1394: PetscErrorCode DMSwarmCollectViewDestroy(DM dm)
1395: {
1397:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

1400:   if (!swarm->collect_view_active) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"CollectView is currently not active");
1401:   DMSwarmSetLocalSizes(dm,swarm->collect_view_reset_nlocal,-1);
1402:   swarm->collect_view_active = PETSC_FALSE;
1403:   return(0);
1404: }

1406: PetscErrorCode DMSwarmSetUpPIC(DM dm)
1407: {
1408:   PetscInt       dim;

1412:   DMGetDimension(dm,&dim);
1413:   if (dim < 1) SETERRQ1(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"Dimension must be 1,2,3 - found %D",dim);
1414:   if (dim > 3) SETERRQ1(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"Dimension must be 1,2,3 - found %D",dim);
1415:   DMSwarmRegisterPetscDatatypeField(dm,DMSwarmPICField_coor,dim,PETSC_DOUBLE);
1416:   DMSwarmRegisterPetscDatatypeField(dm,DMSwarmPICField_cellid,1,PETSC_INT);
1417:   return(0);
1418: }

1420: /*@
1421:    DMSwarmSetType - Set particular flavor of DMSwarm

1423:    Collective on dm

1425:    Input parameters:
1426: +  dm - the DMSwarm
1427: -  stype - the DMSwarm type (e.g. DMSWARM_PIC)

1429:    Level: advanced

1431: .seealso: DMSwarmSetMigrateType(), DMSwarmSetCollectType()
1432: @*/
1433: PetscErrorCode DMSwarmSetType(DM dm,DMSwarmType stype)
1434: {
1435:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

1439:   swarm->swarm_type = stype;
1440:   if (swarm->swarm_type == DMSWARM_PIC) {
1441:     DMSwarmSetUpPIC(dm);
1442:   }
1443:   return(0);
1444: }

1446: PetscErrorCode DMSetup_Swarm(DM dm)
1447: {
1448:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;
1450:   PetscMPIInt    rank;
1451:   PetscInt       p,npoints,*rankval;

1454:   if (swarm->issetup) return(0);

1456:   swarm->issetup = PETSC_TRUE;

1458:   if (swarm->swarm_type == DMSWARM_PIC) {
1459:     /* check dmcell exists */
1460:     if (!swarm->dmcell) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"DMSWARM_PIC requires you call DMSwarmSetCellDM");

1462:     if (swarm->dmcell->ops->locatepointssubdomain) {
1463:       /* check methods exists for exact ownership identificiation */
1464:       PetscInfo(dm, "DMSWARM_PIC: Using method CellDM->ops->LocatePointsSubdomain\n");
1465:       swarm->migrate_type = DMSWARM_MIGRATE_DMCELLEXACT;
1466:     } else {
1467:       /* check methods exist for point location AND rank neighbor identification */
1468:       if (swarm->dmcell->ops->locatepoints) {
1469:         PetscInfo(dm, "DMSWARM_PIC: Using method CellDM->LocatePoints\n");
1470:       } else SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"DMSWARM_PIC requires the method CellDM->ops->locatepoints be defined");

1472:       if (swarm->dmcell->ops->getneighbors) {
1473:         PetscInfo(dm, "DMSWARM_PIC: Using method CellDM->GetNeigbors\n");
1474:       } else SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"DMSWARM_PIC requires the method CellDM->ops->getneighbors be defined");

1476:       swarm->migrate_type = DMSWARM_MIGRATE_DMCELLNSCATTER;
1477:     }
1478:   }

1480:   DMSwarmFinalizeFieldRegister(dm);

1482:   /* check some fields were registered */
1483:   if (swarm->db->nfields <= 2) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"At least one field user must be registered via DMSwarmRegisterXXX()");

1485:   /* check local sizes were set */
1486:   if (swarm->db->L == -1) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"Local sizes must be set via DMSwarmSetLocalSizes()");

1488:   /* initialize values in pid and rank placeholders */
1489:   /* TODO: [pid - use MPI_Scan] */
1490:   MPI_Comm_rank(PetscObjectComm((PetscObject)dm),&rank);
1491:   DMSwarmDataBucketGetSizes(swarm->db,&npoints,NULL,NULL);
1492:   DMSwarmGetField(dm,DMSwarmField_rank,NULL,NULL,(void**)&rankval);
1493:   for (p=0; p<npoints; p++) {
1494:     rankval[p] = (PetscInt)rank;
1495:   }
1496:   DMSwarmRestoreField(dm,DMSwarmField_rank,NULL,NULL,(void**)&rankval);
1497:   return(0);
1498: }

1500: extern PetscErrorCode DMSwarmSortDestroy(DMSwarmSort *_ctx);

1502: PetscErrorCode DMDestroy_Swarm(DM dm)
1503: {
1504:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;

1508:   DMSwarmDataBucketDestroy(&swarm->db);
1509:   if (swarm->sort_context) {
1510:     DMSwarmSortDestroy(&swarm->sort_context);
1511:   }
1512:   PetscFree(swarm);
1513:   return(0);
1514: }

1516: PetscErrorCode DMSwarmView_Draw(DM dm, PetscViewer viewer)
1517: {
1518:   DM             cdm;
1519:   PetscDraw      draw;
1520:   PetscReal     *coords, oldPause;
1521:   PetscInt       Np, p, bs;

1525:   PetscViewerDrawGetDraw(viewer, 0, &draw);
1526:   DMSwarmGetCellDM(dm, &cdm);
1527:   PetscDrawGetPause(draw, &oldPause);
1528:   PetscDrawSetPause(draw, 0.0);
1529:   DMView(cdm, viewer);
1530:   PetscDrawSetPause(draw, oldPause);

1532:   DMSwarmGetLocalSize(dm, &Np);
1533:   DMSwarmGetField(dm, DMSwarmPICField_coor, &bs, NULL, (void **) &coords);
1534:   for (p = 0; p < Np; ++p) {
1535:     const PetscInt i = p*bs;

1537:     PetscDrawEllipse(draw, coords[i], coords[i+1], 0.01, 0.01, PETSC_DRAW_BLUE);
1538:   }
1539:   DMSwarmRestoreField(dm, DMSwarmPICField_coor, &bs, NULL, (void **) &coords);
1540:   PetscDrawFlush(draw);
1541:   PetscDrawPause(draw);
1542:   PetscDrawSave(draw);
1543:   return(0);
1544: }

1546: PetscErrorCode DMView_Swarm(DM dm, PetscViewer viewer)
1547: {
1548:   DM_Swarm       *swarm = (DM_Swarm*)dm->data;
1549:   PetscBool      iascii,ibinary,ishdf5,isvtk,isdraw;

1555:   PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERASCII, &iascii);
1556:   PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERBINARY,&ibinary);
1557:   PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERVTK,   &isvtk);
1558:   PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5,  &ishdf5);
1559:   PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERDRAW,  &isdraw);
1560:   if (iascii) {
1561:     DMSwarmDataBucketView(PetscObjectComm((PetscObject)dm),swarm->db,NULL,DATABUCKET_VIEW_STDOUT);
1562:   } else if (ibinary) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"NO Binary support");
1563: #if defined(PETSC_HAVE_HDF5)
1564:   else if (ishdf5) {
1565:     DMSwarmView_HDF5(dm, viewer);
1566:   }
1567: #else
1568:   else if (ishdf5) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"HDF5 not supported. Please reconfigure using --download-hdf5");
1569: #endif
1570:   else if (isvtk) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"NO VTK support");
1571:   else if (isdraw) {
1572:     DMSwarmView_Draw(dm, viewer);
1573:   }
1574:   return(0);
1575: }

1577: /*MC

1579:  DMSWARM = "swarm" - A DM object used to represent arrays of data (fields) of arbitrary data type.
1580:  This implementation was designed for particle methods in which the underlying
1581:  data required to be represented is both (i) dynamic in length, (ii) and of arbitrary data type.

1583:  User data can be represented by DMSwarm through a registering "fields".
1584:  To register a field, the user must provide;
1585:  (a) a unique name;
1586:  (b) the data type (or size in bytes);
1587:  (c) the block size of the data.

1589:  For example, suppose the application requires a unique id, energy, momentum and density to be stored
1590:  on a set of particles. Then the following code could be used

1592: $    DMSwarmInitializeFieldRegister(dm)
1593: $    DMSwarmRegisterPetscDatatypeField(dm,"uid",1,PETSC_LONG);
1594: $    DMSwarmRegisterPetscDatatypeField(dm,"energy",1,PETSC_REAL);
1595: $    DMSwarmRegisterPetscDatatypeField(dm,"momentum",3,PETSC_REAL);
1596: $    DMSwarmRegisterPetscDatatypeField(dm,"density",1,PETSC_FLOAT);
1597: $    DMSwarmFinalizeFieldRegister(dm)

1599:  The fields represented by DMSwarm are dynamic and can be re-sized at any time.
1600:  The only restriction imposed by DMSwarm is that all fields contain the same number of points.

1602:  To support particle methods, "migration" techniques are provided. These methods migrate data
1603:  between MPI-ranks.

1605:  DMSwarm supports the methods DMCreateGlobalVector() and DMCreateLocalVector().
1606:  As a DMSwarm may internally define and store values of different data types,
1607:  before calling DMCreateGlobalVector() or DMCreateLocalVector(), the user must inform DMSwarm which
1608:  fields should be used to define a Vec object via
1609:    DMSwarmVectorDefineField()
1610:  The specified field can be changed at any time - thereby permitting vectors
1611:  compatible with different fields to be created.

1613:  A dual representation of fields in the DMSwarm and a Vec object is permitted via
1614:    DMSwarmCreateGlobalVectorFromField()
1615:  Here the data defining the field in the DMSwarm is shared with a Vec.
1616:  This is inherently unsafe if you alter the size of the field at any time between
1617:  calls to DMSwarmCreateGlobalVectorFromField() and DMSwarmDestroyGlobalVectorFromField().
1618:  If the local size of the DMSwarm does not match the local size of the global vector
1619:  when DMSwarmDestroyGlobalVectorFromField() is called, an error is thrown.

1621:  Additional high-level support is provided for Particle-In-Cell methods.
1622:  Please refer to the man page for DMSwarmSetType().

1624:  Level: beginner

1626: .seealso: DMType, DMCreate(), DMSetType()
1627: M*/
1628: PETSC_EXTERN PetscErrorCode DMCreate_Swarm(DM dm)
1629: {
1630:   DM_Swarm      *swarm;

1635:   PetscNewLog(dm,&swarm);
1636:   dm->data = swarm;

1638:   DMSwarmDataBucketCreate(&swarm->db);
1639:   DMSwarmInitializeFieldRegister(dm);

1641:   swarm->vec_field_set = PETSC_FALSE;
1642:   swarm->issetup = PETSC_FALSE;
1643:   swarm->swarm_type = DMSWARM_BASIC;
1644:   swarm->migrate_type = DMSWARM_MIGRATE_BASIC;
1645:   swarm->collect_type = DMSWARM_COLLECT_BASIC;
1646:   swarm->migrate_error_on_missing_point = PETSC_FALSE;

1648:   swarm->dmcell = NULL;
1649:   swarm->collect_view_active = PETSC_FALSE;
1650:   swarm->collect_view_reset_nlocal = -1;

1652:   dm->dim  = 0;
1653:   dm->ops->view                            = DMView_Swarm;
1654:   dm->ops->load                            = NULL;
1655:   dm->ops->setfromoptions                  = NULL;
1656:   dm->ops->clone                           = NULL;
1657:   dm->ops->setup                           = DMSetup_Swarm;
1658:   dm->ops->createlocalsection              = NULL;
1659:   dm->ops->createdefaultconstraints        = NULL;
1660:   dm->ops->createglobalvector              = DMCreateGlobalVector_Swarm;
1661:   dm->ops->createlocalvector               = DMCreateLocalVector_Swarm;
1662:   dm->ops->getlocaltoglobalmapping         = NULL;
1663:   dm->ops->createfieldis                   = NULL;
1664:   dm->ops->createcoordinatedm              = NULL;
1665:   dm->ops->getcoloring                     = NULL;
1666:   dm->ops->creatematrix                    = NULL;
1667:   dm->ops->createinterpolation             = NULL;
1668:   dm->ops->createinjection                 = NULL;
1669:   dm->ops->createmassmatrix                = DMCreateMassMatrix_Swarm;
1670:   dm->ops->refine                          = NULL;
1671:   dm->ops->coarsen                         = NULL;
1672:   dm->ops->refinehierarchy                 = NULL;
1673:   dm->ops->coarsenhierarchy                = NULL;
1674:   dm->ops->globaltolocalbegin              = NULL;
1675:   dm->ops->globaltolocalend                = NULL;
1676:   dm->ops->localtoglobalbegin              = NULL;
1677:   dm->ops->localtoglobalend                = NULL;
1678:   dm->ops->destroy                         = DMDestroy_Swarm;
1679:   dm->ops->createsubdm                     = NULL;
1680:   dm->ops->getdimpoints                    = NULL;
1681:   dm->ops->locatepoints                    = NULL;
1682:   return(0);
1683: }