Actual source code: veccuda.c

petsc-master 2020-09-19
Report Typos and Errors
  1: /*
  2:  Implementation of the sequential cuda vectors.

  4:  This file contains the code that can be compiled with a C
  5:  compiler.  The companion file veccuda2.cu contains the code that
  6:  must be compiled with nvcc or a C++ compiler.
  7:  */

  9: #define PETSC_SKIP_SPINLOCK

 11: #include <petscconf.h>
 12: #include <petsc/private/vecimpl.h>
 13: #include <../src/vec/vec/impls/dvecimpl.h>
 14: #include <petsc/private/cudavecimpl.h>

 16: PetscErrorCode VecCUDAGetArrays_Private(Vec v,const PetscScalar** x,const PetscScalar** x_d,PetscOffloadMask* flg)
 17: {
 20:   if (x) {
 21:     Vec_Seq *h = (Vec_Seq*)v->data;

 23:     *x = h->array;
 24:   }
 25:   if (x_d) {
 26:     Vec_CUDA *d = (Vec_CUDA*)v->spptr;

 28:     *x_d = d ? d->GPUarray : NULL;
 29:   }
 30:   if (flg) *flg = v->offloadmask;
 31:   return(0);
 32: }

 34: /*
 35:     Allocates space for the vector array on the Host if it does not exist.
 36:     Does NOT change the PetscCUDAFlag for the vector
 37:     Does NOT zero the CUDA array
 38:  */
 39: PetscErrorCode VecCUDAAllocateCheckHost(Vec v)
 40: {
 42:   PetscScalar    *array;
 43:   Vec_Seq        *s = (Vec_Seq*)v->data;
 44:   PetscInt       n = v->map->n;

 47:   if (!s) {
 48:     PetscNewLog((PetscObject)v,&s);
 49:     v->data = s;
 50:   }
 51:   if (!s->array) {
 52:     if (n*sizeof(PetscScalar) > v->minimum_bytes_pinned_memory) {
 53:       PetscMallocSetCUDAHost();
 54:       v->pinned_memory = PETSC_TRUE;
 55:     }
 56:     PetscMalloc1(n,&array);
 57:     PetscLogObjectMemory((PetscObject)v,n*sizeof(PetscScalar));
 58:     s->array           = array;
 59:     s->array_allocated = array;
 60:     if (n*sizeof(PetscScalar) > v->minimum_bytes_pinned_memory) {
 61:       PetscMallocResetCUDAHost();
 62:     }
 63:     if (v->offloadmask == PETSC_OFFLOAD_UNALLOCATED) {
 64:       v->offloadmask = PETSC_OFFLOAD_CPU;
 65:     }
 66:   }
 67:   return(0);
 68: }

 70: PetscErrorCode VecCopy_SeqCUDA_Private(Vec xin,Vec yin)
 71: {
 72:   PetscScalar       *ya;
 73:   const PetscScalar *xa;
 74:   PetscErrorCode    ierr;

 77:   VecCUDAAllocateCheckHost(xin);
 78:   VecCUDAAllocateCheckHost(yin);
 79:   if (xin != yin) {
 80:     VecGetArrayRead(xin,&xa);
 81:     VecGetArray(yin,&ya);
 82:     PetscArraycpy(ya,xa,xin->map->n);
 83:     VecRestoreArrayRead(xin,&xa);
 84:     VecRestoreArray(yin,&ya);
 85:   }
 86:   return(0);
 87: }

 89: PetscErrorCode VecSetRandom_SeqCUDA_Private(Vec xin,PetscRandom r)
 90: {
 92:   PetscInt       n = xin->map->n,i;
 93:   PetscScalar    *xx;

 96:   VecGetArray(xin,&xx);
 97:   for (i=0; i<n; i++) { PetscRandomGetValue(r,&xx[i]); }
 98:   VecRestoreArray(xin,&xx);
 99:   return(0);
100: }

102: PetscErrorCode VecDestroy_SeqCUDA_Private(Vec v)
103: {
104:   Vec_Seq        *vs = (Vec_Seq*)v->data;

108:   PetscObjectSAWsViewOff(v);
109: #if defined(PETSC_USE_LOG)
110:   PetscLogObjectState((PetscObject)v,"Length=%D",v->map->n);
111: #endif
112:   if (vs) {
113:     if (vs->array_allocated) {
114:       if (v->pinned_memory) {
115:         PetscMallocSetCUDAHost();
116:       }
117:       PetscFree(vs->array_allocated);
118:       if (v->pinned_memory) {
119:         PetscMallocResetCUDAHost();
120:         v->pinned_memory = PETSC_FALSE;
121:       }
122:     }
123:     PetscFree(vs);
124:   }
125:   return(0);
126: }

128: PetscErrorCode VecResetArray_SeqCUDA_Private(Vec vin)
129: {
130:   Vec_Seq *v = (Vec_Seq*)vin->data;

133:   v->array         = v->unplacedarray;
134:   v->unplacedarray = 0;
135:   return(0);
136: }

138: PetscErrorCode VecCUDAAllocateCheck_Public(Vec v)
139: {

143:   VecCUDAAllocateCheck(v);
144:   return(0);
145: }

147: PetscErrorCode VecCUDACopyToGPU_Public(Vec v)
148: {

152:   VecCUDACopyToGPU(v);
153:   return(0);
154: }

156: /*
157:     VecCUDACopyToGPUSome_Public - Copies certain entries down to the GPU from the CPU of a vector

159:    Input Parameters:
160:  +  v    - the vector
161:  .  ci   - the requested indices, this should be created with CUDAIndicesCreate()
162:  -  mode - vec scatter mode used in VecScatterBegin/End
163: */
164: PetscErrorCode VecCUDACopyToGPUSome_Public(Vec v,PetscCUDAIndices ci,ScatterMode mode)
165: {

169:   VecCUDACopyToGPUSome(v,ci,mode);
170:   return(0);
171: }

173: /*
174:   VecCUDACopyFromGPUSome_Public - Copies certain entries up to the CPU from the GPU of a vector

176:   Input Parameters:
177:  +  v    - the vector
178:  .  ci   - the requested indices, this should be created with CUDAIndicesCreate()
179:  -  mode - vec scatter mode used in VecScatterBegin/End
180: */
181: PetscErrorCode VecCUDACopyFromGPUSome_Public(Vec v,PetscCUDAIndices ci,ScatterMode mode)
182: {

186:   VecCUDACopyFromGPUSome(v,ci,mode);
187:   return(0);
188: }

190: PetscErrorCode VecSetRandom_SeqCUDA(Vec xin,PetscRandom r)
191: {

195:   VecSetRandom_SeqCUDA_Private(xin,r);
196:   xin->offloadmask = PETSC_OFFLOAD_CPU;
197:   return(0);
198: }

200: PetscErrorCode VecResetArray_SeqCUDA(Vec vin)
201: {

205:   VecCUDACopyFromGPU(vin);
206:   VecResetArray_SeqCUDA_Private(vin);
207:   vin->offloadmask = PETSC_OFFLOAD_CPU;
208:   return(0);
209: }

211: PetscErrorCode VecPlaceArray_SeqCUDA(Vec vin,const PetscScalar *a)
212: {

216:   VecCUDACopyFromGPU(vin);
217:   VecPlaceArray_Seq(vin,a);
218:   vin->offloadmask = PETSC_OFFLOAD_CPU;
219:   return(0);
220: }

222: PetscErrorCode VecReplaceArray_SeqCUDA(Vec vin,const PetscScalar *a)
223: {
225:   Vec_Seq        *vs = (Vec_Seq*)vin->data;

228:   if (vs->array != vs->array_allocated) {
229:     /* make sure the users array has the latest values */
230:     VecCUDACopyFromGPU(vin);
231:   }
232:   if (vs->array_allocated) {
233:     if (vin->pinned_memory) {
234:       PetscMallocSetCUDAHost();
235:     }
236:     PetscFree(vs->array_allocated);
237:     if (vin->pinned_memory) {
238:       PetscMallocResetCUDAHost();
239:     }
240:   }
241:   vin->pinned_memory = PETSC_FALSE;
242:   vs->array_allocated = vs->array = (PetscScalar*)a;
243:   vin->offloadmask = PETSC_OFFLOAD_CPU;
244:   return(0);
245: }

247: /*@
248:  VecCreateSeqCUDA - Creates a standard, sequential array-style vector.

250:  Collective

252:  Input Parameter:
253:  +  comm - the communicator, should be PETSC_COMM_SELF
254:  -  n - the vector length

256:  Output Parameter:
257:  .  v - the vector

259:  Notes:
260:  Use VecDuplicate() or VecDuplicateVecs() to form additional vectors of the
261:  same type as an existing vector.

263:  Level: intermediate

265:  .seealso: VecCreateMPI(), VecCreate(), VecDuplicate(), VecDuplicateVecs(), VecCreateGhost()
266:  @*/
267: PetscErrorCode VecCreateSeqCUDA(MPI_Comm comm,PetscInt n,Vec *v)
268: {

272:   VecCreate(comm,v);
273:   VecSetSizes(*v,n,n);
274:   VecSetType(*v,VECSEQCUDA);
275:   return(0);
276: }

278: PetscErrorCode VecDuplicate_SeqCUDA(Vec win,Vec *V)
279: {

283:   VecCreateSeqCUDA(PetscObjectComm((PetscObject)win),win->map->n,V);
284:   PetscLayoutReference(win->map,&(*V)->map);
285:   PetscObjectListDuplicate(((PetscObject)win)->olist,&((PetscObject)(*V))->olist);
286:   PetscFunctionListDuplicate(((PetscObject)win)->qlist,&((PetscObject)(*V))->qlist);
287:   (*V)->stash.ignorenegidx = win->stash.ignorenegidx;
288:   return(0);
289: }

291: PetscErrorCode VecCreate_SeqCUDA(Vec V)
292: {

296:   PetscCUDAInitializeCheck();
297:   PetscLayoutSetUp(V->map);
298:   VecCUDAAllocateCheck(V);
299:   VecCreate_SeqCUDA_Private(V,((Vec_CUDA*)V->spptr)->GPUarray_allocated);
300:   VecCUDAAllocateCheckHost(V);
301:   VecSet(V,0.0);
302:   VecSet_Seq(V,0.0);
303:   V->offloadmask = PETSC_OFFLOAD_BOTH;
304:   return(0);
305: }

307: /*@C
308:    VecCreateSeqCUDAWithArray - Creates a CUDA sequential array-style vector,
309:    where the user provides the array space to store the vector values. The array
310:    provided must be a GPU array.

312:    Collective

314:    Input Parameter:
315: +  comm - the communicator, should be PETSC_COMM_SELF
316: .  bs - the block size
317: .  n - the vector length
318: -  array - GPU memory where the vector elements are to be stored.

320:    Output Parameter:
321: .  V - the vector

323:    Notes:
324:    Use VecDuplicate() or VecDuplicateVecs() to form additional vectors of the
325:    same type as an existing vector.

327:    If the user-provided array is NULL, then VecCUDAPlaceArray() can be used
328:    at a later stage to SET the array for storing the vector values.

330:    PETSc does NOT free the array when the vector is destroyed via VecDestroy().
331:    The user should not free the array until the vector is destroyed.

333:    Level: intermediate

335: .seealso: VecCreateMPICUDAWithArray(), VecCreate(), VecDuplicate(), VecDuplicateVecs(),
336:           VecCreateGhost(), VecCreateSeq(), VecCUDAPlaceArray(), VecCreateSeqWithArray(),
337:           VecCreateMPIWithArray()
338: @*/
339: PetscErrorCode  VecCreateSeqCUDAWithArray(MPI_Comm comm,PetscInt bs,PetscInt n,const PetscScalar array[],Vec *V)
340: {

344:   PetscCUDAInitializeCheck();
345:   VecCreate(comm,V);
346:   VecSetSizes(*V,n,n);
347:   VecSetBlockSize(*V,bs);
348:   VecCreate_SeqCUDA_Private(*V,array);
349:   (*V)->offloadmask = PETSC_OFFLOAD_GPU;
350:   return(0);
351: }

353: /*@C
354:    VecCreateSeqCUDAWithArrays - Creates a CUDA sequential array-style vector,
355:    where the user provides the array space to store the vector values.

357:    Collective

359:    Input Parameter:
360: +  comm - the communicator, should be PETSC_COMM_SELF
361: .  bs - the block size
362: .  n - the vector length
363: -  cpuarray - CPU memory where the vector elements are to be stored.
364: -  gpuarray - GPU memory where the vector elements are to be stored.

366:    Output Parameter:
367: .  V - the vector

369:    Notes:
370:    If both cpuarray and gpuarray are provided, the caller must ensure that
371:    the provided arrays have identical values.

373:    PETSc does NOT free the provided arrays when the vector is destroyed via
374:    VecDestroy(). The user should not free the array until the vector is
375:    destroyed.

377:    Level: intermediate

379: .seealso: VecCreateMPICUDAWithArrays(), VecCreate(), VecCreateSeqWithArray(),
380:           VecCUDAPlaceArray(), VecCreateSeqCUDAWithArray(),
381:           VecCUDAAllocateCheckHost()
382: @*/
383: PetscErrorCode  VecCreateSeqCUDAWithArrays(MPI_Comm comm,PetscInt bs,PetscInt n,const PetscScalar cpuarray[],const PetscScalar gpuarray[],Vec *V)
384: {

388:   // set V's gpuarray to be gpuarray, do not allocate memory on host yet.
389:   VecCreateSeqCUDAWithArray(comm,bs,n,gpuarray,V);

391:   if (cpuarray && gpuarray) {
392:     Vec_Seq *s = (Vec_Seq*)((*V)->data);
393:     s->array = (PetscScalar*)cpuarray;
394:     (*V)->offloadmask = PETSC_OFFLOAD_BOTH;
395:   } else if (cpuarray) {
396:     Vec_Seq *s = (Vec_Seq*)((*V)->data);
397:     s->array = (PetscScalar*)cpuarray;
398:     (*V)->offloadmask = PETSC_OFFLOAD_CPU;
399:   } else if (gpuarray) {
400:     (*V)->offloadmask = PETSC_OFFLOAD_GPU;
401:   } else {
402:     (*V)->offloadmask = PETSC_OFFLOAD_UNALLOCATED;
403:   }

405:   return(0);
406: }

408: PetscErrorCode VecGetArrayWrite_SeqCUDA(Vec v,PetscScalar **vv)
409: {

413:   VecCUDAAllocateCheckHost(v);
414:   v->offloadmask = PETSC_OFFLOAD_CPU;
415:   *vv = *((PetscScalar**)v->data);
416:   return(0);
417: }

419: PetscErrorCode VecBindToCPU_SeqCUDA(Vec V,PetscBool pin)
420: {

424:   V->boundtocpu = pin;
425:   if (pin) {
426:     VecCUDACopyFromGPU(V);
427:     V->offloadmask                 = PETSC_OFFLOAD_CPU; /* since the CPU code will likely change values in the vector */
428:     V->ops->dot                    = VecDot_Seq;
429:     V->ops->norm                   = VecNorm_Seq;
430:     V->ops->tdot                   = VecTDot_Seq;
431:     V->ops->scale                  = VecScale_Seq;
432:     V->ops->copy                   = VecCopy_Seq;
433:     V->ops->set                    = VecSet_Seq;
434:     V->ops->swap                   = VecSwap_Seq;
435:     V->ops->axpy                   = VecAXPY_Seq;
436:     V->ops->axpby                  = VecAXPBY_Seq;
437:     V->ops->axpbypcz               = VecAXPBYPCZ_Seq;
438:     V->ops->pointwisemult          = VecPointwiseMult_Seq;
439:     V->ops->pointwisedivide        = VecPointwiseDivide_Seq;
440:     V->ops->setrandom              = VecSetRandom_Seq;
441:     V->ops->dot_local              = VecDot_Seq;
442:     V->ops->tdot_local             = VecTDot_Seq;
443:     V->ops->norm_local             = VecNorm_Seq;
444:     V->ops->mdot_local             = VecMDot_Seq;
445:     V->ops->mtdot_local            = VecMTDot_Seq;
446:     V->ops->maxpy                  = VecMAXPY_Seq;
447:     V->ops->mdot                   = VecMDot_Seq;
448:     V->ops->mtdot                  = VecMTDot_Seq;
449:     V->ops->aypx                   = VecAYPX_Seq;
450:     V->ops->waxpy                  = VecWAXPY_Seq;
451:     V->ops->dotnorm2               = NULL;
452:     V->ops->placearray             = VecPlaceArray_Seq;
453:     V->ops->replacearray           = VecReplaceArray_SeqCUDA;
454:     V->ops->resetarray             = VecResetArray_Seq;
455:     V->ops->duplicate              = VecDuplicate_Seq;
456:     V->ops->conjugate              = VecConjugate_Seq;
457:     V->ops->getlocalvector         = NULL;
458:     V->ops->restorelocalvector     = NULL;
459:     V->ops->getlocalvectorread     = NULL;
460:     V->ops->restorelocalvectorread = NULL;
461:     V->ops->getarraywrite          = NULL;
462:   } else {
463:     V->ops->dot                    = VecDot_SeqCUDA;
464:     V->ops->norm                   = VecNorm_SeqCUDA;
465:     V->ops->tdot                   = VecTDot_SeqCUDA;
466:     V->ops->scale                  = VecScale_SeqCUDA;
467:     V->ops->copy                   = VecCopy_SeqCUDA;
468:     V->ops->set                    = VecSet_SeqCUDA;
469:     V->ops->swap                   = VecSwap_SeqCUDA;
470:     V->ops->axpy                   = VecAXPY_SeqCUDA;
471:     V->ops->axpby                  = VecAXPBY_SeqCUDA;
472:     V->ops->axpbypcz               = VecAXPBYPCZ_SeqCUDA;
473:     V->ops->pointwisemult          = VecPointwiseMult_SeqCUDA;
474:     V->ops->pointwisedivide        = VecPointwiseDivide_SeqCUDA;
475:     V->ops->setrandom              = VecSetRandom_SeqCUDA;
476:     V->ops->dot_local              = VecDot_SeqCUDA;
477:     V->ops->tdot_local             = VecTDot_SeqCUDA;
478:     V->ops->norm_local             = VecNorm_SeqCUDA;
479:     V->ops->mdot_local             = VecMDot_SeqCUDA;
480:     V->ops->maxpy                  = VecMAXPY_SeqCUDA;
481:     V->ops->mdot                   = VecMDot_SeqCUDA;
482:     V->ops->aypx                   = VecAYPX_SeqCUDA;
483:     V->ops->waxpy                  = VecWAXPY_SeqCUDA;
484:     V->ops->dotnorm2               = VecDotNorm2_SeqCUDA;
485:     V->ops->placearray             = VecPlaceArray_SeqCUDA;
486:     V->ops->replacearray           = VecReplaceArray_SeqCUDA;
487:     V->ops->resetarray             = VecResetArray_SeqCUDA;
488:     V->ops->destroy                = VecDestroy_SeqCUDA;
489:     V->ops->duplicate              = VecDuplicate_SeqCUDA;
490:     V->ops->conjugate              = VecConjugate_SeqCUDA;
491:     V->ops->getlocalvector         = VecGetLocalVector_SeqCUDA;
492:     V->ops->restorelocalvector     = VecRestoreLocalVector_SeqCUDA;
493:     V->ops->getlocalvectorread     = VecGetLocalVector_SeqCUDA;
494:     V->ops->restorelocalvectorread = VecRestoreLocalVector_SeqCUDA;
495:     V->ops->getarraywrite          = VecGetArrayWrite_SeqCUDA;
496:   }
497:   return(0);
498: }

500: PetscErrorCode VecCreate_SeqCUDA_Private(Vec V,const PetscScalar *array)
501: {
503:   Vec_CUDA       *veccuda;
504:   PetscMPIInt    size;
505:   PetscBool      option_set;

508:   MPI_Comm_size(PetscObjectComm((PetscObject)V),&size);
509:   if (size > 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Cannot create VECSEQCUDA on more than one process");
510:   VecCreate_Seq_Private(V,0);
511:   PetscObjectChangeTypeName((PetscObject)V,VECSEQCUDA);
512:   VecBindToCPU_SeqCUDA(V,PETSC_FALSE);
513:   V->ops->bindtocpu = VecBindToCPU_SeqCUDA;

515:   /* Later, functions check for the Vec_CUDA structure existence, so do not create it without array */
516:   if (array) {
517:     if (!V->spptr) {
518:       PetscReal pinned_memory_min;
519:       PetscMalloc(sizeof(Vec_CUDA),&V->spptr);
520:       veccuda = (Vec_CUDA*)V->spptr;
521:       veccuda->stream = 0; /* using default stream */
522:       veccuda->GPUarray_allocated = 0;
523:       V->offloadmask = PETSC_OFFLOAD_UNALLOCATED;

525:       pinned_memory_min = 0;
526:       /* Need to parse command line for minimum size to use for pinned memory allocations on host here.
527:          Note: This same code duplicated in VecCUDAAllocateCheck() and VecCreate_MPICUDA_Private(). Is there a good way to avoid this? */
528:       PetscOptionsBegin(PetscObjectComm((PetscObject)V),((PetscObject)V)->prefix,"VECCUDA Options","Vec");
529:       PetscOptionsReal("-vec_pinned_memory_min","Minimum size (in bytes) for an allocation to use pinned memory on host","VecSetPinnedMemoryMin",pinned_memory_min,&pinned_memory_min,&option_set);
530:       if (option_set) V->minimum_bytes_pinned_memory = pinned_memory_min;
531:       PetscOptionsEnd();
532:     }
533:     veccuda = (Vec_CUDA*)V->spptr;
534:     veccuda->GPUarray = (PetscScalar*)array;
535:   }
536:   return(0);
537: }