Line data    Source code

       1             : //* This file is part of the MOOSE framework
       2             : //* https://mooseframework.inl.gov
       3             : //*
       4             : //* All rights reserved, see COPYRIGHT for full restrictions
       5             : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
       6             : //*
       7             : //* Licensed under LGPL 2.1, please see LICENSE for details
       8             : //* https://www.gnu.org/licenses/lgpl-2.1.html
       9             : 
      10             : #include "KokkosHeader.h"
      11             : 
      12             : #include "MooseInit.h"
      13             : 
      14             : #include "libmesh/parallel_implementation.h"
      15             : #include "libmesh/petsc_solver_exception.h"
      16             : 
      17             : PetscErrorCode
      18       35363 : finalizeKokkos()
      19             : {
      20             :   PetscFunctionBegin;
      21       35363 :   Kokkos::finalize();
      22       35363 :   PetscFunctionReturn(PETSC_SUCCESS);
      23             : }
      24             : 
      25             : void
      26       37597 : MooseInit::initKokkos()
      27             : {
      28       37597 :   Kokkos::InitializationSettings settings;
      29             : 
      30             :   // Explicitly set the number of threads consistently with MOOSE
      31       37597 :   settings.set_num_threads(libMesh::n_threads());
      32             : 
      33             :   // Only print warnings on the head process
      34       37597 :   if (comm().rank())
      35        9049 :     settings.set_disable_warnings(true);
      36             : 
      37             : #ifdef MOOSE_ENABLE_KOKKOS_GPU
      38             : 
      39         318 :   unsigned int num_kokkos_gpus = Kokkos::num_devices();
      40             : 
      41         318 :   comm().min(num_kokkos_gpus);
      42             : 
      43         318 :   if (num_kokkos_gpus)
      44             :   {
      45             :     // Create a local communicator defined at each shared memory node
      46         318 :     Parallel::Communicator local_comm;
      47         318 :     comm().split_by_type(MPI_COMM_TYPE_SHARED, comm().rank(), MPI_INFO_NULL, local_comm);
      48             : 
      49             :     // The number of processes in each node is usually larger than the number of
      50             :     // GPU devices in the node, so multiple processes share the same GPU
      51         318 :     unsigned int kokkos_gpu_id = local_comm.rank() % num_kokkos_gpus;
      52             : 
      53             :     // Override the default GPU ID
      54         318 :     settings.set_device_id(kokkos_gpu_id);
      55         318 :   }
      56             : 
      57             : #endif
      58             : 
      59             :   // Set this environment variable if not set already to silence Kokkos warning
      60       37597 :   setenv("OMP_PROC_BIND", "false", false);
      61             : 
      62       37597 :   Kokkos::initialize(settings);
      63             : 
      64       37597 :   LibmeshPetscCall(PetscRegisterFinalize(finalizeKokkos));
      65       37597 : }