Line data    Source code

       1             : //* This file is part of the MOOSE framework
       2             : //* https://mooseframework.inl.gov
       3             : //*
       4             : //* All rights reserved, see COPYRIGHT for full restrictions
       5             : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
       6             : //*
       7             : //* Licensed under LGPL 2.1, please see LICENSE for details
       8             : //* https://www.gnu.org/licenses/lgpl-2.1.html
       9             : 
      10             : #include "KokkosHeader.h"
      11             : 
      12             : #include "MooseInit.h"
      13             : 
      14             : #include "libmesh/parallel_implementation.h"
      15             : #include "libmesh/petsc_solver_exception.h"
      16             : 
      17             : PetscErrorCode
      18       35016 : finalizeKokkos()
      19             : {
      20             :   PetscFunctionBegin;
      21       35016 :   Kokkos::finalize();
      22       35016 :   PetscFunctionReturn(PETSC_SUCCESS);
      23             : }
      24             : 
      25             : void
      26       37208 : MooseInit::initKokkos()
      27             : {
      28       37208 :   Kokkos::InitializationSettings settings;
      29             : 
      30             :   // Explicitly set the number of threads consistently with MOOSE
      31       37208 :   settings.set_num_threads(libMesh::n_threads());
      32             : 
      33             :   // Only print warnings on the head process
      34       37208 :   if (comm().rank())
      35        8974 :     settings.set_disable_warnings(true);
      36             : 
      37             : #ifdef MOOSE_ENABLE_KOKKOS_GPU
      38             : 
      39         302 :   unsigned int num_kokkos_gpus = Kokkos::num_devices();
      40             : 
      41         302 :   comm().min(num_kokkos_gpus);
      42             : 
      43         302 :   if (num_kokkos_gpus)
      44             :   {
      45             :     // Create a local communicator defined at each shared memory node
      46         302 :     Parallel::Communicator local_comm;
      47         302 :     comm().split_by_type(MPI_COMM_TYPE_SHARED, comm().rank(), MPI_INFO_NULL, local_comm);
      48             : 
      49             :     // The number of processes in each node is usually larger than the number of
      50             :     // GPU devices in the node, so multiple processes share the same GPU
      51         302 :     unsigned int kokkos_gpu_id = local_comm.rank() % num_kokkos_gpus;
      52             : 
      53             :     // Override the default GPU ID
      54         302 :     settings.set_device_id(kokkos_gpu_id);
      55         302 :   }
      56             : 
      57             : #endif
      58             : 
      59       37208 :   Kokkos::initialize(settings);
      60             : 
      61       37208 :   LibmeshPetscCall(PetscRegisterFinalize(finalizeKokkos));
      62       37208 : }