ElpaScalapackOperations.h

Go to the documentation of this file.

/******************************************************************************
 * Copyright (c) 2021.                                                        *
 * The Regents of the University of Michigan and DFT-EFE developers.          *
 *                                                                            *
 * This file is part of the DFT-EFE code.                                     *
 *                                                                            *
 * DFT-EFE is free software: you can redistribute it and/or modify            *
 *   it under the terms of the Lesser GNU General Public License as           *
 *   published by the Free Software Foundation, either version 3 of           *
 *   the License, or (at your option) any later version.                      *
 *                                                                            *
 * DFT-EFE is distributed in the hope that it will be useful, but             *
 *   WITHOUT ANY WARRANTY; without even the implied warranty                  *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                     *
 *   See the Lesser GNU General Public License for more details.              *
 *                                                                            *
 * You should have received a copy of the GNU Lesser General Public           *
 *   License at the top level of DFT-EFE distribution.  If not, see           *
 *   <https://www.gnu.org/licenses/>.                                         *
 ******************************************************************************/
 
#ifndef elpaScalapackOperations_h
#define elpaScalapackOperations_h
 
#include "ProcessGrid.h"
#include "ScalapackWrapper.h"
#include <elpa/elpa.h>
#include <unordered_map>
#include <iostream>
#include "LinAlgOpContext.h"
 
namespace dftefe
{
  namespace linearAlgebra
  {
    namespace elpaScalaOpInternal
    {
      void
      setupELPAHandleParameters(
        const utils::mpi::MPIComm &               mpi_communicator,
        utils::mpi::MPIComm &                     processGridCommunicatorActive,
        const std::shared_ptr<const ProcessGrid> &processGrid,
        const size_type                           na,
        const size_type                           nev,
        const size_type                           blockSize,
        elpa_t &                                  elpaHandle,
        const bool                                useELPADeviceKernel = false);
 
      void
      createProcessGridSquareMatrix(
        const utils::mpi::MPIComm &         mpi_communicator,
        const size_type                     size,
        std::shared_ptr<const ProcessGrid> &processGrid,
        const size_type                     scalapackParalProcs,
        const bool                          useELPA,
        const bool                          useOnlyThumbRule = false);
 
      void
      createProcessGridRectangularMatrix(
        const utils::mpi::MPIComm &         mpi_communicator,
        const size_type                     sizeRows,
        const size_type                     sizeColumns,
        std::shared_ptr<const ProcessGrid> &processGrid);
 
 
      template <typename T>
      void
      createGlobalToLocalIdMapsScaLAPACKMat(
        const std::shared_ptr<const ProcessGrid> &processGrid,
        const ScaLAPACKMatrix<T> &                mat,
        std::unordered_map<size_type, size_type> &globalToLocalRowIdMap,
        std::unordered_map<size_type, size_type> &globalToLocalColumnIdMap);
 
      template <typename T>
      void
      scaleScaLAPACKMat(const std::shared_ptr<const ProcessGrid> &processGrid,
                        /*const std::shared_ptr<
                          linearAlgebra::BLASWrapper<utils::MemorySpace::HOST>>
                          &                        BLASWrapperPtr,*/
                        ScaLAPACKMatrix<T> &mat,
                        const T             scalar);
 
      template <typename T>
      void
      fillParallelOverlapMatrix(
        const T *X,
        /*const std::shared_ptr<
          linearAlgebra::BLASWrapper<utils::MemorySpace::HOST>>
          &                                              BLASWrapperPtr,*/
        const size_type                           XLocalSize,
        const size_type                           numberVectors,
        const std::shared_ptr<const ProcessGrid> &processGrid,
        const utils::mpi::MPIComm &               mpiComm,
        const size_type                           wfcBlockSize,
        ScaLAPACKMatrix<T> &                      overlapMatPar);
 
      template <typename ValueType, typename utils::MemorySpace memorySpace>
      void
      subspaceRotation(
        ValueType *     X,
        const size_type M,
        const size_type N,
        /*std::shared_ptr<
          linearAlgebra::BLASWrapper<memorySpace>> &BLASWrapperPtr,*/
        const std::shared_ptr<const ProcessGrid> &processGrid,
        const utils::mpi::MPIComm &               mpiCommDomain,
        LinAlgOpContext<memorySpace> &            linAlgOpContext,
        const ScaLAPACKMatrix<ValueType> &        rotationMatPar,
        const size_type                           subspaceRotDofsBlockSize,
        const size_type                           wfcBlockSize,
        const bool                                rotationMatTranspose = false,
        const bool isRotationMatLowerTria                              = false);
 
    } // namespace elpaScalaOpInternal
  }   // namespace linearAlgebra
} // namespace dftefe
#include <linearAlgebra/ElpaScalapackOperations.t.cpp>
#endif