Amesos2_Superlu_decl.hpp

Go to the documentation of this file.

// @HEADER
//
// ***********************************************************************
//
//           Amesos2: Templated Direct Sparse Solver Package
//                  Copyright 2011 Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
//
// ***********************************************************************
//
// @HEADER
 
#ifndef AMESOS2_SUPERLU_DECL_HPP
#define AMESOS2_SUPERLU_DECL_HPP
 
#include "Amesos2_SolverTraits.hpp"
#include "Amesos2_SolverCore.hpp"
#include "Amesos2_Superlu_FunctionMap.hpp"
 
#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_SUPERLU)
#include "KokkosKernels_Handle.hpp"
#endif
 
namespace Amesos2 {
 
 
template <class Matrix,
          class Vector>
class Superlu : public SolverCore<Amesos2::Superlu, Matrix, Vector>
{
  friend class SolverCore<Amesos2::Superlu,Matrix,Vector>; // Give our base access
                                                          // to our private
                                                          // implementation funcs
public:
 
  static const char* name;      // declaration. Initialization outside.
 
  typedef Superlu<Matrix,Vector>                                       type;
  typedef SolverCore<Amesos2::Superlu,Matrix,Vector>             super_type;
 
  // Since typedef's are not inheritted, go grab them
  typedef typename super_type::scalar_type                      scalar_type;
  typedef typename super_type::local_ordinal_type        local_ordinal_type;
  typedef typename super_type::global_ordinal_type      global_ordinal_type;
  typedef typename super_type::global_size_type            global_size_type;
 
  typedef TypeMap<Amesos2::Superlu,scalar_type>                    type_map;
 
  /*
   * The SuperLU interface will need two other typedef's, which are:
   * - the superlu type that corresponds to scalar_type and
   * - the corresponding type to use for magnitude
   */
  typedef typename type_map::type                                  slu_type;
  typedef typename type_map::convert_type                  slu_convert_type;
  typedef typename type_map::magnitude_type                  magnitude_type;
 
  typedef FunctionMap<Amesos2::Superlu,slu_type>               function_map;
 
#ifdef HAVE_AMESOS2_SUPERLU5_API
  typedef typename function_map::GlobalLU_type                   GlobalLU_t;
#endif
 
 
 
  Superlu(Teuchos::RCP<const Matrix> A,
          Teuchos::RCP<Vector>       X,
          Teuchos::RCP<const Vector> B);
 
 
  ~Superlu( );
 
 
  std::string description() const;
 
private:
 
  int preOrdering_impl();
 
 
  int symbolicFactorization_impl();
 
 
  int numericFactorization_impl();
 
 
  int solve_impl(const Teuchos::Ptr<MultiVecAdapter<Vector> >       X,
                 const Teuchos::Ptr<const MultiVecAdapter<Vector> > B) const;
 
 
  bool matrixShapeOK_impl() const;
 
 
  void setParameters_impl(
    const Teuchos::RCP<Teuchos::ParameterList> & parameterList );
 
 
  Teuchos::RCP<const Teuchos::ParameterList> getValidParameters_impl() const;
 
 
  bool loadA_impl(EPhase current_phase);
 
  typedef Kokkos::DefaultHostExecutionSpace HostExecSpaceType;
 
  // struct holds all data necessary to make a superlu factorization or solve call
  mutable struct SLUData {
    SLU::SuperMatrix A, B, X, L, U; // matrix A in NCformat
    SLU::SuperMatrix AC;        // permuted matrix A in NCPformat
 
    SLU::superlu_options_t options;
    SLU::mem_usage_t mem_usage;
#ifdef HAVE_AMESOS2_SUPERLU5_API
    GlobalLU_t lu;      // Use for gssvx and gsisx in SuperLU 5.0
#endif
    SLU::SuperLUStat_t stat;
 
 
 
    typedef Kokkos::View<magnitude_type*, HostExecSpaceType>    host_mag_array;
    typedef Kokkos::View<int*, HostExecSpaceType>               host_int_array;
    host_mag_array berr; 
    host_mag_array ferr; 
    host_int_array perm_r;
    host_int_array perm_c;
    host_int_array etree;
    host_mag_array R;
    host_mag_array C;
 
#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_SUPERLU)
    host_int_array parents;
#endif
 
    char equed;
    bool rowequ, colequ;        // flags what type of equilibration
                                // has been performed
 
    int relax;
    int panel_size;
  } data_;
 
  typedef int size_type;
  typedef int ordinal_type;
  typedef Kokkos::View<size_type*, HostExecSpaceType>       host_size_type_array;
  typedef Kokkos::View<ordinal_type*, HostExecSpaceType> host_ordinal_type_array;
  typedef Kokkos::View<slu_type*, HostExecSpaceType>       host_value_type_array;
 
  // The following Arrays are persisting storage arrays for A, X, and B
  host_value_type_array host_nzvals_view_;
  Teuchos::Array<slu_convert_type> convert_nzvals_; // copy to SuperLU native array before calling SuperLU
 
  host_size_type_array host_rows_view_;
  host_ordinal_type_array host_col_ptr_view_;
 
  typedef typename Kokkos::View<slu_type**, Kokkos::LayoutLeft, HostExecSpaceType>
    host_solve_array_t;
 
  mutable host_solve_array_t host_xValues_;
  mutable Teuchos::Array<slu_convert_type> convert_xValues_; // copy to SuperLU native array before calling SuperLU
 
  mutable host_solve_array_t host_bValues_;
  mutable Teuchos::Array<slu_convert_type> convert_bValues_; // copy to SuperLU native array before calling SuperLU
 
#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_SUPERLU)
  typedef Kokkos::DefaultExecutionSpace DeviceExecSpaceType;
 
  #ifdef KOKKOS_ENABLE_CUDA
    // solver will be UVM off even though Tpetra is CudaUVMSpace
    typedef typename Kokkos::CudaSpace DeviceMemSpaceType;
  #else
    typedef typename DeviceExecSpaceType::memory_space DeviceMemSpaceType;
  #endif
 
  typedef Kokkos::View<slu_type**, Kokkos::LayoutLeft, DeviceMemSpaceType>
    device_solve_array_t;
  // For triangular solves we have both host and device versions of xValues and
  // bValues because a parameter can turn it on or off.
  mutable device_solve_array_t device_xValues_;
  mutable device_solve_array_t device_bValues_;
  typedef Kokkos::View<int*,            DeviceMemSpaceType>      device_int_array;
  typedef Kokkos::View<magnitude_type*, DeviceMemSpaceType>      device_mag_array;
  device_int_array device_trsv_perm_r_;
  device_int_array device_trsv_perm_c_;
  device_mag_array device_trsv_R_;
  device_mag_array device_trsv_C_;
  mutable device_solve_array_t device_trsv_rhs_;
  mutable device_solve_array_t device_trsv_sol_;
  typedef KokkosKernels::Experimental::KokkosKernelsHandle <size_type, ordinal_type, slu_type,
    DeviceExecSpaceType, DeviceMemSpaceType, DeviceMemSpaceType> kernel_handle_type;
  mutable kernel_handle_type device_khL_;
  mutable kernel_handle_type device_khU_;
  /* parameters for SpTRSV */
  bool sptrsv_invert_diag_;
  bool sptrsv_invert_offdiag_;
  bool sptrsv_u_in_csr_;
  bool sptrsv_merge_supernodes_;
  bool sptrsv_use_spmv_;
#endif
 
  /* Note: In the above, must use "Amesos2::Superlu" rather than
   * "Superlu" because otherwise the compiler references the
   * specialized type of the class, and not the templated type that is
   * required for Amesos2::TypeMap
   */
 
  /* SuperLU can accept input in either compressed-row or
   * compressed-column storage.  We will store and pass matrices in
   * *compressed-column* format.
   */
 
  /*
   * Internal flag that is used for the numericFactorization_impl
   * routine.  If true, then the superlu gstrf routine should have
   * SamePattern_SameRowPerm in its options.  Otherwise, it should
   * factor from scratch.
   *
   * This is somewhat of a kludge to get around the fact that the
   * superlu routines all expect something different from the options
   * struct.  The big issue is that we don't want gstrf doing the
   * symbolic factorization if it doesn't need to.  On the other hand,
   * we can't leave options.Fact set to SamePattern_SameRowPerm
   * because the solver driver needs it to be set at FACTORED.  But
   * having it set at FACTORED upon re-entrance into
   * numericFactorization prompts gstrf to redo the symbolic
   * factorization.
   */
  bool same_symbolic_;
  bool ILU_Flag_;
 
  bool is_contiguous_;
  bool use_triangular_solves_;
 
  void triangular_solve_factor();
 
  /* call metis before SuperLU */
  bool use_metis_;
  bool symmetrize_metis_;
 
  public: // for GPU
    void triangular_solve() const; // Only for internal use - public to support kernels
};                              // End class Superlu
 
 
// Specialize solver_traits struct for SuperLU
template <>
struct solver_traits<Superlu> {
#ifdef HAVE_TEUCHOS_COMPLEX
  typedef Meta::make_list6<float, double,
                           std::complex<float>, std::complex<double>,
                           Kokkos::complex<float>, Kokkos::complex<double>>
                           supported_scalars;
#else
  typedef Meta::make_list2<float, double> supported_scalars;
#endif
};
 
template <typename Scalar, typename LocalOrdinal, typename ExecutionSpace>
struct solver_supports_matrix<Superlu,
  KokkosSparse::CrsMatrix<Scalar, LocalOrdinal, ExecutionSpace>> {
  static const bool value = true;
};
 
} // end namespace Amesos2
 
#endif  // AMESOS2_SUPERLU_DECL_HPP