From 03f4904c34de80b947b4ef0c82be958da7711a7c Mon Sep 17 00:00:00 2001 From: iyamazaki Date: Tue, 19 Oct 2021 01:03:01 -0600 Subject: [PATCH 1/2] Amesos2::SuperLU_dist: add option to apply Equil --- .../src/Amesos2_Superludist_FunctionMap.hpp | 4 +- .../amesos2/src/Amesos2_Superludist_def.hpp | 97 +++++++++++-------- 2 files changed, 60 insertions(+), 41 deletions(-) diff --git a/packages/amesos2/src/Amesos2_Superludist_FunctionMap.hpp b/packages/amesos2/src/Amesos2_Superludist_FunctionMap.hpp index 7eb5862b1b6d..cd92961a72b1 100644 --- a/packages/amesos2/src/Amesos2_Superludist_FunctionMap.hpp +++ b/packages/amesos2/src/Amesos2_Superludist_FunctionMap.hpp @@ -532,14 +532,14 @@ namespace Amesos2 { } static void gsequ_loc(SLUD::SuperMatrix* A, double* r, double* c, - double* rowcnd, double* colcnd, double* amax, int* info, + double* rowcnd, double* colcnd, double* amax, SLUD::int_t* info, SLUD::gridinfo_t* grid) { SLUD::Z::pzgsequ(A, r, c, rowcnd, colcnd, amax, info, grid); } static void gsequ(SLUD::SuperMatrix* A, double* r, double* c, - double* rowcnd, double* colcnd, double* amax, int* info) + double* rowcnd, double* colcnd, double* amax, SLUD::int_t* info) { SLUD::Z::zgsequ_dist(A, r, c, rowcnd, colcnd, amax, info); } diff --git a/packages/amesos2/src/Amesos2_Superludist_def.hpp b/packages/amesos2/src/Amesos2_Superludist_def.hpp index f312b2b48e44..4cd1fea25c5d 100644 --- a/packages/amesos2/src/Amesos2_Superludist_def.hpp +++ b/packages/amesos2/src/Amesos2_Superludist_def.hpp @@ -355,17 +355,17 @@ namespace Amesos2 { free( data_.fstVtxSep ); #endif } + float info = 0.0; + { #ifdef HAVE_AMESOS2_TIMERS - Teuchos::TimeMonitor preOrderTime( this->timers_.preOrderTime_ ); + Teuchos::TimeMonitor preOrderTime( this->timers_.preOrderTime_ ); #endif - - float info = 0.0; - info = SLUD::get_perm_c_parmetis( &(data_.A), - data_.perm_r.getRawPtr(), data_.perm_c.getRawPtr(), - data_.grid.nprow * data_.grid.npcol, data_.domains, - &(data_.sizes), &(data_.fstVtxSep), - &(data_.grid), &(data_.symb_comm) ); - + info = SLUD::get_perm_c_parmetis( &(data_.A), + data_.perm_r.getRawPtr(), data_.perm_c.getRawPtr(), + data_.grid.nprow * data_.grid.npcol, data_.domains, + &(data_.sizes), &(data_.fstVtxSep), + &(data_.grid), &(data_.symb_comm) ); + } TEUCHOS_TEST_FOR_EXCEPTION( info > 0.0, std::runtime_error, "SuperLU_DIST pre-ordering ran out of memory after allocating " @@ -389,18 +389,18 @@ namespace Amesos2 { if( in_grid_ ){ + float info = 0.0; + { #ifdef HAVE_AMESOS2_TIMERS - Teuchos::TimeMonitor symFactTime( this->timers_.symFactTime_ ); + Teuchos::TimeMonitor symFactTime( this->timers_.symFactTime_ ); #endif - - float info = 0.0; - info = SLUD::symbfact_dist((data_.grid.nprow) * (data_.grid.npcol), - data_.domains, &(data_.A), data_.perm_c.getRawPtr(), - data_.perm_r.getRawPtr(), data_.sizes, - data_.fstVtxSep, &(data_.pslu_freeable), - &(data_.grid.comm), &(data_.symb_comm), - &(data_.mem_usage)); - + info = SLUD::symbfact_dist((data_.grid.nprow) * (data_.grid.npcol), + data_.domains, &(data_.A), data_.perm_c.getRawPtr(), + data_.perm_r.getRawPtr(), data_.sizes, + data_.fstVtxSep, &(data_.pslu_freeable), + &(data_.grid.comm), &(data_.symb_comm), + &(data_.mem_usage)); + } TEUCHOS_TEST_FOR_EXCEPTION( info > 0.0, std::runtime_error, "SuperLU_DIST symbolic factorization ran out of memory after" @@ -420,17 +420,25 @@ namespace Amesos2 { // loadA_impl(); // Refresh the matrix values - // if( data_.options.Equil == SLUD::YES ){ - // // Apply the scalings computed in preOrdering - // function_map::laqgs(&(data_.A), data_.R.getRawPtr(), - // data_.C.getRawPtr(), data_.rowcnd, data_.colcnd, - // data_.amax, &(data_.equed)); + if( in_grid_ ) { + if( data_.options.Equil == SLUD::YES ) { + SLUD::int_t info = 0; - // data_.rowequ = (data_.equed == SLUD::ROW) || (data_.equed == SLUD::BOTH); - // data_.colequ = (data_.equed == SLUD::COL) || (data_.equed == SLUD::BOTH); - // } + // Compute scaling + data_.R.resize(this->globalNumRows_); + data_.C.resize(this->globalNumCols_); + function_map::gsequ_loc(&(data_.A), data_.R.getRawPtr(), data_.C.getRawPtr(), + &(data_.rowcnd), &(data_.colcnd), &(data_.amax), &info, &(data_.grid)); + + // Apply the scalings + function_map::laqgs_loc(&(data_.A), data_.R.getRawPtr(), data_.C.getRawPtr(), + data_.rowcnd, data_.colcnd, data_.amax, + &(data_.equed)); + + data_.rowequ = (data_.equed == SLUD::ROW) || (data_.equed == SLUD::BOTH); + data_.colequ = (data_.equed == SLUD::COL) || (data_.equed == SLUD::BOTH); + } - if( in_grid_ ){ // Apply the column ordering, so that AC is the column-permuted A, and compute etree size_t nnz_loc = ((SLUD::NRformat_loc*)data_.A.Store)->nnz_loc; for( size_t i = 0; i < nnz_loc; ++i ) colind_[i] = data_.perm_c[colind_[i]]; @@ -462,7 +470,6 @@ namespace Amesos2 { #ifdef HAVE_AMESOS2_TIMERS Teuchos::TimeMonitor numFactTimer(this->timers_.numFactTime_); #endif - function_map::gstrf(&(data_.options), this->globalNumRows_, this->globalNumCols_, anorm, &(data_.lu), &(data_.grid), &(data_.stat), &info); @@ -511,7 +518,6 @@ namespace Amesos2 { #ifdef HAVE_AMESOS2_TIMERS Teuchos::TimeMonitor convTimer(this->timers_.vecConvTime_); #endif - { // The input dense matrix for B should be distributed in the // same manner as the superlu_dist matrix. That is, if a @@ -522,7 +528,6 @@ namespace Amesos2 { #ifdef HAVE_AMESOS2_TIMERS Teuchos::TimeMonitor redistTimer(this->timers_.vecRedistTime_); #endif - // get grid-distributed mv data. The multivector data will be // distributed across the processes in the SuperLU_DIST grid. typedef Util::get_1d_copy_helper,slu_type> copy_helper; @@ -563,12 +568,20 @@ namespace Amesos2 { same_solve_struct_ = true; } + // Apply row-scaling if requested + if (data_.options.Equil == SLUD::YES && data_.rowequ) { + SLUD::int_t ld = as(local_len_rhs); + for(global_size_type j = 0; j < nrhs; ++j) { + for(size_t i = 0; i < local_len_rhs; ++i) bvals_[i + j*ld] *= data_.R[i]; + } + } + + // Solve int ierr = 0; // returned error code { #ifdef HAVE_AMESOS2_TIMERS Teuchos::TimeMonitor solveTimer(this->timers_.solveTime_); #endif - function_map::gstrs(as(this->globalNumRows_), &(data_.lu), &(data_.scale_perm), &(data_.grid), bvals_.getRawPtr(), as(local_len_rhs), as(first_global_row_b), @@ -606,6 +619,14 @@ namespace Amesos2 { as(nrhs), &(data_.grid)); } + + // Apply col-scaling if requested + if (data_.options.Equil == SLUD::YES && data_.colequ) { + SLUD::int_t ld = as(local_len_rhs); + for(global_size_type j = 0; j < nrhs; ++j) { + for(size_t i = 0; i < local_len_rhs; ++i) xvals_[i + j*ld] *= data_.C[i]; + } + } } /* Update X's global values */ @@ -613,7 +634,6 @@ namespace Amesos2 { #ifdef HAVE_AMESOS2_TIMERS Teuchos::TimeMonitor redistTimer(this->timers_.vecRedistTime_); #endif - typedef Util::put_1d_data_helper,slu_type> put_helper; put_helper::do_put(X, xvals_(), @@ -672,10 +692,9 @@ namespace Amesos2 { data_.options.Trans = SLUD::NOTRANS; // should always be set this way; - // TODO: Uncomment when supported - // bool equil = parameterList->get("Equil", true); - // data_.options.Equil = equil ? SLUD::YES : SLUD::NO; - data_.options.Equil = SLUD::NO; + // Equilbration option + bool equil = parameterList->get("Equil", false); + data_.options.Equil = equil ? SLUD::YES : SLUD::NO; if( parameterList->isParameter("ColPerm") ){ RCP colperm_validator = valid_params->getEntry("ColPerm").validator(); @@ -743,8 +762,8 @@ namespace Amesos2 { tuple(SLUD::NOTRANS), pl.getRawPtr()); - // TODO: uncomment when supported - // pl->set("Equil", false, "Whether to equilibrate the system before solve"); + // Equillbration + pl->set("Equil", false, "Whether to equilibrate the system before solve"); // TODO: uncomment when supported // setStringToIntegralParameter("IterRefine", "NOREFINE", From 98bc0c8eca7212c57c9ec874b440c86117896bcf Mon Sep 17 00:00:00 2001 From: iyamazaki Date: Tue, 26 Oct 2021 00:33:41 -0600 Subject: [PATCH 2/2] Amesos2::SuperLU_dist: shift the local row and col scaling by offset --- packages/amesos2/src/Amesos2_Superludist_def.hpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/amesos2/src/Amesos2_Superludist_def.hpp b/packages/amesos2/src/Amesos2_Superludist_def.hpp index 4cd1fea25c5d..ef0ed950d7ef 100644 --- a/packages/amesos2/src/Amesos2_Superludist_def.hpp +++ b/packages/amesos2/src/Amesos2_Superludist_def.hpp @@ -572,7 +572,9 @@ namespace Amesos2 { if (data_.options.Equil == SLUD::YES && data_.rowequ) { SLUD::int_t ld = as(local_len_rhs); for(global_size_type j = 0; j < nrhs; ++j) { - for(size_t i = 0; i < local_len_rhs; ++i) bvals_[i + j*ld] *= data_.R[i]; + for(size_t i = 0; i < local_len_rhs; ++i) { + bvals_[i + j*ld] *= data_.R[first_global_row_b + i]; + } } } @@ -624,7 +626,9 @@ namespace Amesos2 { if (data_.options.Equil == SLUD::YES && data_.colequ) { SLUD::int_t ld = as(local_len_rhs); for(global_size_type j = 0; j < nrhs; ++j) { - for(size_t i = 0; i < local_len_rhs; ++i) xvals_[i + j*ld] *= data_.C[i]; + for(size_t i = 0; i < local_len_rhs; ++i) { + xvals_[i + j*ld] *= data_.C[first_global_row_b + i]; + } } } }