diff --git a/.gitmodules b/.gitmodules index 7798b4b9a2..cd4a79fb45 100644 --- a/.gitmodules +++ b/.gitmodules @@ -99,8 +99,11 @@ fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git [submodule "share"] path = share -url = https://github.com/ESCOMP/CESM_share -fxtag = share1.1.9 +#url = https://github.com/ESCOMP/CESM_share +url = https://github.com/ekluzek/CESM_share +#fxtag = share1.1.9 +#fxtag = add_jdennis_procstatus_module +fxtag = 9973692556da54f9562935be43c1d43b0607d24b fxrequired = ToplevelRequired # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed fxDONOTUSEurl = https://github.com/ESCOMP/CESM_share diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml index 03c9ba420e..ed85f4b407 100644 --- a/bld/namelist_files/namelist_definition_ctsm.xml +++ b/bld/namelist_files/namelist_definition_ctsm.xml @@ -1242,12 +1242,23 @@ Whether to use subgrid fluxes for snow Whether snow on the vegetation canopy affects the radiation/albedo calculations + +Whether to exit early after the initialization self tests are run. This is typically only used in automated tests. + + Whether to run some tests of ncdio_pio as part of the model run. This is typically only used in automated tests. + +Whether to run some tests of decompInit (to get the gridcell to MPI task decomposition) as part of the model run. This is +typically only used in automated tests. + + If true, allocate memory for and use a second crop grain pool. This is diff --git a/cime_config/config_pes.xml b/cime_config/config_pes.xml index bb10b8019c..d0794339db 100644 --- a/cime_config/config_pes.xml +++ b/cime_config/config_pes.xml @@ -2092,6 +2092,44 @@ + + + + + none + + -1 + -80 + -80 + -80 + -80 + -80 + -80 + -80 + -80 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index ac35ad812e..51f85ba8f3 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -363,11 +363,4 @@ - - - FAIL - #3316 - - - diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 478dc59bdd..1a53e25ae9 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -14,6 +14,7 @@ matrixcn: Tests exercising the matrix-CN capability aux_clm_mpi_serial: aux_clm tests using mpi-serial. Useful for redoing tests that failed due to https://github.com/ESCOMP/CTSM/issues/2916, after having replaced libraries/mpi-serial with a fresh copy. decomp_init: Initialization tests specifically for examining the PE layout decomposition initialization + uhr_decomp_init: Initialization tests at Ultra High Resolution - specifically for examining the PE layout decomposition initialization --> @@ -4209,6 +4210,15 @@ + + + + + + + + + diff --git a/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm new file mode 100644 index 0000000000..bfdcfb115b --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm @@ -0,0 +1,6 @@ +! Settings currently required to run at the mpasa3p75 grid +! urbantv files at that resolution and use a redistribution mapping + +stream_fldfilename_urbantv = '/glade/derecho/scratch/bdobbins/ko/tbuildmax.nc' +stream_meshfile_urbantv = '/glade/derecho/scratch/bdobbins/ko/mesh.nc' +urbantvmapalgo = 'redist' diff --git a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/README b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/README index 938dffbe6f..985b2bfae4 100644 --- a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/README +++ b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/README @@ -1,5 +1,10 @@ -The purpose of this testmod directory is to trigger the runtime -self-tests. This runs a suite of unit/integration tests. +The purpose of this testmod directory is to trigger runtime +initialization self-tests. This runs a set of unit/integration tests +that apply at initialization. We use cold start so that we can get through initialization faster, since how we initialize the model is unimportant for these self-tests. +We also exit as early as possible to minimize the time spent. + +There are other self_tests that need to be exercised in the model time stepping +and are done outside of these. diff --git a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands index 7762f69e36..0d8a5d36e1 100755 --- a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands @@ -4,6 +4,9 @@ # We use this testmod in a _Ln1 test; this requires forcing the ROF coupling frequency to same frequency as DATM ./xmlchange ROF_NCPL='$ATM_NCPL' +# Turn off ROF model when used with compsets that have them +./xmlchange RTM_MODE='NULL' + # Turn MEGAN off to run faster ./xmlchange CLM_BLDNML_OPTS='--no-megan' --append @@ -11,5 +14,15 @@ ./xmlchange CLM_STRUCTURE="fast" ./xmlchange CLM_CONFIGURATION="nwp" -# Turn cpl history off -./xmlchange HIST_OPTION="never" \ No newline at end of file +# Restarts aren't allowed for these tests, and turn off CPL history +# First change in env_test.xml, then in the standard one so it won't complain there +./xmlchange --force REST_OPTION="never" --file env_test.xml +./xmlchange --force HIST_OPTION="never" --file env_test.xml +./xmlchange REST_OPTION="never" +./xmlchange HIST_OPTION="never" + +# Timer settings +./xmlchange TIMER_DETAIL="2" +./xmlchange SAVE_TIMING="TRUE" +./xmlchange CHECK_TIMING="TRUE" +./xmlchange ESMF_PROFILING_LEVEL="10" diff --git a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm index 6187386336..499770f153 100644 --- a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm @@ -1 +1,9 @@ -for_testing_run_ncdiopio_tests = .true. +for_testing_run_ncdiopio_tests = .false. +for_testing_run_decomp_init_tests = .true. +for_testing_exit_after_self_tests = .true. + +! Turn off history, restarts, and output +hist_empty_htapes = .true. +use_noio = .true. +urban_hac = 'OFF' +toosmall_urban = 98.0d00 ! Minimize urban in gridcells diff --git a/share b/share index 14338bef3f..9973692556 160000 --- a/share +++ b/share @@ -1 +1 @@ -Subproject commit 14338bef3fa604d49160e376257264db1d3313e5 +Subproject commit 9973692556da54f9562935be43c1d43b0607d24b diff --git a/src/cpl/nuopc/lnd_comp_nuopc.F90 b/src/cpl/nuopc/lnd_comp_nuopc.F90 index af1426bf9b..7245954b4d 100644 --- a/src/cpl/nuopc/lnd_comp_nuopc.F90 +++ b/src/cpl/nuopc/lnd_comp_nuopc.F90 @@ -39,6 +39,7 @@ module lnd_comp_nuopc use clm_varctl , only : single_column, clm_varctl_set, iulog use clm_varctl , only : nsrStartup, nsrContinue, nsrBranch use clm_varctl , only : FL => fname_len + use clm_varctl , only : for_testing_exit_after_self_tests use clm_time_manager , only : set_timemgr_init, advance_timestep use clm_time_manager , only : update_rad_dtime use clm_time_manager , only : get_nstep, get_step_size @@ -80,6 +81,7 @@ module lnd_comp_nuopc logical :: glc_present logical :: rof_prognostic + logical :: atm_present logical :: atm_prognostic integer, parameter :: dbug = 0 character(*),parameter :: modName = "(lnd_comp_nuopc)" @@ -284,6 +286,11 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) else atm_prognostic = .true. end if + if (trim(atm_model) == 'satm') then + atm_present = .false. + else + atm_present = .true. + end if call NUOPC_CompAttributeGet(gcomp, name='GLC_model', value=glc_model, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return if (trim(glc_model) == 'sglc') then @@ -310,6 +317,9 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) write(iulog,'(a )')' rof component = '//trim(rof_model) write(iulog,'(a )')' glc component = '//trim(glc_model) write(iulog,'(a,L2)')' atm_prognostic = ',atm_prognostic + if (.not. atm_present) then + write(iulog,'(a,L2)')' atm_present = ',atm_present + end if write(iulog,'(a,L2)')' rof_prognostic = ',rof_prognostic write(iulog,'(a,L2)')' glc_present = ',glc_present if (glc_present) then @@ -328,7 +338,8 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) call control_setNL("lnd_in"//trim(inst_suffix)) - call advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, rof_prognostic, atm_prognostic, rc) + call advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, rof_prognostic, & + atm_prognostic, atm_present, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return !---------------------------------------------------------------------------- @@ -492,6 +503,12 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) else single_column = .false. end if + if ( for_testing_exit_after_self_tests) then + ! ******************* + ! *** RETURN HERE *** + ! ******************* + RETURN + end if !---------------------------------------------------------------------------- ! Reset shr logging to my log file @@ -771,6 +788,9 @@ subroutine ModelAdvance(gcomp, rc) if (single_column .and. .not. scol_valid) then RETURN end if + if (for_testing_exit_after_self_tests) then + RETURN + end if !$ call omp_set_num_threads(nthrds) @@ -1002,6 +1022,7 @@ subroutine ModelSetRunClock(gcomp, rc) rc = ESMF_SUCCESS call ESMF_LogWrite(subname//' called', ESMF_LOGMSG_INFO) if (.not. scol_valid) return + if (for_testing_exit_after_self_tests) return ! query the Component for its clocks call NUOPC_ModelGet(gcomp, driverClock=dclock, modelClock=mclock, rc=rc) @@ -1285,6 +1306,7 @@ subroutine clm_orbital_update(clock, logunit, mastertask, eccen, obliqr, lambm0 end subroutine clm_orbital_update subroutine CheckImport(gcomp, rc) + use clm_varctl, only : for_testing_exit_after_self_tests type(ESMF_GridComp) :: gcomp integer, intent(out) :: rc character(len=*) , parameter :: subname = "("//__FILE__//":CheckImport)" @@ -1313,6 +1335,9 @@ subroutine CheckImport(gcomp, rc) if (single_column .and. .not. scol_valid) then RETURN end if + if (for_testing_exit_after_self_tests) then + RETURN + end if ! The remander of this should be equivalent to the NUOPC internal routine ! from NUOPC_ModeBase.F90 diff --git a/src/cpl/nuopc/lnd_import_export.F90 b/src/cpl/nuopc/lnd_import_export.F90 index 624590b9a6..b8a5efeb8d 100644 --- a/src/cpl/nuopc/lnd_import_export.F90 +++ b/src/cpl/nuopc/lnd_import_export.F90 @@ -156,7 +156,8 @@ module lnd_import_export contains !=============================================================================== - subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, rof_prognostic, atm_prognostic, rc) + subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, rof_prognostic, & + atm_prognostic, atm_present, rc) use shr_carma_mod , only : shr_carma_readnl use shr_ndep_mod , only : shr_ndep_readnl @@ -173,6 +174,7 @@ subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, r logical , intent(in) :: cism_evolve logical , intent(in) :: rof_prognostic logical , intent(in) :: atm_prognostic + logical , intent(in) :: atm_present integer , intent(out) :: rc ! local variables @@ -210,7 +212,9 @@ subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, r ! Need to determine if there is no land for single column before the advertise call is done - if (atm_prognostic .or. force_send_to_atm) then + if (.not. atm_present)then + send_to_atm = .false. + else if (atm_prognostic .or. force_send_to_atm) then send_to_atm = .true. else send_to_atm = .false. @@ -253,12 +257,11 @@ subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, r if (shr_megan_mechcomps_n .ne. megan_nflds) call shr_sys_abort('ERROR: megan field count mismatch') ! CARMA volumetric soil water from land - call shr_carma_readnl('drv_flds_in', carma_fields) ! export to atm call fldlist_add(fldsFrLnd_num, fldsFrlnd, trim(flds_scalar_name)) - call fldlist_add(fldsFrLnd_num, fldsFrlnd, 'Sl_lfrin') if (send_to_atm) then + call fldlist_add(fldsFrLnd_num, fldsFrlnd, 'Sl_lfrin') call fldlist_add(fldsFrLnd_num, fldsFrlnd, Sl_t ) call fldlist_add(fldsFrLnd_num, fldsFrlnd, Sl_tref ) call fldlist_add(fldsFrLnd_num, fldsFrlnd, Sl_qref ) @@ -339,6 +342,9 @@ subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, r call fldlist_add(fldsToLnd_num, fldsToLnd, trim(flds_scalar_name)) + !!!!!!!!!!!!!!!!!!!!!!!!!!! new if section !!!!!!!!!!!!!!!!!!!!!!!!!! + if ( atm_present ) then + ! from atm call fldlist_add(fldsToLnd_num, fldsToLnd, Sa_z ) call fldlist_add(fldsToLnd_num, fldsToLnd, Sa_topo ) @@ -389,6 +395,9 @@ subroutine advertise_fields(gcomp, flds_scalar_name, glc_present, cism_evolve, r call fldlist_add(fldsToLnd_num, fldsToLnd, Sa_co2diag) end if + end if ! atm_present + !!!!!!!!!!!!!!!!!!!!!!!!!!! new if section !!!!!!!!!!!!!!!!!!!!!!!!!! + if (rof_prognostic) then ! from river call fldlist_add(fldsToLnd_num, fldsToLnd, Flrr_flood ) @@ -773,6 +782,7 @@ subroutine export_fields( gcomp, bounds, glc_present, rof_prognostic, & ! output to mediator ! ----------------------- + if (send_to_atm) then call state_setexport_1d(exportState, Sl_lfrin, ldomain%frac(begg:), init_spval=.false., rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -780,7 +790,6 @@ subroutine export_fields( gcomp, bounds, glc_present, rof_prognostic, & ! output to atm ! ----------------------- - if (send_to_atm) then call state_setexport_1d(exportState, Sl_t , lnd2atm_inst%t_rad_grc(begg:), & init_spval=.true., rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return diff --git a/src/cpl/share_esmf/UrbanTimeVarType.F90 b/src/cpl/share_esmf/UrbanTimeVarType.F90 index 1e6d004e96..2870ae9b5c 100644 --- a/src/cpl/share_esmf/UrbanTimeVarType.F90 +++ b/src/cpl/share_esmf/UrbanTimeVarType.F90 @@ -16,6 +16,7 @@ module UrbanTimeVarType use clm_varcon , only : spval use LandunitType , only : lun use GridcellType , only : grc + use perf_mod , only : t_startf, t_stopf ! implicit none private @@ -143,6 +144,8 @@ subroutine urbantv_init(this, bounds, NLFilename) stream_meshfile_urbantv, & urbantv_tintalgo + call t_startf("urbantv_init") + ! Default values for namelist stream_year_first_urbantv = 1 ! first year in stream to use stream_year_last_urbantv = 1 ! last year in stream to use @@ -195,6 +198,7 @@ subroutine urbantv_init(this, bounds, NLFilename) endif ! Initialize the cdeps data type this%sdat_urbantv + call t_startf("str_data_init") call shr_strdata_init_from_inline(this%sdat_urbantv, & my_task = iam, & logunit = iulog, & @@ -219,6 +223,9 @@ subroutine urbantv_init(this, bounds, NLFilename) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) then call ESMF_Finalize(endflag=ESMF_END_ABORT) end if + call t_stopf("str_data_init") + + call t_stopf("urbantv_init") end subroutine urbantv_init diff --git a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 index 0b066ceb5b..01580db90d 100644 --- a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 +++ b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 @@ -20,6 +20,7 @@ module lnd_set_decomp_and_domain use spmdMod , only : masterproc, mpicom use clm_varctl , only : iulog, inst_suffix, FL => fname_len use abortutils , only : endrun + use perf_mod , only : t_startf, t_stopf implicit none private ! except @@ -55,6 +56,9 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes use decompMod , only : gindex_global, bounds_type, get_proc_bounds use clm_varpar , only : nlevsoi use clm_varctl , only : use_soil_moisture_streams + use proc_status_vm, only : prt_vm_status, shr_malloc_trim + use shr_mem_mod , only : shr_mem_getusage + use shr_sys_mod , only : shr_sys_flush ! input/output variables character(len=*) , intent(in) :: driver ! cmeps or lilac @@ -68,7 +72,7 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes ! local variables type(ESMF_Mesh) :: mesh_maskinput type(ESMF_Mesh) :: mesh_lndinput - type(ESMF_DistGrid) :: distgrid_ctsm + type(ESMF_DistGrid) :: distgrid_ctsm ! This appears to be local but is used later in lnd_import_export type(ESMF_Field) :: field_lnd type(ESMF_Field) :: field_ctsm type(ESMF_RouteHandle) :: rhandle_lnd2ctsm @@ -85,8 +89,8 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes real(r8) , pointer :: lndfrac_glob(:) real(r8) , pointer :: lndfrac_loc_input(:) real(r8) , pointer :: dataptr1d(:) + real(r8) :: msize, mrss !------------------------------------------------------------------------------- - rc = ESMF_SUCCESS ! Write diag info @@ -102,10 +106,18 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes write(iulog,*) end if + if(masterproc) then + call prt_vm_status('CTSM: lnd_set_decomp_and_domain_from_readmesh: before allocate') + call shr_mem_getusage( msize, mrss, prt=.true.) + write(iulog,*) 'msize, mrss = ',msize, mrss + call shr_sys_flush(iulog) + endif + ! Determine global 2d sizes from read of dimensions of surface dataset and allocate global memory call lnd_get_global_dims(ni, nj, gsize, isgrid2d) ! Read in the land mesh from the file + call t_startf('lnd_set_decomp_and_domain_from_readmesh: ESMF mesh') mesh_lndinput = ESMF_MeshCreate(filename=trim(meshfile_lnd), fileformat=ESMF_FILEFORMAT_ESMFMESH, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -142,6 +154,7 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes else call shr_sys_abort('driver '//trim(driver)//' is not supported, must be lilac or cmeps') end if + call t_stopf('lnd_set_decomp_and_domain_from_readmesh: ESMF mesh') ! Determine lnd decomposition that will be used by ctsm from lndmask_glob call decompInit_lnd(lni=ni, lnj=nj, amask=lndmask_glob) @@ -190,6 +203,8 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes end do ! Generate a new mesh on the gindex decomposition + ! NOTE: The distgrid_ctsm will be used later in lnd_import_export, even though it appears to just be local + call t_startf('lnd_set_decomp_and_domain_from_readmesh: ESMF mesh on new decomposition') distGrid_ctsm = ESMF_DistGridCreate(arbSeqIndexList=gindex_ctsm, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return mesh_ctsm = ESMF_MeshCreate(mesh_lndinput, elementDistGrid=distgrid_ctsm, rc=rc) @@ -198,6 +213,7 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes ! Set ldomain%lonc, ldomain%latc and ldomain%area call lnd_set_ldomain_gridinfo_from_mesh(mesh_ctsm, vm, gindex_ctsm, begg, endg, isgrid2d, ni, nj, ldomain, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + call t_stopf('lnd_set_decomp_and_domain_from_readmesh: ESMF mesh on new decomposition') ! Set ldomain%lfrac ! Create fields on the input decomp and ctsm decomp @@ -206,6 +222,7 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes ! Redistribute field_lnd to field_ctsm ! Determine ldomain%frac using ctsm decomposition + call t_startf('lnd_set_decomp_and_domain_from_readmesh: land frac') if (trim(driver) == 'cmeps') then if (trim(meshfile_mask) /= trim(meshfile_lnd)) then @@ -245,11 +262,64 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes deallocate(lndfrac_glob) end if + call t_stopf('lnd_set_decomp_and_domain_from_readmesh: land frac') + if(masterproc) then + call prt_vm_status('CTSM: lnd_set_decomp_and_domain_from_readmesh: just before deallocates') + call shr_mem_getusage( msize, mrss, prt=.true.) + write(iulog,*) 'msize, mrss = ',msize, mrss + call shr_sys_flush(iulog) + endif + + ! Deallocate local pointer memory including ESMF objects + call from_readmesh_dealloc( rc ) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call shr_malloc_trim() + + if(masterproc) then + call prt_vm_status('CTSM: lnd_set_decomp_and_domain_from_readmesh: after deallocates') + call shr_mem_getusage( msize, mrss, prt=.true.) + write(iulog,*) 'msize, mrss = ',msize, mrss + call shr_sys_flush(iulog) + endif + + + !=============================================================================== + ! Internal subroutines for this subroutine + contains + !=============================================================================== + + subroutine from_readmesh_dealloc( rc ) + use ESMF, only : ESMF_FieldRedistRelease, ESMF_DistGridDestroy, ESMF_FieldDestroy, ESMF_MeshDestroy + integer, intent(out) :: rc ! ESMF return code to indicate deallocate was successful + + logical :: no_esmf_garbage = .true. ! If .true. release all ESMF data (which can be problematic if referenced again) + + rc = ESMF_SUCCESS + + deallocate(gindex_lnd) + deallocate(gindex_ocn) + deallocate(gindex_ctsm) + ! Destroy or release all of the ESMF objects + call ESMF_FieldRedistRelease( rhandle_lnd2ctsm, noGarbage=no_esmf_garbage, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + !-------------------------------------------------------------------------- + ! NOTE: We can't destroy the distgrid -- because it will be used later + ! As such we don't do the following... EBK 08/01/2025 + !call ESMF_DistGridDestroy( distgrid_ctsm, rc=rc) + !if (chkerr(rc,__LINE__,u_FILE_u)) return + !-------------------------------------------------------------------------- + call ESMF_FieldDestroy( field_lnd, noGarbage=no_esmf_garbage, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_FieldDestroy( field_ctsm, noGarbage=no_esmf_garbage, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_MeshDestroy( mesh_maskinput, noGarbage=no_esmf_garbage, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_MeshDestroy( mesh_lndinput, noGarbage=no_esmf_garbage, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return - ! Deallocate local pointer memory - deallocate(gindex_lnd) - deallocate(gindex_ocn) - deallocate(gindex_ctsm) + end subroutine from_readmesh_dealloc + + !------------------------------------------------------------------------------- end subroutine lnd_set_decomp_and_domain_from_readmesh diff --git a/src/main/clm_initializeMod.F90 b/src/main/clm_initializeMod.F90 index da8185be31..6647af0dda 100644 --- a/src/main/clm_initializeMod.F90 +++ b/src/main/clm_initializeMod.F90 @@ -144,6 +144,7 @@ subroutine initialize2(ni,nj, currtime) use clm_varctl , only : use_hillslope use clm_varorb , only : eccen, mvelpp, lambm0, obliqr use clm_varctl , only : use_cropcal_streams + use clm_varctl , only : use_noio use landunit_varcon , only : landunit_varcon_init, max_lunit, numurbl use pftconMod , only : pftcon use decompInitMod , only : decompInit_clumps, decompInit_glcp @@ -225,6 +226,7 @@ subroutine initialize2(ni,nj, currtime) call t_startf('clm_init2') + call t_startf('clm_init2_part1') ! Get processor bounds for gridcells call get_proc_bounds(bounds_proc) begg = bounds_proc%begg; endg = bounds_proc%endg @@ -277,12 +279,14 @@ subroutine initialize2(ni,nj, currtime) call CLMFatesGlobals2() end if + call t_stopf('clm_init2_part1') ! Determine decomposition of subgrid scale landunits, columns, patches call t_startf('clm_decompInit_clumps') call decompInit_clumps(ni, nj, glc_behavior) call t_stopf('clm_decompInit_clumps') + call t_startf('clm_init2_subgrid') ! *** Get ALL processor bounds - for gridcells, landunit, columns and patches *** call get_proc_bounds(bounds_proc) @@ -304,12 +308,14 @@ subroutine initialize2(ni,nj, currtime) call initGridCells(bounds_clump, glc_behavior) end do !$OMP END PARALLEL DO + call t_stopf('clm_init2_subgrid') ! Set global seg maps for gridcells, landlunits, columns and patches call t_startf('clm_decompInit_glcp') call decompInit_glcp(ni, nj, glc_behavior) call t_stopf('clm_decompInit_glcp') + call t_startf('clm_init2_part2') if (use_hillslope) then ! Initialize hillslope properties call InitHillslope(bounds_proc, hillslope_file) @@ -369,20 +375,20 @@ subroutine initialize2(ni,nj, currtime) if (use_fates) call CLMFatesTimesteps() ! Initialize daylength from the previous time step (needed so prev_dayl can be set correctly) - call t_startf('init_orbd') calday = get_curr_calday(reuse_day_365_for_day_366=.true.) call shr_orb_decl( calday, eccen, mvelpp, lambm0, obliqr, declin, eccf ) dtime = get_step_size_real() caldaym1 = get_curr_calday(offset=-int(dtime), reuse_day_365_for_day_366=.true.) call shr_orb_decl( caldaym1, eccen, mvelpp, lambm0, obliqr, declinm1, eccf ) - call t_stopf('init_orbd') call InitDaylength(bounds_proc, declin=declin, declinm1=declinm1, obliquity=obliqr) + call t_stopf('clm_init2_part2') + call t_startf('clm_init2_part3') ! Initialize Balance checking (after time-manager) call BalanceCheckInit() ! History file variables - if (use_cn) then + if (use_cn .and. .not. use_noio ) then call hist_addfld1d (fname='DAYL', units='s', & avgflag='A', long_name='daylength', & ptr_gcell=grc%dayl, default='inactive') @@ -398,39 +404,45 @@ subroutine initialize2(ni,nj, currtime) ! First put in history calls for subgrid data structures - these cannot appear in the ! module for the subgrid data definition due to circular dependencies that are introduced - data2dptr => col%dz(:,-nlevsno+1:0) - col%dz(bounds_proc%begc:bounds_proc%endc,:) = spval - call hist_addfld2d (fname='SNO_Z', units='m', type2d='levsno', & - avgflag='A', long_name='Snow layer thicknesses', & - ptr_col=data2dptr, no_snow_behavior=no_snow_normal, default='inactive') - - call hist_addfld2d (fname='SNO_Z_ICE', units='m', type2d='levsno', & - avgflag='A', long_name='Snow layer thicknesses (ice landunits only)', & - ptr_col=data2dptr, no_snow_behavior=no_snow_normal, & - l2g_scale_type='ice', default='inactive') - - col%zii(bounds_proc%begc:bounds_proc%endc) = spval - call hist_addfld1d (fname='ZII', units='m', & - avgflag='A', long_name='convective boundary height', & - ptr_col=col%zii, default='inactive') + if ( .not. use_noio )then + data2dptr => col%dz(:,-nlevsno+1:0) + col%dz(bounds_proc%begc:bounds_proc%endc,:) = spval + call hist_addfld2d (fname='SNO_Z', units='m', type2d='levsno', & + avgflag='A', long_name='Snow layer thicknesses', & + ptr_col=data2dptr, no_snow_behavior=no_snow_normal, default='inactive') + + call hist_addfld2d (fname='SNO_Z_ICE', units='m', type2d='levsno', & + avgflag='A', long_name='Snow layer thicknesses (ice landunits only)', & + ptr_col=data2dptr, no_snow_behavior=no_snow_normal, & + l2g_scale_type='ice', default='inactive') + + col%zii(bounds_proc%begc:bounds_proc%endc) = spval + call hist_addfld1d (fname='ZII', units='m', & + avgflag='A', long_name='convective boundary height', & + ptr_col=col%zii, default='inactive') + end if ! Initialize instances of all derived types as well as time constant variables call clm_instInit(bounds_proc) + call t_stopf('clm_init2_part3') + call t_startf('clm_init2_snow_soil_init') call CNParamsSetSoilDepth() ! Initialize SNICAR optical and aging parameters call SnowOptics_init( ) ! SNICAR optical parameters: call SnowAge_init( ) ! SNICAR aging parameters: ! Print history field info to standard out - call hist_printflds() + if ( .not. use_noio )then + call hist_printflds() + end if + call t_stopf('clm_init2_snow_soil_init') + call t_startf('clm_init2_part4') ! Initializate dynamic subgrid weights (for prescribed transient Patches, CNDV ! and/or dynamic landunits); note that these will be overwritten in a restart run - call t_startf('init_dyn_subgrid') call init_subgrid_weights_mod(bounds_proc) call dynSubgrid_init(bounds_proc, glc_behavior, crop_inst) - call t_stopf('init_dyn_subgrid') ! Initialize fates LUH2 usage if (use_fates_luh) then @@ -546,10 +558,12 @@ subroutine initialize2(ni,nj, currtime) call restFile_read(bounds_proc, fnamer, glc_behavior, & reset_dynbal_baselines_lake_columns = reset_dynbal_baselines_lake_columns) end if + call t_stopf('clm_init2_part4') ! If appropriate, create interpolated initial conditions if (nsrest == nsrStartup .and. finidat_interp_source /= ' ') then + call t_startf('clm_init2_init_interp') ! Check that finidat is not cold start - abort if it is if (finidat /= ' ') then call endrun(msg='ERROR clm_initializeMod: '//& @@ -599,8 +613,10 @@ subroutine initialize2(ni,nj, currtime) close(iun) write(iulog,'(a)')' Successfully wrote finidat status file '//trim(locfn) end if + call t_stopf('clm_init2_init_interp') end if + call t_startf('clm_init2_part5') ! If requested, reset dynbal baselines ! This needs to happen after reading the restart file (including after reading the ! interpolated restart file, if applicable). @@ -773,7 +789,6 @@ subroutine initialize2(ni,nj, currtime) deallocate(topo_glc_mec, fert_cft, irrig_method) ! Write log output for end of initialization - call t_startf('init_wlog') if (masterproc) then write(iulog,*) 'Successfully initialized the land model' if (nsrest == nsrStartup) then @@ -788,15 +803,17 @@ subroutine initialize2(ni,nj, currtime) write(iulog,'(72a1)') ("*",i=1,60) write(iulog,*) endif - call t_stopf('init_wlog') + call t_stopf('clm_init2_part5') if (water_inst%DoConsistencyCheck()) then + call t_startf('tracer_consistency_check') !$OMP PARALLEL DO PRIVATE (nc, bounds_clump) do nc = 1,nclumps call get_clump_bounds(nc, bounds_clump) call water_inst%TracerConsistencyCheck(bounds_clump, 'end of initialization') end do !$OMP END PARALLEL DO + call t_stopf('tracer_consistency_check') end if call t_stopf('clm_init2') diff --git a/src/main/clm_instMod.F90 b/src/main/clm_instMod.F90 index 7d9a0f6ad2..bc86749d06 100644 --- a/src/main/clm_instMod.F90 +++ b/src/main/clm_instMod.F90 @@ -229,6 +229,7 @@ subroutine clm_instInit(bounds) integer :: dummy_to_make_pgi_happy !---------------------------------------------------------------------- + call t_startf('clm_instInit_part1') ! Note: h2osno_col and snow_depth_col are initialized as local variables ! since they are needed to initialize vertical data structures @@ -286,6 +287,9 @@ subroutine clm_instInit(bounds) call setSoilLayerClass(bounds) endif + call t_stopf('clm_instInit_part1') + + call t_startf('clm_instInit_part2') !----------------------------------------------- ! Set cold-start values for snow levels, snow layers and snow interfaces !----------------------------------------------- @@ -338,6 +342,10 @@ subroutine clm_instInit(bounds) call glacier_smb_inst%Init(bounds) + call t_stopf('clm_instInit_part2') + + call t_startf('clm_instInit_part3') + ! COMPILER_BUG(wjs, 2014-11-29, pgi 14.7) Without the following assignment, the ! assertion in energyflux_inst%Init fails with pgi 14.7 on yellowstone, presumably due ! to a compiler bug. @@ -473,6 +481,7 @@ subroutine clm_instInit(bounds) deallocate (h2osno_col) deallocate (snow_depth_col) deallocate (exice_init_conc_col) + call t_stopf('clm_instInit_part3') ! ------------------------------------------------------------------------ ! Initialize accumulated fields diff --git a/src/main/clm_varctl.F90 b/src/main/clm_varctl.F90 index 41978ae695..e51ce4d33c 100644 --- a/src/main/clm_varctl.F90 +++ b/src/main/clm_varctl.F90 @@ -52,6 +52,12 @@ module clm_varctl ! true => run tests of ncdio_pio logical, public :: for_testing_run_ncdiopio_tests = .false. + ! true => run tests of decompInit + logical, public :: for_testing_run_decomp_init_tests = .false. + + ! true => exit after the self-tests run + logical, public :: for_testing_exit_after_self_tests = .false. + ! true => allocate memory for and use a second grain pool. This is meant only for ! software testing of infrastructure to support the AgSys crop model integration. This ! option can be dropped once AgSys is integrated and we have tests of it. diff --git a/src/main/controlMod.F90 b/src/main/controlMod.F90 index 6d363a9a6e..51bc9ce5c0 100644 --- a/src/main/controlMod.F90 +++ b/src/main/controlMod.F90 @@ -211,6 +211,7 @@ subroutine control_init(dtime) snow_thermal_cond_lake_method, snow_cover_fraction_method, & irrigate, run_zero_weight_urban, all_active, & crop_fsat_equals_zero, for_testing_run_ncdiopio_tests, & + for_testing_run_decomp_init_tests, for_testing_exit_after_self_tests, & for_testing_use_second_grain_pool, for_testing_use_repr_structure_pool, & for_testing_no_crop_seed_replenishment, & z0param_method, use_z0m_snowmelt @@ -764,8 +765,11 @@ subroutine control_spmd() ! Crop saturated excess runoff call mpi_bcast(crop_fsat_equals_zero, 1, MPI_LOGICAL, 0, mpicom, ier) - ! Whether to run tests of ncdio_pio + ! Whether to run self tests call mpi_bcast(for_testing_run_ncdiopio_tests, 1, MPI_LOGICAL, 0, mpicom, ier) + call mpi_bcast(for_testing_run_decomp_init_tests, 1, MPI_LOGICAL, 0, mpicom, ier) + + call mpi_bcast(for_testing_exit_after_self_tests, 1, MPI_LOGICAL, 0, mpicom, ier) ! Various flags used for testing infrastructure for having multiple crop reproductive pools call mpi_bcast(for_testing_use_second_grain_pool, 1, MPI_LOGICAL, 0, mpicom, ier) diff --git a/src/main/decompInitMod.F90 b/src/main/decompInitMod.F90 index bebcd9d358..187d074d63 100644 --- a/src/main/decompInitMod.F90 +++ b/src/main/decompInitMod.F90 @@ -12,12 +12,14 @@ module decompInitMod use spmdMod , only : masterproc, iam, npes, mpicom use abortutils , only : endrun use clm_varctl , only : iulog + use proc_status_vm, only : prt_vm_status + use shr_mem_mod , only : shr_mem_init, shr_mem_getusage ! implicit none private ! ! !PUBLIC TYPES: - ! + !b ! !PUBLIC MEMBER FUNCTIONS: public :: decompInit_lnd ! initializes lnd grid decomposition into clumps and processors public :: decompInit_clumps ! initializes atm grid decomposition into clumps @@ -34,6 +36,7 @@ module decompInitMod integer, parameter :: dbug=0 ! 0 = min, 1=normal, 2=much, 3=max character(len=*), parameter :: sourcefile = & __FILE__ + real(r8) :: msize, mrss ! memory usage variables #include ! mpi library include file !------------------------------------------------------------------------------ @@ -72,8 +75,17 @@ subroutine decompInit_lnd(lni, lnj, amask) integer, pointer :: clumpcnt(:) ! clump index counter integer, allocatable :: gdc2glo(:)! used to create gindex_global type(bounds_type) :: bounds ! contains subgrid bounds data + real(r8) :: msize, mrss !------------------------------------------------------------------------------ + if(masterproc) then + call shr_mem_init(prt=.true.) ! initialize memory tracking + call prt_vm_status('CTSM: decompInit_lnd: before') + call shr_mem_getusage( msize, mrss, prt=.true.) + write(iulog,*) 'msize, mrss = ',msize, mrss + call shr_sys_flush(iulog) + endif + lns = lni * lnj !--- set and verify nclumps --- @@ -291,6 +303,13 @@ subroutine decompInit_lnd(lni, lnj, amask) gindex_global(n-procinfo%begg+1) = gdc2glo(n) enddo + if(masterproc) then + call prt_vm_status('CTSM: decompInit_lnd: afterwards before deallocate') + call shr_mem_getusage( msize, mrss, prt=.true.) + write(iulog,*) 'msize, mrss = ',msize, mrss + call shr_sys_flush(iulog) + endif + deallocate(clumpcnt) deallocate(gdc2glo) @@ -349,6 +368,9 @@ subroutine decompInit_clumps(lni,lnj,glc_behavior) character(len=32), parameter :: subname = 'decompInit_clumps' !------------------------------------------------------------------------------ + if(masterproc) then + call prt_vm_status('CTSM: decompInit_clumps: before') + endif !--- assign gridcells to clumps (and thus pes) --- call get_proc_bounds(bounds) begg = bounds%begg; endg = bounds%endg @@ -471,6 +493,10 @@ subroutine decompInit_clumps(lni,lnj,glc_behavior) endif enddo + if(masterproc) then + call prt_vm_status('CTSM: decompInit_clumps: after before deallocate') + endif + deallocate(allvecg,allvecl) deallocate(lcid) diff --git a/src/main/initVerticalMod.F90 b/src/main/initVerticalMod.F90 index 64383e7a7c..4a1177666e 100644 --- a/src/main/initVerticalMod.F90 +++ b/src/main/initVerticalMod.F90 @@ -29,6 +29,7 @@ module initVerticalMod use ColumnType , only : col use glcBehaviorMod , only : glc_behavior_type use abortUtils , only : endrun + use perf_mod , only : t_startf, t_stopf use ncdio_pio ! ! !PUBLIC TYPES: @@ -189,6 +190,7 @@ subroutine initVertical(bounds, glc_behavior, thick_wall, thick_roof) integer :: jmin_bedrock character(len=*), parameter :: subname = 'initVertical' !------------------------------------------------------------------------ + call t_startf('initVertical') begc = bounds%begc; endc= bounds%endc begl = bounds%begl; endl= bounds%endl @@ -669,6 +671,8 @@ subroutine initVertical(bounds, glc_behavior, thick_wall, thick_roof) call ncd_pio_closefile(ncid) + call t_stopf('initVertical') + end subroutine initVertical !----------------------------------------------------------------------- diff --git a/src/self_tests/SelfTestDriver.F90 b/src/self_tests/SelfTestDriver.F90 index d109a27827..dd44a185df 100644 --- a/src/self_tests/SelfTestDriver.F90 +++ b/src/self_tests/SelfTestDriver.F90 @@ -6,10 +6,6 @@ module SelfTestDriver ! ! See the README file in this directory for a high-level overview of these self-tests. - use clm_varctl, only : for_testing_run_ncdiopio_tests - use decompMod, only : bounds_type - use TestNcdioPio, only : test_ncdio_pio - implicit none private save @@ -32,17 +28,42 @@ subroutine self_test_driver(bounds) ! This subroutine should be called all the time, but each set of self tests is only ! run if the appropriate flag is set. ! + ! !USES: + use clm_varctl, only : for_testing_run_ncdiopio_tests, for_testing_run_decomp_init_tests + use clm_varctl, only : for_testing_exit_after_self_tests, iulog + use decompMod, only : bounds_type + use TestNcdioPio, only : test_ncdio_pio + use ESMF, only : ESMF_LogWrite, ESMF_LOGMSG_INFO, ESMF_Finalize + use shr_sys_mod, only : shr_sys_flush + use spmdMod, only : masterproc ! !ARGUMENTS: type(bounds_type), intent(in) :: bounds ! ! !LOCAL VARIABLES: character(len=*), parameter :: subname = 'self_test_driver' + integer :: ntests = 0 !----------------------------------------------------------------------- if (for_testing_run_ncdiopio_tests) then + ntests = ntests + 1 call test_ncdio_pio(bounds) end if + if (for_testing_run_decomp_init_tests) then + ntests = ntests + 1 + end if + if (for_testing_exit_after_self_tests) then + ! Print out some messaging if we are exiting after self tests. + if ( masterproc ) then + if ( ntests == 0 )then + write(iulog,*) 'WARNING: You are exiting after self tests were run -- but no self tests were run.' + else + write(iulog,*) 'Exiting after running ', ntests, ' self tests.' + end if + call shr_sys_flush(iulog) + call ESMF_LogWrite(' exiting after running self tests', ESMF_LOGMSG_INFO) + end if + end if end subroutine self_test_driver