From e1f35749fd1f3142f0a86823282ecf93bad238a6 Mon Sep 17 00:00:00 2001 From: Tony Craig Date: Mon, 30 Aug 2021 17:17:50 -0700 Subject: [PATCH] Netcdf History Capability (#368) * Add primitive netcdf history capability - Add history_cdf namelist. This turns on netcdf history files. Netcdf history files are written for each column at each timestep for a hardcoded set of fields. There is no control of fields, frequency, or averaging. History filenames are also hardwired in the code. - History netcdf files require the USE_NETCDF C Preprocessor flag. This is turned on by setting ICE_IOTYPE to netcdf in icepack.settings. In addition, the env and Macros files for the machine have to support builds with netcdf. - Add icedrv_history.F90. Implements history capability. - Add set_env.ionetcdf, set_nml.ionetcdf - Add io_suite.ts * update documentation * fix badger machine files * update conda_macos * update documentation * update travis testing to include netcdf * update github actions suites Co-authored-by: Elizabeth Hunke --- .github/workflows/test-icepack.yml | 2 +- .travis.yml | 2 +- configuration/driver/icedrv_RunMod.F90 | 11 +- configuration/driver/icedrv_calendar.F90 | 2 + configuration/driver/icedrv_history.F90 | 423 ++++++++++++++++++ configuration/driver/icedrv_init.F90 | 5 +- configuration/scripts/Makefile | 16 +- configuration/scripts/icepack.build | 4 + configuration/scripts/icepack_in | 1 + .../scripts/machines/Macros.badger_intel | 13 +- .../scripts/machines/Macros.conda_macos | 6 + .../scripts/machines/env.badger_intel | 10 +- .../scripts/machines/environment.yml | 2 + .../scripts/options/set_env.ionetcdf | 1 + .../scripts/options/set_nml.ionetcdf | 1 + configuration/scripts/setup_run_dirs.csh | 2 +- configuration/scripts/tests/io_suite.ts | 2 + doc/source/icepack_index.rst | 1 + doc/source/user_guide/ug_case_settings.rst | 4 +- doc/source/user_guide/ug_implementation.rst | 30 +- doc/source/user_guide/ug_running.rst | 7 +- 21 files changed, 518 insertions(+), 27 deletions(-) create mode 100644 configuration/driver/icedrv_history.F90 create mode 100644 configuration/scripts/options/set_env.ionetcdf create mode 100644 configuration/scripts/options/set_nml.ionetcdf create mode 100644 configuration/scripts/tests/io_suite.ts diff --git a/.github/workflows/test-icepack.yml b/.github/workflows/test-icepack.yml index 883031a00..fe8dc1946 100644 --- a/.github/workflows/test-icepack.yml +++ b/.github/workflows/test-icepack.yml @@ -115,7 +115,7 @@ jobs: - name: run suite run: | cd $HOME/icepack - ./icepack.setup -m conda -e ${{ matrix.envdef }} --suite travis_suite --testid ${{ matrix.os }} + ./icepack.setup -m conda -e ${{ matrix.envdef }} --suite travis_suite,io_suite --testid ${{ matrix.os }} - name: write output run: | cd $HOME/icepack diff --git a/.travis.yml b/.travis.yml index bcbcd0382..12608ff0e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ script: - "./icepack.setup --case trcase --mach travisCI --env gnu --pes 1x1 -s diag1 && sleep 4" - "./icepack.setup --test smoke --testid trtest --mach travisCI --env gnu --pes 1x1 -s diag1 && sleep 4" - - "./icepack.setup --suite travis_suite --testid travisCItest + - "./icepack.setup --suite travis_suite,io_suite --testid travisCItest --mach travisCI --env gnu && cd testsuite.travisCItest && ./results.csh" diff --git a/configuration/driver/icedrv_RunMod.F90 b/configuration/driver/icedrv_RunMod.F90 index 6ed183cd8..5455607d5 100644 --- a/configuration/driver/icedrv_RunMod.F90 +++ b/configuration/driver/icedrv_RunMod.F90 @@ -35,6 +35,7 @@ subroutine icedrv_run use icedrv_forcing, only: get_forcing, get_wave_spec use icedrv_forcing_bgc, only: faero_default, fiso_default, get_forcing_bgc use icedrv_flux, only: init_flux_atm_ocn + use icedrv_history, only: history_cdf, history_close logical (kind=log_kind) :: skl_bgc, z_tracers, tr_aero, tr_zaero, & wave_spec, tr_fsd, tr_iso @@ -61,7 +62,10 @@ subroutine icedrv_run call calendar(time) ! at the end of the timestep - if (stop_now >= 1) exit timeLoop + if (stop_now >= 1) then + if (history_cdf) call history_close() + exit timeLoop + endif call icepack_query_parameters(skl_bgc_out=skl_bgc, z_tracers_out=z_tracers,& wave_spec_out=wave_spec) @@ -97,6 +101,7 @@ subroutine ice_step use icedrv_diagnostics_bgc, only: hbrine_diags, zsal_diags, bgc_diags use icedrv_flux, only: init_history_therm, init_history_bgc, & daidtt, daidtd, dvidtt, dvidtd, dagedtt, dagedtd, init_history_dyn + use icedrv_history, only: history_cdf, history_write use icedrv_restart, only: dumpfile, final_restart use icedrv_restart_bgc, only: write_restart_bgc use icedrv_step, only: prep_radiation, step_therm1, step_therm2, & @@ -219,6 +224,10 @@ subroutine ice_step if (skl_bgc .or. z_tracers) call bgc_diags if (tr_brine) call hbrine_diags endif + + if (history_cdf) then + call history_write() + endif if (write_restart == 1) then call dumpfile ! core variables for restarting diff --git a/configuration/driver/icedrv_calendar.F90 b/configuration/driver/icedrv_calendar.F90 index 126f29e83..dc6642eb4 100644 --- a/configuration/driver/icedrv_calendar.F90 +++ b/configuration/driver/icedrv_calendar.F90 @@ -72,6 +72,7 @@ module icedrv_calendar real (kind=dbl_kind), public :: & dt , & ! thermodynamics timestep (s) dt_dyn , & ! dynamics/transport/ridging timestep (s) + time0 , & ! total elapsed time at istep0 for idate0 (s) time , & ! total elapsed time (s) time_forc , & ! time of last forcing update (s) yday , & ! day of the year @@ -123,6 +124,7 @@ subroutine init_calendar !----------------------------------------------------------------- istep = 0 ! local timestep number + time0=istep0*dt ! start time time=istep0*dt ! s yday=c0 ! absolute day number mday=0 ! day of the month diff --git a/configuration/driver/icedrv_history.F90 b/configuration/driver/icedrv_history.F90 new file mode 100644 index 000000000..6c30e5f91 --- /dev/null +++ b/configuration/driver/icedrv_history.F90 @@ -0,0 +1,423 @@ +!======================================================================= + +! Diagnostic information output during run +! +! authors: T. Craig + + module icedrv_history + + use icedrv_kinds + use icedrv_constants, only: nu_diag, nu_diag_out + use icedrv_domain_size, only: nx, ncat + use icedrv_diagnostics, only: nx_names + use icepack_intfc, only: icepack_warnings_flush, icepack_warnings_aborted + use icepack_intfc, only: icepack_query_parameters, icepack_query_tracer_sizes + use icepack_intfc, only: icepack_query_tracer_flags, icepack_query_tracer_indices + use icedrv_system, only: icedrv_system_abort + + implicit none + private + public :: history_write, & + history_close + + ! history output file info + + logical (kind=log_kind), public :: history_cdf ! flag to turn on cdf history files + + character (len=char_len_long) :: hist_file ! hist file name + + integer (kind=int_kind) :: ncid ! cdf file id + integer (kind=int_kind) :: nxid, ncatid, ntrcrid, timid ! cdf dim ids + integer (kind=int_kind) :: timcnt ! time counter + +!======================================================================= + + contains + +!======================================================================= + +! Writes history information + + subroutine history_write() + + use icedrv_calendar, only: idate0, days_per_year, use_leap_years + use icedrv_calendar, only: time, time0, secday, istep1, idate, sec + use icedrv_state, only: aice, vice, vsno, uvel, vvel, divu, shear, strength + use icedrv_state, only: trcr, trcrn + use icedrv_state, only: aicen, vicen, vsnon + use icedrv_flux, only: evap, fsnow, frain, frazil + use icedrv_flux, only: fswabs, flw, flwout, fsens, fsurf, flat + use icedrv_flux, only: Tair, Qa, fsw, fcondtop + use icedrv_flux, only: meltt, meltb, meltl, snoice + use icedrv_flux, only: dsnow, congel, sst, sss, Tf, fhocn +#ifdef USE_NETCDF + use netcdf +#endif + + ! local variables + + logical (kind=log_kind), save :: & + first_call = .true. ! first call flag + + integer (kind=int_kind) :: & + n, & ! counters + ntrcr, & ! tracer count from icepack + dimid1(1), dimid2(2), dimid3(3), dimid4(4), & ! cdf dimids + start1(1), start2(2), start3(3), start4(4), & ! cdf start/count arrays + count1(1), count2(2), count3(3), count4(4), & ! cdf start/count arrays + varid, & ! cdf varid + status, & ! cdf status flag + iflag ! history file attributes + + character (len=8) :: & + cdate ! date string + + real (kind=dbl_kind) :: & + value ! temporary + real (kind=dbl_kind),allocatable :: & + value1(:), value2(:,:), value3(:,:,:), value4(:,:,:,:) ! temporary + + integer (kind=dbl_kind), parameter :: num_2d = 32 + character(len=16), parameter :: fld_2d(num_2d) = & + (/ 'aice ', 'vice ', 'vsno ', & + 'uvel ', 'vvel ', 'divu ', & + 'shear ', 'strength ', & + 'evap ', 'fsnow ', 'frazil ', & + 'fswabs ', 'flw ', 'flwout ', & + 'fsens ', 'fsurf ', 'flat ', & + 'frain ', 'Tair ', 'Qa ', & + 'fsw ', 'fcondtop ', 'meltt ', & + 'meltb ', 'meltl ', 'snoice ', & + 'dsnow ', 'congel ', 'sst ', & + 'sss ', 'Tf ', 'fhocn ' /) + + integer (kind=dbl_kind), parameter :: num_3d_ncat = 3 + character(len=16), parameter :: fld_3d_ncat(num_3d_ncat) = & + (/ 'aicen ', 'vicen ', 'vsnon ' /) + + integer (kind=dbl_kind), parameter :: num_3d_ntrcr = 1 + character(len=16), parameter :: fld_3d_ntrcr(num_3d_ntrcr) = & + (/ 'trcr ' /) + + integer (kind=dbl_kind), parameter :: num_4d_ncat_ntrcr = 1 + character(len=16), parameter :: fld_4d_ncat_ntrcr(num_4d_ncat_ntrcr) = & + (/ 'trcrn ' /) + + character (len=char_len_long) :: tmpstr + + character(len=*), parameter :: subname='(history_write)' + +#ifdef USE_NETCDF + call icepack_query_tracer_sizes(ntrcr_out=ntrcr) + if (first_call) then + timcnt = 0 + write(hist_file,'(a,i8.8,a)') './history/icepack.h.',idate,'.nc' + iflag = nf90_clobber + status = nf90_create(trim(hist_file),iflag,ncid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: nf90_create '//trim(hist_file)) + + ! nx columns dimension + status = nf90_def_dim(ncid,'ni',nx,nxid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: def_dim ni') + status = nf90_def_var(ncid,'ni',NF90_INT,nxid,varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: def_var ni') + do n = 1,nx + write(tmpstr,'(a,i3.3)') 'column_name_',n + status = nf90_put_att(ncid,varid,trim(tmpstr),trim(nx_names(n))) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_att columns names') + enddo + + ! ncat category dimension + status = nf90_def_dim(ncid,'ncat',ncat,ncatid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: def_dim ncat') + status = nf90_def_var(ncid,'ncat',NF90_INT,ncatid,varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: def_var ncat') + + ! ntrcr dimension + status = nf90_def_dim(ncid,'ntrcr',ntrcr,ntrcrid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: def_dim ntrcr') + status = nf90_def_var(ncid,'ntrcr',NF90_INT,ntrcrid,varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: def_var ntrcr') + + ! time dimension + status = nf90_def_dim(ncid,'time',NF90_UNLIMITED,timid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: def_dim time') + status = nf90_def_var(ncid,'time',NF90_DOUBLE,timid,varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: def_var time') + status = nf90_put_att(ncid,varid,'long_name','model time') + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_att time long_name') + write(cdate,'(i8.8)') idate0 + write(tmpstr,'(a,a,a,a,a,a,a,a)') 'days since ', & + cdate(1:4),'-',cdate(5:6),'-',cdate(7:8),' 00:00:00' + status = nf90_put_att(ncid,varid,'units',trim(tmpstr)) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_att time units') + if (days_per_year == 360) then + status = nf90_put_att(ncid,varid,'calendar','360_day') + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_att calendar 360_day') + elseif (days_per_year == 365 .and. .not.use_leap_years ) then + status = nf90_put_att(ncid,varid,'calendar','NoLeap') + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_att calendar noleap') + elseif (use_leap_years) then + status = nf90_put_att(ncid,varid,'calendar','Gregorian') + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_att calendar gregorian') + else + call icedrv_system_abort(string=subname//' ERROR: invalid calendar settings') + endif + status = nf90_def_var(ncid,'timestep',NF90_INT,timid,varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: def_var timestep') + status = nf90_def_var(ncid,'date',NF90_DOUBLE,timid,varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: def_var date') + + ! 2d fields + + dimid2(1) = nxid + dimid2(2) = timid + + do n = 1,num_2d + status = nf90_def_var(ncid,trim(fld_2d(n)),NF90_DOUBLE,dimid2,varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR in def_var '//trim(fld_2d(n))) + enddo + + ! 3d ncat fields + + dimid3(1) = nxid + dimid3(2) = ncatid + dimid3(3) = timid + + do n = 1,num_3d_ncat + status = nf90_def_var(ncid,trim(fld_3d_ncat(n)),NF90_DOUBLE,dimid3,varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR in def_var '//trim(fld_3d_ncat(n))) + enddo + + ! 3d ntrcr fields + + dimid3(1) = nxid + dimid3(2) = ntrcrid + dimid3(3) = timid + + do n = 1,num_3d_ntrcr + status = nf90_def_var(ncid,trim(fld_3d_ntrcr(n)),NF90_DOUBLE,dimid3,varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR in def_var '//trim(fld_3d_ntrcr(n))) + enddo + + ! 4d ncat ntrcr fields + + dimid4(1) = nxid + dimid4(2) = ncatid + dimid4(3) = ntrcrid + dimid4(4) = timid + + do n = 1,num_4d_ncat_ntrcr + status = nf90_def_var(ncid,trim(fld_4d_ncat_ntrcr(n)),NF90_DOUBLE,dimid4,varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR in def_var '//trim(fld_4d_ncat_ntrcr(n))) + enddo + + ! enddef + + status = nf90_enddef(ncid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR in nf90_enddef') + + ! static dimension variables + + status = nf90_inq_varid(ncid,'ni',varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: inq_var '//'ni') + status = nf90_put_var(ncid,varid,(/(n,n=1,nx)/)) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_var '//'ni') + + status = nf90_inq_varid(ncid,'ncat',varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: inq_var '//'ncat') + status = nf90_put_var(ncid,varid,(/(n,n=1,ncat)/)) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_var '//'ncat') + + status = nf90_inq_varid(ncid,'ntrcr',varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: inq_var '//'ntrcr') + status = nf90_put_var(ncid,varid,(/(n,n=1,ntrcr)/)) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_var '//'ntrcr') + + endif + + first_call = .false. + + ! Time + + timcnt = timcnt + 1 + + status = nf90_inq_varid(ncid,'time',varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: inq_var '//'time') + value = (time-time0)/secday + status = nf90_put_var(ncid,varid,value,start=(/timcnt/)) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_var '//'time') + + status = nf90_inq_varid(ncid,'timestep',varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: inq_var '//'timestep') + status = nf90_put_var(ncid,varid,istep1,start=(/timcnt/)) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_var '//'timestep') + + status = nf90_inq_varid(ncid,'date',varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: inq_var '//'date') + value = real(idate,kind=dbl_kind) + real(sec,kind=dbl_kind)/(secday) + status = nf90_put_var(ncid,varid,value,start=(/timcnt/)) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_var '//'date') + + ! 2d fields + + start2(1) = 1 + count2(1) = nx + start2(2) = timcnt + count2(2) = 1 + + do n = 1,num_2d + allocate(value2(count2(1),1)) + + value2 = -9999._dbl_kind + if (trim(fld_2d(n)) == 'aice') value2(1:count2(1),1) = aice(1:count2(1)) + if (trim(fld_2d(n)) == 'vice') value2(1:count2(1),1) = vice(1:count2(1)) + if (trim(fld_2d(n)) == 'vsno') value2(1:count2(1),1) = vsno(1:count2(1)) + if (trim(fld_2d(n)) == 'uvel') value2(1:count2(1),1) = uvel(1:count2(1)) + if (trim(fld_2d(n)) == 'vvel') value2(1:count2(1),1) = vvel(1:count2(1)) + if (trim(fld_2d(n)) == 'divu') value2(1:count2(1),1) = divu(1:count2(1)) + if (trim(fld_2d(n)) == 'shear') value2(1:count2(1),1) = shear(1:count2(1)) + if (trim(fld_2d(n)) == 'strength') value2(1:count2(1),1) = strength(1:count2(1)) + if (trim(fld_2d(n)) == 'evap') value2(1:count2(1),1) = evap(1:count2(1)) + if (trim(fld_2d(n)) == 'fsnow') value2(1:count2(1),1) = fsnow(1:count2(1)) + if (trim(fld_2d(n)) == 'frazil') value2(1:count2(1),1) = frazil(1:count2(1)) + if (trim(fld_2d(n)) == 'fswabs') value2(1:count2(1),1) = fswabs(1:count2(1)) + if (trim(fld_2d(n)) == 'flw') value2(1:count2(1),1) = flw(1:count2(1)) + if (trim(fld_2d(n)) == 'flwout') value2(1:count2(1),1) = flwout(1:count2(1)) + if (trim(fld_2d(n)) == 'fsens') value2(1:count2(1),1) = fsens(1:count2(1)) + if (trim(fld_2d(n)) == 'fsurf') value2(1:count2(1),1) = fsurf(1:count2(1)) + if (trim(fld_2d(n)) == 'flat') value2(1:count2(1),1) = flat(1:count2(1)) + if (trim(fld_2d(n)) == 'frain') value2(1:count2(1),1) = frain(1:count2(1)) + if (trim(fld_2d(n)) == 'Tair') value2(1:count2(1),1) = Tair(1:count2(1)) + if (trim(fld_2d(n)) == 'Qa') value2(1:count2(1),1) = Qa(1:count2(1)) + if (trim(fld_2d(n)) == 'fsw') value2(1:count2(1),1) = fsw(1:count2(1)) + if (trim(fld_2d(n)) == 'fcondtop') value2(1:count2(1),1) = fcondtop(1:count2(1)) + if (trim(fld_2d(n)) == 'meltt') value2(1:count2(1),1) = meltt(1:count2(1)) + if (trim(fld_2d(n)) == 'meltb') value2(1:count2(1),1) = meltb(1:count2(1)) + if (trim(fld_2d(n)) == 'meltl') value2(1:count2(1),1) = meltl(1:count2(1)) + if (trim(fld_2d(n)) == 'snoice') value2(1:count2(1),1) = snoice(1:count2(1)) + if (trim(fld_2d(n)) == 'dsnow') value2(1:count2(1),1) = dsnow(1:count2(1)) + if (trim(fld_2d(n)) == 'congel') value2(1:count2(1),1) = congel(1:count2(1)) + if (trim(fld_2d(n)) == 'sst') value2(1:count2(1),1) = sst(1:count2(1)) + if (trim(fld_2d(n)) == 'sss') value2(1:count2(1),1) = sss(1:count2(1)) + if (trim(fld_2d(n)) == 'Tf') value2(1:count2(1),1) = Tf(1:count2(1)) + if (trim(fld_2d(n)) == 'fhocn') value2(1:count2(1),1) = fhocn(1:count2(1)) + + status = nf90_inq_varid(ncid,trim(fld_2d(n)),varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: inq_var '//trim(fld_2d(n))) + status = nf90_put_var(ncid,varid,value2,start=start2,count=count2) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_var '//trim(fld_2d(n))) + + deallocate(value2) + enddo + + ! 3d ncat fields + + start3(1) = 1 + count3(1) = nx + start3(2) = 1 + count3(2) = ncat + start3(3) = timcnt + count3(3) = 1 + + do n = 1,num_3d_ncat + allocate(value3(count3(1),count3(2),1)) + + value3 = -9999._dbl_kind + if (trim(fld_3d_ncat(n)) == 'aicen') value3(1:count3(1),1:count3(2),1) = aicen(1:count3(1),1:count3(2)) + if (trim(fld_3d_ncat(n)) == 'vicen') value3(1:count3(1),1:count3(2),1) = vicen(1:count3(1),1:count3(2)) + if (trim(fld_3d_ncat(n)) == 'vsnon') value3(1:count3(1),1:count3(2),1) = vsnon(1:count3(1),1:count3(2)) + + status = nf90_inq_varid(ncid,trim(fld_3d_ncat(n)),varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: inq_var '//trim(fld_3d_ncat(n))) + status = nf90_put_var(ncid,varid,value3,start=start3,count=count3) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_var '//trim(fld_3d_ncat(n))) + + deallocate(value3) + enddo + + ! 3d ntrcr fields + + start3(1) = 1 + count3(1) = nx + start3(2) = 1 + count3(2) = ntrcr + start3(3) = timcnt + count3(3) = 1 + + do n = 1,num_3d_ntrcr + allocate(value3(count3(1),count3(2),1)) + + value3 = -9999._dbl_kind + if (trim(fld_3d_ntrcr(n)) == 'trcr') value3(1:count3(1),1:count3(2),1) = trcr(1:count3(1),1:count3(2)) + + status = nf90_inq_varid(ncid,trim(fld_3d_ntrcr(n)),varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: inq_var '//trim(fld_3d_ntrcr(n))) + status = nf90_put_var(ncid,varid,value3,start=start3,count=count3) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_var '//trim(fld_3d_ntrcr(n))) + + deallocate(value3) + enddo + + ! 4d ncat ntrcr fields + + start4(1) = 1 + count4(1) = nx + start4(2) = 1 + count4(2) = ncat + start4(3) = 1 + count4(3) = ntrcr + start4(4) = timcnt + count4(4) = 1 + + do n = 1,num_4d_ncat_ntrcr + allocate(value4(count4(1),count4(2),count4(3),1)) + + value4 = -9999._dbl_kind + if (trim(fld_4d_ncat_ntrcr(n)) == 'trcr') value4(1:count4(1),1:count4(2),1:count4(3),1) = trcrn(1:count4(1),1:count4(2),1:count4(3)) + + status = nf90_inq_varid(ncid,trim(fld_4d_ncat_ntrcr(n)),varid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: inq_var '//trim(fld_4d_ncat_ntrcr(n))) + status = nf90_put_var(ncid,varid,value4,start=start4,count=count4) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: put_var '//trim(fld_4d_ncat_ntrcr(n))) + + deallocate(value4) + enddo + +#else + call icedrv_system_abort(string=subname//' ERROR: history requires USE_NETCDF',file=__FILE__,line=__LINE__) +#endif + + end subroutine history_write + +!======================================================================= + +! Close history file + + subroutine history_close() + +#ifdef USE_NETCDF + use netcdf +#endif + + ! local variables + + integer (kind=int_kind) :: & + status ! cdf status flag + + character(len=*), parameter :: subname='(history_close)' + +#ifdef USE_NETCDF + status = nf90_close(ncid) + if (status /= nf90_noerr) call icedrv_system_abort(string=subname//' ERROR: nf90_close') +#else + call icedrv_system_abort(string=subname//' ERROR: history requires USE_NETCDF',file=__FILE__,line=__LINE__) +#endif + + end subroutine history_close + +!======================================================================= + + end module icedrv_history + +!======================================================================= diff --git a/configuration/driver/icedrv_init.F90 b/configuration/driver/icedrv_init.F90 index 7d66af1a4..d4f1e75e1 100644 --- a/configuration/driver/icedrv_init.F90 +++ b/configuration/driver/icedrv_init.F90 @@ -62,6 +62,7 @@ subroutine input_data use icedrv_calendar, only: year_init, istep0 use icedrv_calendar, only: dumpfreq, diagfreq, dump_last use icedrv_calendar, only: npt, dt, ndtd, days_per_year, use_leap_years + use icedrv_history, only: history_cdf use icedrv_restart_shared, only: restart, restart_dir, restart_file use icedrv_flux, only: update_ocn_f, l_mpond_fresh, cpl_bgc use icedrv_flux, only: default_season @@ -129,7 +130,7 @@ subroutine input_data dt, npt, ndtd, dump_last, & ice_ic, restart, restart_dir, restart_file, & dumpfreq, diagfreq, diag_file, cpl_bgc, & - conserv_check + conserv_check, history_cdf namelist /grid_nml/ & kcatbound @@ -248,6 +249,7 @@ subroutine input_data restart = .false. ! if true, read restart files for initialization restart_dir = './' ! write to executable dir for default restart_file = 'iced' ! restart file name prefix + history_cdf = .false. ! history netcdf file flag ice_ic = 'default' ! initial conditions are specified in the code ! otherwise, the filename for reading restarts ndtd = 1 ! dynamic time steps per thermodynamic time step @@ -558,6 +560,7 @@ subroutine input_data trim(restart_dir) write(nu_diag,*) ' restart_file = ', & trim(restart_file) + write(nu_diag,1010) ' history_cdf = ', history_cdf write(nu_diag,*) ' ice_ic = ', & trim(ice_ic) write(nu_diag,1010) ' conserv_check = ', conserv_check diff --git a/configuration/scripts/Makefile b/configuration/scripts/Makefile index 5041310ec..0407d917c 100644 --- a/configuration/scripts/Makefile +++ b/configuration/scripts/Makefile @@ -68,6 +68,7 @@ OBJS := $(addsuffix .o, $(sort $(basename $(notdir $(SRCS))))) DEPS := $(addsuffix .d, $(sort $(basename $(notdir $(SRCS))))) INCS := $(patsubst %,-I%, $(VPATH) ) RM := rm +MODDIR:= -I. .SUFFIXES: .SUFFIXES: .F90 .F .c .o @@ -95,15 +96,16 @@ db_files: @echo "* VPATH := $(VPATH)" @echo "* SRCFILE := $(SRCFILE)" @echo "* INCS := $(INCS)" + @echo "* MODDIR := $(MODDIR)" @echo "* SRCS := $(SRCS)" @echo "* OBJS := $(OBJS)" @echo "* DEPS := $(DEPS)" db_flags: @echo " " - @echo "* cpp := $(CPP) $(CPPFLAGS) $(CPPDEFS) $(INCS) $(INCLDIR)" - @echo "* cc := cc -c $(CFLAGS) $(INCS) $(INCLDIR)" - @echo "* .F.o := $(FC) -c $(FFLAGS) $(FIXEDFLAGS) $(INCS) $(INCLDIR)" - @echo "* .F90.o := $(FC) -c $(FFLAGS) $(FREEFLAGS) $(INCS) $(INCLDIR)" + @echo "* cpp := $(CPP) $(CPPFLAGS) $(CPPDEFS) $(INCLDIR)" + @echo "* cc := $(CC) -c $(CFLAGS) $(INCLDIR)" + @echo "* .F.o := $(FC) -c $(FFLAGS) $(FIXEDFLAGS) $(INCLDIR)" + @echo "* .F90.o := $(FC) -c $(FFLAGS) $(FREEFLAGS) $(MODDIR) $(INCLDIR)" #------------------------------------------------------------------------------- # build rule for makdep: MACFILE, cmd-line, or env vars must provide @@ -121,13 +123,13 @@ $(EXEC): $(OBJS) $(LD) -o $(EXEC) $(LDFLAGS) $(OBJS) $(ULIBS) $(SLIBS) .c.o: - cc $(CFLAGS) $(CPPDEFS) $(INCS) $(INCLDIR) $< + $(CC) $(CFLAGS) $(CPPDEFS) $(INCLDIR) $< .F.o: - $(FC) -c $(FFLAGS) $(FIXEDFLAGS) $(CPPDEFS) $(INCS) $(INCLDIR) $< + $(FC) -c $(FFLAGS) $(FIXEDFLAGS) $(CPPDEFS) $(INCLDIR) $< .F90.o: - $(FC) -c $(FFLAGS) $(FREEFLAGS) $(CPPDEFS) $(INCS) $(INCLDIR) $< + $(FC) -c $(FFLAGS) $(FREEFLAGS) $(CPPDEFS) $(MODDIR) $(INCLDIR) $< mostlyclean: $(RM) -f *.f *.f90 diff --git a/configuration/scripts/icepack.build b/configuration/scripts/icepack.build index ddd87cbeb..43a47661b 100755 --- a/configuration/scripts/icepack.build +++ b/configuration/scripts/icepack.build @@ -26,6 +26,10 @@ cd ${ICE_OBJDIR} setenv ICE_CPPDEFS "${ICE_CPPDEFS} -DNXGLOB=${ICE_NXGLOB} -DNICELYR=${NICELYR} -DNSNWLYR=${NSNWLYR} -DNICECAT=${NICECAT} -DNFSDCAT=${NFSDCAT} -DTRAGE=${TRAGE} -DTRFY=${TRFY} -DTRLVL=${TRLVL} -DTRPND=${TRPND} -DTRSNOW=${TRSNOW} -DTRBRI=${TRBRI} -DNTRISO=${NTRISO} -DNTRAERO=${NTRAERO} -DTRZS=${TRZS} -DNBGCLYR=${NBGCLYR} -DTRALG=${TRALG} -DTRBGCZ=${TRBGCZ} -DTRDOC=${TRDOC} -DTRDOC=${TRDOC} -DTRDIC=${TRDIC} -DTRDON=${TRDON} -DTRFED=${TRFED} -DTRFEP=${TRFEP} -DTRZAERO=${TRZAERO} -DTRBGCS=${TRBGCS} " +if (${ICE_IOTYPE} == 'netcdf') then + setenv ICE_CPPDEFS "${ICE_CPPDEFS} -DUSE_NETCDF" +endif + ### List of source code directories (in order of importance). cat >! Filepath << EOF ${ICE_SANDBOX}/configuration/driver diff --git a/configuration/scripts/icepack_in b/configuration/scripts/icepack_in index ef945a326..ff5f4b166 100644 --- a/configuration/scripts/icepack_in +++ b/configuration/scripts/icepack_in @@ -13,6 +13,7 @@ dump_last = .false. diagfreq = 24 diag_file = 'ice_diag' + history_cdf = .false. cpl_bgc = .false. conserv_check = .false. / diff --git a/configuration/scripts/machines/Macros.badger_intel b/configuration/scripts/machines/Macros.badger_intel index c8488ab84..c572515b9 100644 --- a/configuration/scripts/machines/Macros.badger_intel +++ b/configuration/scripts/machines/Macros.badger_intel @@ -24,28 +24,23 @@ CC := $(SCC) FC := $(SFC) LD := $(FC) -# set in Macros file -NETCDF_PATH := /usr/projects/climate/SHARED_CLIMATE/software/conejo/netcdf/3.6.3/intel-13.0.1 -PNETCDF_PATH := /usr/projects/climate/SHARED_CLIMATE/software/conejo/parallel-netcdf/1.3.1/intel-13.0.1/openmpi-1.6.3 -#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib +NETCDF_PATH := /usr/projects/hpcsoft/toss3/common/netcdf/4.4.0_intel-18.0.5 +PNETCDF_PATH := /usr/projects/hpcsoft/toss3/badger/netcdf/4.4.0_intel-18.0.5_openmpi-2.1.2 PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs ifeq ($(ICE_IOTYPE), netcdf) - INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include -# INCLDIR := $(INCLDIR) -I/usr/projects/climate/SHARED_CLIMATE/software/conejo/netcdf/3.6.3/intel-13.0.1/include + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include -I$(PNETCDF_PATH)/include LIB_NETCDF := $(NETCDF_PATH)/lib LIB_PNETCDF := $(PNETCDF_PATH)/lib LIB_MPI := $(IMPILIBDIR) - #SLIBS := -L$(LIB_NETCDF) -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -L$(LAPACK_LIBDIR) -llapack -lblas - SLIBS := -L$(LIB_NETCDF) -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf + SLIBS := -L$(LIB_NETCDF) -lnetcdf -L$(LIB_PNETCDF) -lnetcdff else SLIBS := endif - ifeq ($(ICE_THREADED), true) LDFLAGS += -qopenmp CFLAGS += -qopenmp diff --git a/configuration/scripts/machines/Macros.conda_macos b/configuration/scripts/machines/Macros.conda_macos index 583a37773..a9381791f 100644 --- a/configuration/scripts/machines/Macros.conda_macos +++ b/configuration/scripts/machines/Macros.conda_macos @@ -26,6 +26,9 @@ CC := $(SCC) FC := $(SFC) LD := $(FC) +# Location of the compiled Fortran modules (NetCDF) +MODDIR += -I$(CONDA_PREFIX)/include + # Location of the system C header files (required on recent macOS to compile makdep) SDKPATH = $(shell xcrun --show-sdk-path) ifeq ($(strip $(SDKPATH)),) @@ -35,6 +38,9 @@ else LD += -L$(SDKPATH)/usr/lib endif +# Libraries to be passed to the linker +SLIBS := -L$(CONDA_PREFIX)/lib -lnetcdf -lnetcdff + # Necessary flag to compile with OpenMP support ifeq ($(ICE_THREADED), true) LDFLAGS += -fopenmp diff --git a/configuration/scripts/machines/env.badger_intel b/configuration/scripts/machines/env.badger_intel index 9239d2e84..4a1e5d668 100755 --- a/configuration/scripts/machines/env.badger_intel +++ b/configuration/scripts/machines/env.badger_intel @@ -12,6 +12,14 @@ if ("$inp" != "-nomodules") then #module purge #module load intel #module load openmpi +module unload hdf5-serial +module unload hdf5-parallel +module unload netcdf-serial +module unload netcdf-h5parallel +module load hdf5-serial +module load netcdf-serial/4.4.0 +module load hdf5-parallel +module load netcdf-h5parallel/4.4.0 setenv NETCDF_PATH /usr/projects/climate/SHARED_CLIMATE/software/conejo/netcdf/3.6.3/intel-13.0.1 setenv PNETCDF_PATH /usr/projects/climate/SHARED_CLIMATE/software/conejo/parallel-netcdf/1.3.1/intel-13.0.1/openmpi-1.6.3 @@ -29,7 +37,7 @@ endif setenv ICE_MACHINE_MACHNAME badger setenv ICE_MACHINE_MACHINFO "Penguin Intel Xeon Broadwell" setenv ICE_MACHINE_ENVNAME intel -setenv ICE_MACHINE_ENVINFO "(Note: can vary) ifort 19.0.4.243 20190416" +setenv ICE_MACHINE_ENVINFO "(Note: can vary) ifort 19.0.4.243 20190416, netcdf4.4.0" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /net/scratch3/$user/ICEPACK_RUNS setenv ICE_MACHINE_INPUTDATA /usr/projects/climate/eclare/DATA/Consortium diff --git a/configuration/scripts/machines/environment.yml b/configuration/scripts/machines/environment.yml index c63ccbf5b..dfbcaf85e 100644 --- a/configuration/scripts/machines/environment.yml +++ b/configuration/scripts/machines/environment.yml @@ -5,6 +5,8 @@ channels: dependencies: # Build dependencies - compilers + - netcdf-fortran + - netcdf4 - make # Python dependencies for building the HTML documentation - sphinx diff --git a/configuration/scripts/options/set_env.ionetcdf b/configuration/scripts/options/set_env.ionetcdf new file mode 100644 index 000000000..1824dde94 --- /dev/null +++ b/configuration/scripts/options/set_env.ionetcdf @@ -0,0 +1 @@ +setenv ICE_IOTYPE netcdf diff --git a/configuration/scripts/options/set_nml.ionetcdf b/configuration/scripts/options/set_nml.ionetcdf new file mode 100644 index 000000000..6aaa620e7 --- /dev/null +++ b/configuration/scripts/options/set_nml.ionetcdf @@ -0,0 +1 @@ +history_cdf = .true. diff --git a/configuration/scripts/setup_run_dirs.csh b/configuration/scripts/setup_run_dirs.csh index a034bfbf8..e9001347f 100755 --- a/configuration/scripts/setup_run_dirs.csh +++ b/configuration/scripts/setup_run_dirs.csh @@ -7,7 +7,7 @@ if !(-d ${ICE_RUNDIR}) then echo "mkdir ${ICE_RUNDIR}" mkdir -p ${ICE_RUNDIR} endif -#if !(-d ${ICE_HSTDIR}) mkdir -p ${ICE_HSTDIR} +if !(-d ${ICE_HSTDIR}) mkdir -p ${ICE_HSTDIR} if !(-d ${ICE_RSTDIR}) mkdir -p ${ICE_RSTDIR} exit 0 diff --git a/configuration/scripts/tests/io_suite.ts b/configuration/scripts/tests/io_suite.ts new file mode 100644 index 000000000..2af2980d5 --- /dev/null +++ b/configuration/scripts/tests/io_suite.ts @@ -0,0 +1,2 @@ +restart col 1x1 debug,ionetcdf +smoke col 1x1 run1year,diag1,ionetcdf diff --git a/doc/source/icepack_index.rst b/doc/source/icepack_index.rst index f4e1329fd..293ddd797 100755 --- a/doc/source/icepack_index.rst +++ b/doc/source/icepack_index.rst @@ -210,6 +210,7 @@ either Celsius or Kelvin units). "highfreq", ":math:`\bullet` high-frequency atmo coupling", "F" "hin_old", "ice thickness prior to growth/melt", "m" "hin_max", "category thickness limits", "m" + "history_cdf", "flag to turn on netcdf history output", "F" "hmix", "ocean mixed layer depth", "20. m" "hour", "hour of the year", "" "hp0", "pond depth at which shortwave transition to bare ice occurs", "0.2 m" diff --git a/doc/source/user_guide/ug_case_settings.rst b/doc/source/user_guide/ug_case_settings.rst index 09891acb1..7ee316339 100755 --- a/doc/source/user_guide/ug_case_settings.rst +++ b/doc/source/user_guide/ug_case_settings.rst @@ -31,6 +31,7 @@ can be found in :ref:`cicecpps`. The following CPPs are available. "**General Macros**", "" "NO_I8", "Converts ``integer*8`` to ``integer*4``." "NO_R16", "Converts ``real*16`` to ``real*8``." + "USE_NETCDF", "Turns on netcdf capabilities in Icepack. By default and generally, Icepack does not need netcdf." "","" "**Application Macros**", "" "CESMCOUPLED", "Turns on code changes for the CESM coupled application " @@ -65,7 +66,7 @@ can be modified as needed. "ICE_LOGDIR", "string", "log directory", "${ICE_CASEDIR}/logs" "ICE_RSTPFILE", "string", "unused", "undefined" "ICE_DRVOPT", "string", "unused", "icepack" - "ICE_IOTYPE", "string", "unused", "none" + "ICE_IOTYPE", "none,netcdf", "IO options", "none" "ICE_CLEANBUILD", "true,false", "automatically clean before building", "true" "ICE_CPPDEFS", "string", "user defined preprocessor macros for build", "null" "ICE_QUIETMODE", "true, false", "reduce build output to the screen", "false" @@ -139,6 +140,7 @@ setup_nml "", "``y``", "write restart every ``dumpfreq_n`` years", "" "``dump_last``", "true/false", "write restart at end of run", "false" "``dt``", "seconds", "thermodynamics time step length", "3600." + "``history_cdf``", "logical", "netcdf history output", "``.false.``" "``ice_ic``", "``default``", "latitude and sst dependent initial condition", "``default``" "", "``none``", "no ice", "" "", "'path/file'", "restart file name", "" diff --git a/doc/source/user_guide/ug_implementation.rst b/doc/source/user_guide/ug_implementation.rst index 70a806bf0..fe244fcea 100755 --- a/doc/source/user_guide/ug_implementation.rst +++ b/doc/source/user_guide/ug_implementation.rst @@ -150,9 +150,9 @@ default in this distribution, this is not a stringent limitation: Model output ------------ -History output from Icepack is not currently supported in the Icepack driver, except -in restart files. -The sea ice model `CICE `_ provides extensive +The Icepack model provides diagnostic output files, binary restart files, and a primitive +netcdf history file capability. +The sea ice model `CICE `_ provides more extensive options for model output, including many derived output variables. Diagnostic files @@ -176,6 +176,30 @@ the namelist option ``restart`` must be set to ``.true.`` to use the file. ``dump_last`` namelist can also be set to true to trigger restarts automatically at then end of runs. +History files +~~~~~~~~~~~~~ + +Icepack has a primitive netcdf history capability that is turned on with the +``history_cdf`` namelist. When ``history_cdf`` is set to true, history files +are created for each run with a naming convention of **icepack.h.yyyymmdd.nc** +in the run directory history directory. The yyyymmdd is the start date for each run. + +When Icepack history files are turned on, data for a set of fixed fields is written +to the history file for each column at every timestep without ability to control +fields, frequencies, or temporal averaging. All output fields are hardwired into +the implementation in **configuration/driver/icedrv_history.F90** file. The netcdf file +does NOT meet NetCDF CF conventions and is provided as an amenity in the standalone +Icepack model. Users are free to modify the output fields or +extend the implementation and are encouraged to share any updates with the Consortium. + +The default configuration of Icepack does not require NetCDF. If history files are +written, the USE_NETCDF C preprocessor directive must be set during compilation. This +is done by setting ``ICE_IOTYPE`` to ``netcdf`` in **icepack.settings**. In addition, +the machine env and Macros files must include support for compilation with NetCDF. The +``icepack.setup -s`` option ``ionetcdf`` will set the ICE_IOTYPE to netcdf, which turns on +the USE_NETCDF C preprocessor. ``ionetcdf`` also sets the ``history_cdf`` flag to true. + + .. _bgc-hist: Biogeochemistry History Fields diff --git a/doc/source/user_guide/ug_running.rst b/doc/source/user_guide/ug_running.rst index 94b66c405..85a9c7cd9 100755 --- a/doc/source/user_guide/ug_running.rst +++ b/doc/source/user_guide/ug_running.rst @@ -585,7 +585,12 @@ Next, create the "icepack" conda environment from the ``environment.yml`` file i conda env create -f configuration/scripts/machines/environment.yml -This step needs to be done only once. +This step needs to be done only once. If you ever need to update the conda environment +because the required packages change or packages are out of date, do + +.. code-block:: bash + + conda env update -f configuration/scripts/machines/environment.yml .. _using_conda_env: