mod_atmos_phy_tb_driver.F90

Go to the documentation of this file.

!-------------------------------------------------------------------------------
!-------------------------------------------------------------------------------
#include "scalelib.h"
module mod_atmos_phy_tb_driver
  !-----------------------------------------------------------------------------
  !
  !++ used modules
  !
  use scale_precision
  use scale_io
  use scale_prof
  use scale_atmos_grid_cartesc_index
  use scale_tracer
  !-----------------------------------------------------------------------------
  implicit none
  private
  !-----------------------------------------------------------------------------
  !
  !++ Public procedure
  !
  public :: atmos_phy_tb_driver_tracer_setup
  public :: atmos_phy_tb_driver_setup
  public :: atmos_phy_tb_driver_finalize
  public :: atmos_phy_tb_driver_calc_tendency
 
  !-----------------------------------------------------------------------------
  !
  !++ Public parameters & variables
  !
  !-----------------------------------------------------------------------------
  !
  !++ Private procedure
  !
  !-----------------------------------------------------------------------------
  !
  !++ Private parameters & variables
  !
  integer, private :: monit_west
  integer, private :: monit_east
  integer, private :: monit_south
  integer, private :: monit_north
 
  !-----------------------------------------------------------------------------
contains
  !-----------------------------------------------------------------------------
  subroutine atmos_phy_tb_driver_tracer_setup
    use scale_prc, only: &
       prc_abort
    use scale_atmos_phy_tb_d1980, only: &
       atmos_phy_tb_d1980_config
    use scale_atmos_phy_tb_dns, only: &
       atmos_phy_tb_dns_config
    use mod_atmos_admin, only: &
       atmos_phy_tb_type, &
       atmos_sw_phy_tb
    use mod_atmos_phy_tb_vars, only: &
       i_tke
    implicit none
    !---------------------------------------------------------------------------
 
    if ( .not. atmos_sw_phy_tb ) return
 
    log_newline
    log_info("ATMOS_PHY_TB_driver_tracer_setup",*) 'Setup'
 
    select case( atmos_phy_tb_type )
    case( 'SMAGORINSKY' )
       i_tke = -1
    case( 'D1980' )
       call atmos_phy_tb_d1980_config( atmos_phy_tb_type, & ! [IN]
                                       i_tke              ) ! [OUT]
    case( 'DNS' )
       call atmos_phy_tb_dns_config( atmos_phy_tb_type, & ! [IN]
                                     i_tke              ) ! [OUT]
    case('OFF')
       ! do nothing
    case default
       log_error("ATMOS_PHY_TB_driver_tracer_setup",*) 'ATMOS_PHY_TB_TYPE is invalid: ', atmos_phy_tb_type
       call prc_abort
    end select
 
    return
  end subroutine atmos_phy_tb_driver_tracer_setup
 
  !-----------------------------------------------------------------------------
  subroutine atmos_phy_tb_driver_setup
    use scale_atmos_grid_cartesc, only: &
       cdz => atmos_grid_cartesc_cdz, &
       cdx => atmos_grid_cartesc_cdx, &
       cdy => atmos_grid_cartesc_cdy
    use scale_atmos_grid_cartesc_real, only: &
       real_cz  => atmos_grid_cartesc_real_cz, &
       real_fz  => atmos_grid_cartesc_real_fz
    use scale_atmos_grid_cartesc_metric, only: &
       mapf  => atmos_grid_cartesc_metric_mapf
    use scale_atmos_phy_tb_smg, only: &
       atmos_phy_tb_smg_setup
    use scale_atmos_phy_tb_d1980, only: &
       atmos_phy_tb_d1980_setup
    use scale_atmos_phy_tb_dns, only: &
       atmos_phy_tb_dns_setup
    use mod_atmos_admin, only: &
       atmos_phy_tb_type, &
       atmos_sw_phy_tb,   &
       atmos_sw_phy_bl
    use mod_atmos_phy_tb_vars, only: &
       momz_t_tb => atmos_phy_tb_momz_t
    use scale_monitor, only: &
       monitor_reg
    implicit none
 
    integer :: i, j
    logical :: horizontal = .false.
    !---------------------------------------------------------------------------
 
    if ( .not. atmos_sw_phy_tb ) return
 
    log_newline
    log_info("ATMOS_PHY_TB_driver_setup",*) 'Setup'
 
    ! initialize
    !$omp parallel do
    !$acc kernels copy(MOMZ_t_TB)
    do j = js, je
    do i = is, ie
       momz_t_tb(ks-1,i,j) = 0.0_rp
       momz_t_tb(ke  ,i,j) = 0.0_rp
    enddo
    enddo
    !$acc end kernels
 
    ! setup library component
    select case( atmos_phy_tb_type )
    case( 'SMAGORINSKY' )
       if ( atmos_sw_phy_bl ) horizontal = .true.
       call atmos_phy_tb_smg_setup( real_fz, real_cz, cdx, cdy, mapf(:,:,:,i_xy), horizontal ) ! [IN]
    case( 'D1980' )
       call atmos_phy_tb_d1980_setup( cdz, cdx, cdy, real_cz ) ! [IN]
    case( 'DNS' )
       call atmos_phy_tb_dns_setup( cdz, cdx, cdy, real_cz ) ! [IN]
    end select
 
 
    ! monitor
    call monitor_reg( "QTOTFLX_TB_WEST",  "water mass flux at the western boundary",  "kg", & ! [IN]
                      monit_west,                                                           & ! [OUT]
                      dim_type="ZY-W", is_tendency=.true.                                   ) ! [IN]
    call monitor_reg( "QTOTFLX_TB_EAST",  "water mass flux at the eastern boundary",  "kg", & ! [IN]
                      monit_east,                                                           & ! [OUT]
                      dim_type="ZY-E", is_tendency=.true.                                   ) ! [IN]
    call monitor_reg( "QTOTFLX_TB_SOUTH", "water mass flux at the southern boundary", "kg", & ! [IN]
                      monit_south,                                                          & ! [OUT]
                      dim_type="ZX-S", is_tendency=.true.                                   ) ! [IN]
    call monitor_reg( "QTOTFLX_TB_NORTH", "water mass flux at the northern boundary", "kg", & ! [IN]
                      monit_north,                                                          & ! [OUT]
                      dim_type="ZX-N", is_tendency=.true.                                   ) ! [IN]
 
 
    return
  end subroutine atmos_phy_tb_driver_setup
 
  !-----------------------------------------------------------------------------
  subroutine atmos_phy_tb_driver_finalize
    use scale_atmos_phy_tb_smg, only: &
       atmos_phy_tb_smg_finalize
    use scale_atmos_phy_tb_d1980, only: &
       atmos_phy_tb_d1980_finalize
    use mod_atmos_admin, only: &
       atmos_phy_tb_type, &
       atmos_sw_phy_tb
    implicit none
    !---------------------------------------------------------------------------
 
    log_newline
    log_info("ATMOS_PHY_TB_driver_finalize",*) 'Finalize'
 
    if ( .not. atmos_sw_phy_tb ) return
 
    select case( atmos_phy_tb_type )
    case( 'SMAGORINSKY' )
       call atmos_phy_tb_smg_finalize
    case( 'D1980' )
       call atmos_phy_tb_d1980_finalize
    end select
 
    return
  end subroutine atmos_phy_tb_driver_finalize
 
  !-----------------------------------------------------------------------------
  subroutine atmos_phy_tb_driver_calc_tendency( update_flag )
    use scale_prc_cartesc, only: &
       prc_has_w, &
       prc_has_e, &
       prc_has_s, &
       prc_has_n
    use scale_statistics, only: &
       statistics_checktotal, &
       statistics_total
    use scale_atmos_grid_cartesc, only: &
       fdz  => atmos_grid_cartesc_fdz, &
       rcdz => atmos_grid_cartesc_rcdz, &
       rfdz => atmos_grid_cartesc_rfdz, &
       cdx  => atmos_grid_cartesc_cdx, &
       fdx  => atmos_grid_cartesc_fdx, &
       cdy  => atmos_grid_cartesc_cdy, &
       fdy  => atmos_grid_cartesc_fdy
    use scale_atmos_grid_cartesc_real, only: &
       fz  => atmos_grid_cartesc_real_fz, &
       atmos_grid_cartesc_real_vol,       &
       atmos_grid_cartesc_real_totvol,    &
       atmos_grid_cartesc_real_volwxy,    &
       atmos_grid_cartesc_real_totvolwxy, &
       atmos_grid_cartesc_real_volzuy,    &
       atmos_grid_cartesc_real_totvolzuy, &
       atmos_grid_cartesc_real_volzxv,    &
       atmos_grid_cartesc_real_totvolzxv
    use scale_atmos_grid_cartesc_metric, only: &
       gsqrt => atmos_grid_cartesc_metric_gsqrt, &
       j13g  => atmos_grid_cartesc_metric_j13g,  &
       j23g  => atmos_grid_cartesc_metric_j23g,  &
       j33g  => atmos_grid_cartesc_metric_j33g,  &
       mapf  => atmos_grid_cartesc_metric_mapf
    use scale_file_history, only: &
       file_history_in
    use scale_monitor, only: &
       monitor_put
    use scale_time, only: &
       dt_tb => time_dtsec_atmos_phy_tb
    use scale_atmos_phy_tb_smg, only: &
       atmos_phy_tb_smg
    use scale_atmos_phy_tb_d1980, only: &
       atmos_phy_tb_d1980
    use scale_atmos_phy_tb_dns, only: &
       atmos_phy_tb_dns
    use scale_atmos_phy_tb_common, only: &
       calc_tend_momz => atmos_phy_tb_calc_tend_momz, &
       calc_tend_momx => atmos_phy_tb_calc_tend_momx, &
       calc_tend_momy => atmos_phy_tb_calc_tend_momy, &
       calc_tend_phi  => atmos_phy_tb_calc_tend_phi
    use mod_atmos_admin, only: &
       atmos_phy_tb_type
    use mod_atmos_vars, only: &
       atmos_vars_get_diagnostic, &
       dens => dens_av,   &
       momz => momz_av,   &
       momx => momx_av,   &
       momy => momy_av,   &
       rhot => rhot_av,   &
       pott,              &
       qtrc => qtrc_av,   &
       momz_t => momz_tp, &
       momx_t => momx_tp, &
       momy_t => momy_tp, &
       rhot_t => rhot_tp, &
       rhoq_t => rhoq_tp
    use mod_atmos_phy_tb_vars, only: &
       i_tke, &
       momz_t_tb => atmos_phy_tb_momz_t, &
       momx_t_tb => atmos_phy_tb_momx_t, &
       momy_t_tb => atmos_phy_tb_momy_t, &
       rhot_t_tb => atmos_phy_tb_rhot_t, &
       rhoq_t_tb => atmos_phy_tb_rhoq_t
    use mod_atmos_phy_sf_vars, only: &
       sflx_mw => atmos_phy_sf_sflx_mw, &
       sflx_mu => atmos_phy_sf_sflx_mu, &
       sflx_mv => atmos_phy_sf_sflx_mv, &
       sflx_sh => atmos_phy_sf_sflx_sh, &
       sflx_q  => atmos_phy_sf_sflx_qtrc
    implicit none
 
    logical, intent(in) :: update_flag
 
    ! eddy viscosity/diffusion flux
    real(rp) :: qflx_momz(ka,ia,ja,3)
    real(rp) :: qflx_momx(ka,ia,ja,3)
    real(rp) :: qflx_momy(ka,ia,ja,3)
    real(rp) :: qflx_rhot(ka,ia,ja,3)
    real(rp) :: qflx_rhoq(ka,ia,ja,3,qa)
 
    real(rp) :: nu(ka,ia,ja) ! eddy viscosity
    real(rp) :: ri(ka,ia,ja) ! Richardson number
    real(rp) :: pr(ka,ia,ja) ! Prandtl number
 
    real(rp) :: n2(ka,ia,ja)
 
    real(rp) :: tend(ka,ia,ja)
 
    ! for monitor
    real(rp) :: qflx_x(ka,ja)
    real(rp) :: qflx_y(ka,ia)
 
    integer  :: jjs, jje
    integer  :: iis, iie
 
    integer  :: k, i, j, iq
    !---------------------------------------------------------------------------
 
    !$acc data create(QFLX_RHOQ)
 
 
    if ( update_flag ) then
 
       !$acc data copyin(GSQRT, J13G, J23G, MAPF) &
       !$acc      create(QFLX_MOMZ, QFLX_MOMX, QFLX_MOMY, QFLX_RHOT, &
       !$acc             Nu, Ri, Pr, N2, tend)
 
       call atmos_vars_get_diagnostic( "N2", n2 )
 
       select case( atmos_phy_tb_type )
       case( 'SMAGORINSKY' )
          call atmos_phy_tb_smg( qflx_momz, qflx_momx, qflx_momy,         & ! [OUT]
                                 qflx_rhot, qflx_rhoq,                    & ! [OUT]
                                 momz_t_tb, momx_t_tb, momy_t_tb,         & ! [OUT]
                                 rhot_t_tb, rhoq_t_tb,                    & ! [OUT]
                                 nu, ri, pr,                              & ! [OUT]
                                 momz, momx, momy, pott, dens, qtrc, n2,  & ! [IN]
                                 fz, fdz, rcdz, rfdz, cdx, fdx, cdy, fdy, & ! [IN]
                                 gsqrt, j13g, j23g, j33g, mapf,           & ! [IN]
                                 dt_tb                                    ) ! [IN]
       case( 'D1980' )
          !$acc kernels
          momz_t_tb(:,:,:)   = 0.0_rp
          momx_t_tb(:,:,:)   = 0.0_rp
          momy_t_tb(:,:,:)   = 0.0_rp
          rhot_t_tb(:,:,:)   = 0.0_rp
          !$acc end kernels
          !$acc update host(MOMZ,MOMX,MOMY,RHOT,DENS,QTRC,N2,SFLX_MW,SFLX_MU,SFLX_MV,SFLX_SH,SFLX_Q)
          call atmos_phy_tb_d1980( qflx_momz, qflx_momx, qflx_momy,        & ! [OUT]
                                   qflx_rhot, qflx_rhoq,                   & ! [OUT]
                                   rhoq_t_tb,                              & ! [OUT]
                                   nu, ri, pr,                             & ! [OUT]
                                   momz, momx, momy, rhot, dens, qtrc, n2, & ! [IN]
                                   sflx_mw, sflx_mu, sflx_mv,              & ! [IN]
                                   sflx_sh, sflx_q,                        & ! [IN]
                                   gsqrt, j13g, j23g, j33g, mapf,          & ! [IN]
                                   dt_tb                                   ) ! [IN]
          !$acc update device(QFLX_MOMZ,QFLX_MOMX,QFLX_MOMY,QFLX_RHOT,QFLX_RHOQ,RHOQ_t_TB,Nu,Ri,Pr)
       case( 'DNS' )
          !$acc kernels
          momz_t_tb(:,:,:)   = 0.0_rp
          momx_t_tb(:,:,:)   = 0.0_rp
          momy_t_tb(:,:,:)   = 0.0_rp
          rhot_t_tb(:,:,:)   = 0.0_rp
          !$acc end kernels
          call atmos_phy_tb_dns( qflx_momz, qflx_momx, qflx_momy,         & ! [OUT]
                                 qflx_rhot, qflx_rhoq,                    & ! [OUT]
                                 rhoq_t_tb,                               & ! [OUT]
                                 nu, ri, pr,                              & ! [OUT]
                                 momz, momx, momy, pott, dens, qtrc, n2,  & ! [IN]
                                 sflx_mw, sflx_mu, sflx_mv,               & ! [IN]
                                 sflx_sh, sflx_q,                         & ! [IN]
                                 fz, fdz, rcdz, rfdz, cdx, fdx, cdy, fdy, & ! [IN]
                                 gsqrt, j13g, j23g, j33g, mapf,           & ! [IN]
                                 dt_tb                                    ) ! [IN]
       end select
 
 
       do jjs = js, je, jblock
       jje = jjs+jblock-1
       do iis = is, ie, iblock
       iie = iis+iblock-1
 
          call calc_tend_momz( tend(:,:,:), & ! (out)
                               qflx_momz,   & ! (in)
                               gsqrt, j13g, j23g, j33g, mapf, & ! (in)
                               iis, iie, jjs, jje ) ! (in)
          !$omp parallel do
          !$acc kernels
          do j = jjs, jje
          do i = iis, iie
          do k = ks, ke-1
             momz_t_tb(k,i,j) = momz_t_tb(k,i,j) + tend(k,i,j)
          end do
          end do
          end do
          !$acc end kernels
 
          call calc_tend_momx( tend(:,:,:), & ! (out)
                               qflx_momx,   & ! (in)
                               gsqrt, j13g, j23g, j33g, mapf, & ! (in)
                               iis, iie, jjs, jje ) ! (in)
          !$omp parallel do
          !$acc kernels
          do j = jjs, jje
          do i = iis, iie
          do k = ks, ke
             momx_t_tb(k,i,j) = momx_t_tb(k,i,j) + tend(k,i,j)
          end do
          end do
          end do
          !$acc end kernels
 
          call calc_tend_momy( tend(:,:,:), & ! (out)
                               qflx_momy,   & ! (in)
                               gsqrt, j13g, j23g, j33g, mapf, & ! (in)
                               iis, iie, jjs, jje ) ! (in)
          !$omp parallel do
          !$acc kernels
          do j = jjs, jje
          do i = iis, iie
          do k = ks, ke
             momy_t_tb(k,i,j) = momy_t_tb(k,i,j) + tend(k,i,j)
          end do
          end do
          end do
          !$acc end kernels
 
          call calc_tend_phi ( tend(:,:,:), & ! (out)
                               qflx_rhot,   & ! (in)
                               gsqrt, j13g, j23g, j33g, mapf, & ! (in)
                               iis, iie, jjs, jje ) ! (in)
          !$omp parallel do
          !$acc kernels
          do j = jjs, jje
          do i = iis, iie
          do k = ks, ke
             rhot_t_tb(k,i,j) = rhot_t_tb(k,i,j) + tend(k,i,j)
          end do
          end do
          end do
          !$acc end kernels
 
          do iq = 1, qa
             if ( iq == i_tke .or. .not. tracer_advc(iq) ) cycle
 
             call calc_tend_phi( tend(:,:,:),                   & ! (out)
                                 qflx_rhoq(:,:,:,:,iq),         & ! (in)
                                 gsqrt, j13g, j23g, j33g, mapf, & ! (in)
                                 iis, iie, jjs, jje ) ! (in)
 
             !$omp parallel do
             !$acc kernels
             do j = jjs, jje
             do i = iis, iie
             do k = ks, ke
                rhoq_t_tb(k,i,j,iq) = rhoq_t_tb(k,i,j,iq) + tend(k,i,j)
             end do
             end do
             end do
             !$acc end kernels
 
          end do
 
       end do
       end do
 
 
       call file_history_in( nu(:,:,:), 'NU',  'eddy viscosity',           'm2/s' , fill_halo=.true. )
       call file_history_in( ri(:,:,:), 'Ri',  'Richardson number',        'NIL'  , fill_halo=.true. )
       call file_history_in( pr(:,:,:), 'Pr',  'Prantle number',           'NIL'  , fill_halo=.true. )
 
       call file_history_in( momz_t_tb(:,:,:), 'MOMZ_t_TB', 'MOMZ tendency (TB)', 'kg/m2/s2',  fill_halo=.true. )
       call file_history_in( momx_t_tb(:,:,:), 'MOMX_t_TB', 'MOMX tendency (TB)', 'kg/m2/s2',  fill_halo=.true. )
       call file_history_in( momy_t_tb(:,:,:), 'MOMY_t_TB', 'MOMY tendency (TB)', 'kg/m2/s2',  fill_halo=.true. )
       call file_history_in( rhot_t_tb(:,:,:), 'RHOT_t_TB', 'RHOT tendency (TB)', 'K.kg/m3/s', fill_halo=.true. )
 
       do iq = 1, qa
          call file_history_in( rhoq_t_tb(:,:,:,iq), trim(tracer_name(iq))//'_t_TB',                      &
                        'tendency rho*'//trim(tracer_name(iq))//' in TB', 'kg/m3/s', fill_halo=.true. )
       enddo
 
       call file_history_in( qflx_momz(:,:,:,zdir), 'SGS_ZFLX_MOMZ', 'SGS Z FLUX of MOMZ', 'kg/m/s2', &
                     fill_halo=.true.)
       call file_history_in( qflx_momz(:,:,:,xdir), 'SGS_XFLX_MOMZ', 'SGS X FLUX of MOMZ', 'kg/m/s2', &
                     dim_type='ZHXHY', fill_halo=.true.)
       call file_history_in( qflx_momz(:,:,:,ydir), 'SGS_YFLX_MOMZ', 'SGS Y FLUX of MOMZ', 'kg/m/s2', &
                     dim_type='ZHXYH', fill_halo=.true.)
 
       call file_history_in( qflx_momx(:,:,:,zdir), 'SGS_ZFLX_MOMX', 'SGS Z FLUX of MOMX', 'kg/m/s2', &
                     dim_type='ZHXHY', fill_halo=.true.)
       call file_history_in( qflx_momx(:,:,:,xdir), 'SGS_XFLX_MOMX', 'SGS X FLUX of MOMX', 'kg/m/s2', &
                     fill_halo=.true.)
       call file_history_in( qflx_momx(:,:,:,ydir), 'SGS_YFLX_MOMX', 'SGS Y FLUX of MOMX', 'kg/m/s2', &
                     dim_type='ZXHYH', fill_halo=.true.)
 
       call file_history_in( qflx_momy(:,:,:,zdir), 'SGS_ZFLX_MOMY', 'SGS Z FLUX of MOMY', 'kg/m/s2', &
                     dim_type='ZHXYH', fill_halo=.true.)
       call file_history_in( qflx_momy(:,:,:,xdir), 'SGS_XFLX_MOMY', 'SGS X FLUX of MOMY', 'kg/m/s2', &
                     dim_type='ZXHYH', fill_halo=.true.)
       call file_history_in( qflx_momy(:,:,:,ydir), 'SGS_YFLX_MOMY', 'SGS Y FLUX of MOMY', 'kg/m/s2', &
                     fill_halo=.true.)
 
       call file_history_in( qflx_rhot(:,:,:,zdir), 'SGS_ZFLX_RHOT', 'SGS Z FLUX of RHOT', 'K*kg/m2/s', &
                     dim_type='ZHXY', fill_halo=.true.)
       call file_history_in( qflx_rhot(:,:,:,xdir), 'SGS_XFLX_RHOT', 'SGS X FLUX of RHOT', 'K*kg/m2/s', &
                     dim_type='ZXHY', fill_halo=.true.)
       call file_history_in( qflx_rhot(:,:,:,ydir), 'SGS_YFLX_RHOT', 'SGS Y FLUX of RHOT', 'K*kg/m2/s', &
                     dim_type='ZXYH', fill_halo=.true.)
 
 
       do iq = 1, qa
          if ( iq == i_tke .or. .not. tracer_advc(iq) ) cycle
 
          call file_history_in( qflx_rhoq(:,:,:,zdir,iq), &
               'SGS_ZFLX_'//trim(tracer_name(iq)), 'SGS Z FLUX of '//trim(tracer_name(iq)), 'kg/m2/s', &
               dim_type='ZHXY', fill_halo=.true.)
          call file_history_in( qflx_rhoq(:,:,:,xdir,iq), &
               'SGS_XFLX_'//trim(tracer_name(iq)), 'SGS X FLUX of '//trim(tracer_name(iq)), 'kg/m2/s', &
               dim_type='ZXHY', fill_halo=.true.)
          call file_history_in( qflx_rhoq(:,:,:,ydir,iq), &
               'SGS_YFLX_'//trim(tracer_name(iq)), 'SGS Y FLUX of '//trim(tracer_name(iq)), 'kg/m2/s', &
               dim_type='ZXYH', fill_halo=.true.)
       end do
 
       if ( statistics_checktotal ) then
          !$acc data copyin(ATMOS_GRID_CARTESC_REAL_VOLWXY, ATMOS_GRID_CARTESC_REAL_VOLZUY, ATMOS_GRID_CARTESC_REAL_VOLZXV, ATMOS_GRID_CARTESC_REAL_VOL)
          call statistics_total( ka, ks, ke, ia, is, ie, ja, js, je, &
                                 momz_t_tb(:,:,:), 'MOMZ_t_TB',         &
                                 atmos_grid_cartesc_real_volwxy(:,:,:), &
                                 atmos_grid_cartesc_real_totvolwxy      )
          call statistics_total( ka, ks, ke, ia, is, ie, ja, js, je, &
                                 momx_t_tb(:,:,:), 'MOMX_t_TB',         &
                                 atmos_grid_cartesc_real_volzuy(:,:,:), &
                                 atmos_grid_cartesc_real_totvolzuy      )
          call statistics_total( ka, ks, ke, ia, is, ie, ja, js, je, &
                                 momy_t_tb(:,:,:), 'MOMY_t_TB',         &
                                 atmos_grid_cartesc_real_volzxv(:,:,:), &
                                 atmos_grid_cartesc_real_totvolzxv      )
          call statistics_total( ka, ks, ke, ia, is, ie, ja, js, je, &
                                 rhot_t_tb(:,:,:), 'RHOT_t_TB',       &
                                 atmos_grid_cartesc_real_vol(:,:,:),  &
                                 atmos_grid_cartesc_real_totvol       )
          call statistics_total( ka, ks, ke, ia, is, ie, ja, js, je, &
                                 nu(:,:,:),        'Nu',              &
                                 atmos_grid_cartesc_real_vol(:,:,:),  &
                                 atmos_grid_cartesc_real_totvol       )
          call statistics_total( ka, ks, ke, ia, is, ie, ja, js, je, &
                                 ri(:,:,:),        'Ri',              &
                                 atmos_grid_cartesc_real_vol(:,:,:),  &
                                 atmos_grid_cartesc_real_totvol       )
          call statistics_total( ka, ks, ke, ia, is, ie, ja, js, je, &
                                 pr(:,:,:),        'Pr',              &
                                 atmos_grid_cartesc_real_vol(:,:,:),  &
                                 atmos_grid_cartesc_real_totvol       )
 
          do iq = 1, qa
             call statistics_total( ka, ks, ke, ia, is, ie, ja, js, je, &
                                    rhoq_t_tb(:,:,:,iq), trim(tracer_name(iq))//'_t_TB', &
                                    atmos_grid_cartesc_real_vol(:,:,:),                  &
                                    atmos_grid_cartesc_real_totvol                       )
          enddo
          !$acc end data
       endif
 
       !$acc end data
 
    endif
 
    !$omp parallel do default(none) private(i,j,k) OMP_SCHEDULE_ collapse(2) &
    !$omp shared(JS,JE,IS,IE,KS,KE,MOMZ_t,MOMZ_t_TB,MOMX_t,MOMX_t_TB,MOMY_t,MOMY_t_TB,RHOT_t,RHOT_t_TB)
    !$acc kernels
    do j = js, je
    do i = is, ie
    do k = ks, ke
       momz_t(k,i,j) = momz_t(k,i,j) + momz_t_tb(k,i,j)
       momx_t(k,i,j) = momx_t(k,i,j) + momx_t_tb(k,i,j)
       momy_t(k,i,j) = momy_t(k,i,j) + momy_t_tb(k,i,j)
       rhot_t(k,i,j) = rhot_t(k,i,j) + rhot_t_tb(k,i,j)
    enddo
    enddo
    enddo
    !$acc end kernels
 
    do iq = 1, qa
 
       if ( .not. ( iq == i_tke .or. tracer_advc(iq) ) ) cycle
 
       !$omp parallel do private(i,j,k) OMP_SCHEDULE_ collapse(2)
!       !$acc kernels async(iq)
       !$acc kernels
       do j  = js, je
       do i  = is, ie
       do k  = ks, ke
          rhoq_t(k,i,j,iq) = rhoq_t(k,i,j,iq) + rhoq_t_tb(k,i,j,iq)
       enddo
       enddo
       enddo
       !$acc end kernels
 
    enddo
 
    !$acc data create(qflx_x, qflx_y)
 
    if ( monit_west > 0 ) then
       !$acc kernels
       qflx_x(:,:) = 0.0_rp
       !$acc end kernels
       if ( .not. prc_has_w ) then
          do iq = 1, qa
             if ( tracer_advc(iq) .and. tracer_mass(iq) == 1.0_rp ) then
                !$acc kernels
                do j = js, je
                do k = ks, ke
                   qflx_x(k,j) = qflx_x(k,j) + qflx_rhoq(k,is-1,j,xdir,iq)
                end do
                end do
                !$acc end kernels
             end if
          end do
       end if
!$acc update host(qflx_x)
       call monitor_put( monit_west, qflx_x(:,:) )
    end if
    if ( monit_east > 0 ) then
       !$acc kernels
       qflx_x(:,:) = 0.0_rp
       !$acc end kernels
       if ( .not. prc_has_e ) then
          do iq = 1, qa
             if ( tracer_advc(iq) .and. tracer_mass(iq) == 1.0_rp ) then
                !$acc kernels
                do j = js, je
                do k = ks, ke
                   qflx_x(k,j) = qflx_x(k,j) + qflx_rhoq(k,ie,j,xdir,iq)
                end do
                end do
                !$acc end kernels
             end if
          end do
       end if
!$acc update host(qflx_x)
       call monitor_put( monit_east, qflx_x(:,:) )
    end if
    if ( monit_south > 0 ) then
       !$acc kernels
       qflx_y(:,:) = 0.0_rp
       !$acc end kernels
       if ( .not. prc_has_s ) then
          do iq = 1, qa
             if ( tracer_advc(iq) .and. tracer_mass(iq) == 1.0_rp ) then
                !$acc kernels
                do i = is, ie
                do k = ks, ke
                   qflx_y(k,i) = qflx_y(k,i) + qflx_rhoq(k,i,js-1,ydir,iq)
                end do
                end do
                !$acc end kernels
             end if
          end do
       end if
!$acc update host(qflx_y)
       call monitor_put( monit_south, qflx_y(:,:) )
    end if
    if ( monit_north > 0 ) then
       !$acc kernels
       qflx_y(:,:) = 0.0_rp
       !$acc end kernels
       if ( .not. prc_has_n ) then
          do iq = 1, qa
             if ( tracer_advc(iq) .and. tracer_mass(iq) == 1.0_rp ) then
                !$acc kernels
                do i = is, ie
                do k = ks, ke
                   qflx_y(k,i) = qflx_y(k,i) + qflx_rhoq(k,i,je,ydir,iq)
                end do
                end do
                !$acc end kernels
             end if
          end do
       end if
!$acc update host(qflx_y)
       call monitor_put( monit_north, qflx_y(:,:) )
    end if
 
!    !$acc wait
 
    !$acc end data
    !$acc end data
 
    return
  end subroutine atmos_phy_tb_driver_calc_tendency
 
end module mod_atmos_phy_tb_driver