! (C) Copyright 2005- ECMWF.
! (C) Copyright 2013- Meteo-France.
! 
! This software is licensed under the terms of the Apache Licence Version 2.0
! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
! In applying this licence, ECMWF does not waive the privileges and immunities
! granted to it by virtue of its status as an intergovernmental organisation
! nor does it submit to any jurisdiction.
!

SUBROUTINE GSTATS_PRINT(KULOUT,PAVEAVE,KLEN)

!**** *GSTATS_PRINT* - print timing statistics

!     PURPOSE.
!     --------
!       To print out timings gathered by GSTATS


!**   INTERFACE.
!     ----------
!       *CALL* *GSTATS_PRINT*

!        EXPLICIT ARGUMENTS     None
!        --------------------


!        IMPLICIT ARGUMENTS
!        --------------------
!        Module YOMSTATS

!     METHOD.
!     -------


!     EXTERNALS.   
!     ----------   

!     REFERENCE.
!     ----------
!        ECMWF Research Department documentation of the IFS

!     AUTHOR.
!     -------
!        Mats Hamrud ECMWF

!     MODIFICATIONS.
!     --------------
!        ORIGINAL : 98-11-15
!        D.Salmond: 99-09-21 : Timer for SLCOMM2
!        G.Mozdzynski 05-09-25 : fix master ncalls overwrite for nproc>1
!        C.Larsson    8-May-2006 : Added xml file output
!        G.Mozdzynski 16-Oct-2007 : xml file output under switch LXML_STATS
!        P.Towers     11-May-2011 : mpl comms statistics output
!        F. Vana  05-Mar-2015  Support for single precision
!        G. Mozdzynski 18-Aug-2015 Avoid confusion, procs are tasks
!        M. Plesske 05-Nov-2025 added LCSV_STATS
!     ------------------------------------------------------------------

USE EC_PARKIND, ONLY: JPRD, JPIM
USE YOMGSTATS, ONLY: JPMAXDELAYS, JPMAXSTAT, NPROC_STATS, LXML_STATS, LCSV_STATS, MYPROC_STATS, LSYNCSTATS, &
  &                  LDETAILED_STATS, LSTATS_COMMS, LSTATS_OMP, LBARRIER_STATS2, JBMAXBASE, &
  &                  NBAR_PTR, CCDESC, CCTYPE, LSTATS, TIMESUM, TIMESQSUM, TIMEMAX, TIMESUMB, &
  &                  TIMELCALL, NCALLS, TTCPUSUM, TVCPUSUM, JPTAGSTAT, NPRCIDS_STATS, LSTATS_MPL, &
  &                  NUMSEND, SENDBYTES, NUMRECV, RECVBYTES, LSTATSCPU, NPRNT_STATS, TIME_START, &
  &                  UNKNOWN_NUMSEND, UNKNOWN_NUMRECV, UNKNOWN_SENDBYTES, UNKNOWN_RECVBYTES, &
  &                  NDELAY_INDEX, NDELAY_COUNTER, TDELAY_VALUE, CDELAY_TIME, LTRACE_STATS, &
  &                  NCALLS_TOTAL, NTRACE_STATS, NCALL_TRACE, TIME_TRACE, LSTATS_MEM, NTMEM
USE MPL_MODULE, ONLY: MPL_SEND, MPL_RECV, MPL_BARRIER

IMPLICIT NONE

INTEGER(KIND=JPIM) :: KULOUT,KLEN
REAL(KIND=JPRD) :: PAVEAVE(0:KLEN)
CHARACTER*7  CLACTION(0:3)
CHARACTER(LEN=JPMAXDELAYS*10) CLTEMP
CHARACTER(LEN=128) :: CLCSVNAME
CHARACTER(LEN=32) :: CLCSVCOLS(15)

INTEGER(KIND=JPIM),PARAMETER :: JPARRAYS=8
REAL(KIND=JPRD) :: ZREABUF(JPARRAYS*(JPMAXSTAT+1))
REAL(KIND=JPRD) :: ZAVEAVE(0:JPMAXSTAT),ZAVEMAX(0:JPMAXSTAT),ZTIMELCALL(0:JPMAXSTAT),&
         &ZTHISTIME(0:JPMAXSTAT),ZFRACMAX(0:JPMAXSTAT),&
         &ZSUMMAX(0:JPMAXSTAT),ZSUMTOT(0:JPMAXSTAT)
REAL(KIND=JPRD) :: ZT_SUM,ZT_SUM2,ZT_SUM3,ZT_SUMIO,ZT_SUM4,ZT_SUM5,ZT_SUMB
REAL(KIND=JPRD) :: ZDELAY,ZDELAY_MAX

REAL(KIND=JPRD) :: ZMPL(NPROC_STATS)
REAL(KIND=JPRD) :: ZBAR(NPROC_STATS)
REAL(KIND=JPRD) :: ZGBR(NPROC_STATS)
REAL(KIND=JPRD) :: ZGB2(NPROC_STATS)
REAL(KIND=JPRD) :: ZOMP(NPROC_STATS)
REAL(KIND=JPRD) :: ZIO (NPROC_STATS)
REAL(KIND=JPRD) :: ZSER(NPROC_STATS)
REAL(KIND=JPRD) :: ZMXD(NPROC_STATS)

INTEGER(KIND=JPIM) :: ICALLSX(0:JPMAXSTAT)

!     LOCAL INTEGER SCALARS
INTEGER(KIND=JPIM) :: ICALLS, ILBUF, ILSEND, ILRECV, &
             &ISEND, ITAG, JJ, JNUM, JROC, JCALL, ICALLER,IACTION
INTEGER(KIND=JPIM) :: IMEM, INUM, JMEM
INTEGER(KIND=JPIM) :: JDELAY, IDELAY
INTEGER(KIND=JPIM) :: NSEND,NRECV

!     LOCAL REAL SCALARS
REAL(KIND=JPRD) :: ZAVE, ZAVETCPU, ZAVEVCPU, ZCOMTIM, ZDETAIL,&
          &ZFRAC, ZMAX, ZMEAN, ZSTDDEV, ZSUM, ZSUMB, &
          &ZTOTAL, ZTOTCPU, ZTOTUNBAL, ZTOTVCPU, &
          &ZUNBAL, ZMEANT, ZMAXT

REAL(KIND=JPRD)    :: SBYTES,RBYTES,SENDRATE,RECVRATE
REAL(KIND=JPRD)    :: AVGSENDLEN,AVGRECVLEN
REAL(KIND=JPRD)    :: MAXCOMMTIME(501:1000)
REAL(KIND=JPRD)    :: TOTSENDBYTES(501:1000)
REAL(KIND=JPRD)    :: TOTRECVBYTES(501:1000)
REAL(KIND=JPRD)    :: TOTSENDBYTESSUM
REAL(KIND=JPRD)    :: TOTRECVBYTESSUM
INTEGER(KIND=JPIM) :: IXMLLUN, ICSVUN

!     ------------------------------------------------------------------

ILBUF = JPARRAYS*(JPMAXSTAT+1)
ILRECV = 500*4
ZAVEAVE(:) = 0.0_JPRD
ZAVEMAX(:) = 0.0_JPRD
ZFRACMAX(:)= 0.0_JPRD
ZSUMMAX(:)= 0.0_JPRD
ZSUMTOT(:)= 0.0_JPRD

! OPEN GSTATS.XML for xml statistics
IF(LXML_STATS .AND. MYPROC_STATS==1)THEN
  IXMLLUN=40
  OPEN (UNIT=IXMLLUN, FILE='gstats.xml',ACTION='write')
  WRITE(IXMLLUN,'(A)')'<?xml version="1.0" encoding="UTF-8"?>'
  WRITE(IXMLLUN,'(A)')'<gstats '
  WRITE(IXMLLUN,'(A)')' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"&
  & xsi:schemaLocation="http://www.ecmwf.int/services/prepifs/gstats   ./gstats.xsd">'
ENDIF

WRITE(KULOUT,'(A)')'===-=== START OF TIMING STATISTICS ===-==='
IF(LSYNCSTATS.AND.NPROC_STATS>1) THEN
  WRITE(KULOUT,'(A)')'START OF TIMINGS SYNCRONIZED'
ENDIF
IF(LDETAILED_STATS) THEN
  LSTATS_COMMS=.TRUE.
  LSTATS_OMP=.TRUE.
ENDIF
IF(LBARRIER_STATS2) THEN
  DO JNUM=1,JBMAXBASE
    IF(NBAR_PTR(JNUM) > 0) THEN
      INUM=NBAR_PTR(JNUM)
      CCDESC(INUM)=CCDESC(JNUM)
      CCTYPE(INUM)='GB2'
    ENDIF
  ENDDO
ENDIF

! Number 400 and 401 are reserved for overall GSTATS timing, so if no label is set, use "GSTATS" and "GSTATS HOOK"
! These are the labels used in gstats_ifs_setup.F90
IF (LEN_TRIM(CCDESC(400)) == 0) THEN
  CCDESC(400) = 'GSTATS'
ENDIF
IF (LEN_TRIM(CCDESC(401)) == 0) THEN
  CCDESC(401) = 'GSTATS HOOK'
ENDIF

! CSV OUTPUT FOR EACH RANK
IF (LSTATS .AND. LCSV_STATS) THEN
  ! OPENING THE FILE
  WRITE(CLCSVNAME, '(A,I0,A)') 'gstats.', MYPROC_STATS, '.csv'
  IF (NPROC_STATS <= 1) WRITE(KULOUT, '(A,A)')    "GSTATS writing file ", CLCSVNAME
  IF (NPROC_STATS  > 1) WRITE(KULOUT, '(A,I0,A)') "GSTATS writing files gstats.{1..", NPROC_STATS, "}.csv"
  OPEN (NEWUNIT=ICSVUN, FILE=CLCSVNAME, ACTION='write')
  !write header
  WRITE(ICSVUN,'(A)') 'GSTATS ID, SECTION, NUMCALLS, TOTAL TIME, MAX TIME, AVE TIME, AVE TIME CPU, AVE TIME VECTOR, STDDEV, NUMSENDS, BYTES SEND, AVE BYTES SEND, NUMRECVS, BYTES RECV, AVE BYTES RECV'

  ! WRITING OUTPUT FOR EVERY RECORDED GSTATS SECTION
  DO JNUM=0,JPMAXSTAT
    CLCSVCOLS(:)='NaN'
    IF (NCALLS(JNUM) > 1) THEN   
      ! TIMING INFORMATION
      ICALLS = NCALLS(JNUM)/2
      IF (ICALLS > 1) THEN
        ZSTDDEV = 1000._JPRD*SQRT(MAX((TIMESQSUM(JNUM)-TIMESUM(JNUM)**2/REAL(ICALLS,JPRD))/REAL(ICALLS-1,JPRD),0.0_JPRD))
      ELSE
        ZSTDDEV= 0.0_JPRD
      ENDIF
      WRITE(CLCSVCOLS(1), '(I0)') JNUM
      CLCSVCOLS(2) = CCDESC(JNUM)
      WRITE(CLCSVCOLS(3), '(I0)') ICALLS
      WRITE(CLCSVCOLS(4), '(F0.6)') TIMESUM(JNUM)
      WRITE(CLCSVCOLS(5), '(F0.6)') TIMEMAX(JNUM)*1000._JPRD
      WRITE(CLCSVCOLS(6), '(F0.6)') TIMESUM(JNUM)/ICALLS*1000._JPRD
      WRITE(CLCSVCOLS(7), '(F0.6)') TTCPUSUM(JNUM)/ICALLS*1000._JPRD
      WRITE(CLCSVCOLS(8), '(F0.6)') TVCPUSUM(JNUM)/ICALLS*1000._JPRD
      WRITE(CLCSVCOLS(9), '(F0.6)') ZSTDDEV
     
      ! COMMUNICATION INFORMATION
      IF (LSTATS_MPL .AND. JNUM > 500 .AND. JNUM < 1001) THEN
        IF(NUMSEND(JNUM) /= 0) THEN
          AVGSENDLEN=SENDBYTES(JNUM)*1.E-3_JPRD/NUMSEND(JNUM)
        ELSE
          AVGSENDLEN=0.0_JPRD
        ENDIF
        IF(NUMRECV(JNUM) /= 0) THEN
          AVGRECVLEN=RECVBYTES(JNUM)*1.E-3_JPRD/NUMRECV(JNUM)
        ELSE
          AVGRECVLEN=0.0_JPRD
        ENDIF
        WRITE(CLCSVCOLS(10), '(I0)') NUMSEND(JNUM)
        WRITE(CLCSVCOLS(11), '(F0.6)') SENDBYTES(JNUM)
        WRITE(CLCSVCOLS(12), '(F0.6)') AVGSENDLEN
        WRITE(CLCSVCOLS(13), '(I0)') NUMRECV(JNUM)
        WRITE(CLCSVCOLS(14), '(F0.6)') RECVBYTES(JNUM)
        WRITE(CLCSVCOLS(15), '(F0.6)') AVGRECVLEN
      ENDIF
      ! WRITE TO CSV FILE
      DO JJ = 1, 14
        WRITE(ICSVUN, '(2A)', advance='no') TRIM(CLCSVCOLS(JJ)), ','
      ENDDO
      WRITE(ICSVUN, '(A)') TRIM(CLCSVCOLS(15))
    ENDIF
  ENDDO

  ! CLOSING CSV FILES
  CLOSE(ICSVUN) 
ENDIF

! COLLECTING ALL INFORMATION AT RANK 1 AND COMPUTING FURTHER METRICS
IF (LSTATS .AND. MYPROC_STATS /= 1) THEN
  JJ = 1
  DO JNUM=0,JPMAXSTAT
    ZREABUF(JJ  ) = TIMESUM(JNUM)
    ZREABUF(JJ+1) = TIMESQSUM(JNUM)
    ZREABUF(JJ+2) = TIMEMAX(JNUM)
    ZREABUF(JJ+3) = TIMESUMB(JNUM)
    ZREABUF(JJ+4) = TIMELCALL(JNUM)
    ZREABUF(JJ+5) = NCALLS(JNUM)
    ZREABUF(JJ+6) = TTCPUSUM(JNUM)
    ZREABUF(JJ+7) = TVCPUSUM(JNUM)
    JJ = JJ+JPARRAYS
  ENDDO
  ILSEND = ILBUF
  ISEND =1
  ITAG = JPTAGSTAT
  CALL MPL_SEND(ZREABUF(1:ILSEND),KDEST=NPRCIDS_STATS(ISEND), &
       & KTAG=ITAG,CDSTRING='GSTATS_PRINT:')

  IF(LSTATS_MPL) THEN
    JJ=1
    DO JNUM=501,1000
      ZREABUF(JJ  ) = NUMSEND(JNUM)
      ZREABUF(JJ+1) = SENDBYTES(JNUM)
      ZREABUF(JJ+2) = NUMRECV(JNUM)
      ZREABUF(JJ+3) = RECVBYTES(JNUM)
      JJ=JJ+4
    ENDDO

    ILSEND = JJ-1
    ITAG = JPTAGSTAT + 1

    CALL MPL_SEND(ZREABUF(1:ILSEND),KDEST=NPRCIDS_STATS(ISEND), &
         & KTAG=ITAG,CDSTRING='GSTATS_PRINT:')

  ENDIF
ELSEIF(LSTATS) THEN
  IF(LSTATS_MPL) THEN
    DO JNUM=501,1000
      MAXCOMMTIME(JNUM)=0.0_JPRD
      TOTSENDBYTES(JNUM)=0.0_JPRD
      TOTRECVBYTES(JNUM)=0.0_JPRD
    ENDDO
  ENDIF
  DO JROC=1,NPROC_STATS
    IF (JROC /= 1) THEN
      ITAG = JPTAGSTAT
      CALL MPL_RECV(ZREABUF(1:ILBUF),KSOURCE=NPRCIDS_STATS(JROC), &
       & KTAG=ITAG,CDSTRING='GSTATS_PRINT:')
      JJ = 1
      DO JNUM=0,JPMAXSTAT
        TIMESUM(JNUM)   = ZREABUF(JJ  )
        TIMESQSUM(JNUM) = ZREABUF(JJ+1)
        TIMEMAX(JNUM)   = ZREABUF(JJ+2)
        TIMESUMB(JNUM)  = ZREABUF(JJ+3)
        TIMELCALL(JNUM) = ZREABUF(JJ+4)
        ICALLSX(JNUM)    = NINT(ZREABUF(JJ+5))
        TTCPUSUM(JNUM)  = ZREABUF(JJ+6)
        TVCPUSUM(JNUM)  = ZREABUF(JJ+7)
        JJ = JJ+JPARRAYS
      ENDDO

      IF(LSTATS_MPL) THEN
        ITAG = JPTAGSTAT+1
        CALL MPL_RECV(ZREABUF(1:ILRECV),KSOURCE=NPRCIDS_STATS(JROC), &
         & KTAG=ITAG,CDSTRING='GSTATS_PRINT:')
        JJ = 1
        DO JNUM=501,1000
          NUMSEND(JNUM)   = NINT(ZREABUF(JJ))
          SENDBYTES(JNUM) = ZREABUF(JJ+1)
          NUMRECV(JNUM)   = NINT(ZREABUF(JJ+2))
          RECVBYTES(JNUM) = ZREABUF(JJ+3)
          JJ=JJ+4
        ENDDO
      ENDIF

    ELSE
      ICALLSX(:)=NCALLS(:)

    ENDIF
    IF (JROC == 1) THEN
      ZTOTAL=TIMESUM(0)
      ZTOTCPU = TTCPUSUM(0)
      ZTOTVCPU = TVCPUSUM(0)
    ENDIF
    IF(.NOT. LSTATSCPU) THEN
      TTCPUSUM(1:JPMAXSTAT) = -0.0_JPRD
      TVCPUSUM(1:JPMAXSTAT) = -0.0_JPRD
    ENDIF
    ZT_SUM=0.0_JPRD
    ZT_SUM2=0.0_JPRD
    ZT_SUM3=0.0_JPRD
    ZT_SUM4=0.0_JPRD
    ZT_SUM5=0.0_JPRD
    ZT_SUMIO=0.0_JPRD
    ZT_SUMB=0.0_JPRD
    IF( LDETAILED_STATS .AND. JROC <= NPRNT_STATS ) THEN
      WRITE(KULOUT,'(A,I4)') 'TIMING STATISTICS:TASK=',JROC
      IF(LXML_STATS)THEN
        WRITE(IXMLLUN,'(A,I4,A)')'<timing task="',JROC,'">'
      ENDIF
      IF(NPROC_STATS > 1) THEN
        WRITE(KULOUT,'(A,F6.1,A)')'STARTUP COST ',&
         &TIME_START(JROC),' SECONDS'
      ENDIF
      WRITE(KULOUT,'(A)')&
       &' NUM     ROUTINE                                     '//&
       &'CALLS  SUM(s)   AVE(ms)   CPUAVE(ms) VAVE(ms) '//&
       &'STDDEV(ms)  MAX(ms) '//&
       &'SUMB(s) FRAC(%)'
    ENDIF
    DO JNUM=0,JPMAXSTAT
      IF(ICALLSX(JNUM) > 1) THEN
        ICALLS = ICALLSX(JNUM)/2
        ZSUM = TIMESUM(JNUM)
        ZAVE = TIMESUM(JNUM)/ICALLS*1000._JPRD
        ZMAX = TIMEMAX(JNUM)*1000._JPRD
        ZSUMB = TIMESUMB(JNUM)
        ZFRAC = TIMESUM(JNUM)/ZTOTAL*100.0_JPRD
        ZFRACMAX(JNUM)=MAX(ZFRACMAX(JNUM),ZFRAC)
        ZSUMMAX(JNUM)=MAX(ZSUMMAX(JNUM),TIMESUM(JNUM))
        ZSUMTOT(JNUM)=ZSUMTOT(JNUM)+ZSUM
        ZAVEAVE(JNUM)=ZAVEAVE(JNUM)+ZAVE
        ZAVEMAX(JNUM)=MAX(ZAVEMAX(JNUM),ZAVE)
        ZAVETCPU = TTCPUSUM(JNUM)/ICALLS*1000._JPRD
        ZAVEVCPU = TVCPUSUM(JNUM)/ICALLS*1000._JPRD
        IF(ICALLS > 1 ) THEN
          ZSTDDEV = 1000._JPRD*&
           &SQRT(MAX((TIMESQSUM(JNUM)-TIMESUM(JNUM)**2/ICALLS)&
           &/(ICALLS-1),0.0_JPRD))
        ELSE
          ZSTDDEV = 0.0_JPRD
        ENDIF
        IF(CCTYPE(JNUM).EQ.'MPL') THEN
          ZT_SUM=ZT_SUM+ZSUM
          ZT_SUMB=ZT_SUMB+ZSUMB
        ENDIF
        IF(CCTYPE(JNUM).EQ.'BAR' .OR. CCTYPE(JNUM).EQ.'GBR' .OR. CCTYPE(JNUM).EQ.'GB2' ) THEN
          ZT_SUM4=ZT_SUM4+ZSUM
          ZT_SUMB=ZT_SUMB+ZSUMB
        ENDIF
        IF(CCTYPE(JNUM).EQ.'OMP') THEN
          ZT_SUM2=ZT_SUM2+ZSUM
          ZT_SUMB=ZT_SUMB+ZSUMB
        ENDIF
        IF(CCTYPE(JNUM).EQ.'IO-') THEN
          ZT_SUMIO=ZT_SUMIO+ZSUM
          ZT_SUMB=ZT_SUMB+ZSUMB
        ENDIF
        IF(CCTYPE(JNUM).EQ.'SER') THEN
          ZT_SUM3=ZT_SUM3+ZSUM
          ZT_SUMB=ZT_SUMB+ZSUMB
        ENDIF
        IF(CCTYPE(JNUM).EQ.'MXD') THEN
          ZT_SUM5=ZT_SUM5+ZSUM
          ZT_SUMB=ZT_SUMB+ZSUMB
        ENDIF
        IF( LDETAILED_STATS .AND. JROC <= NPRNT_STATS ) THEN
            WRITE(KULOUT,'(I4,1X,A3,1X,A40,1X,I5,6(1X,F9.1),1X,F5.1,1X,F8.2)')&
             &JNUM,CCTYPE(JNUM),CCDESC(JNUM),ICALLS,ZSUM,ZAVE,ZAVETCPU,ZAVEVCPU,&
             &ZSTDDEV,ZMAX,ZSUMB,ZFRAC
            IF(LXML_STATS)THEN
              WRITE(IXMLLUN,&
               & '(A,I4,A,/,A,A40,A,/,A,I5,A,/,6(A,F9.1,A,/),A,F5.1,A,/,A,F8.2,A,/,A)')&
               & '<num id="',JNUM,'">',&
               & '<description>',CCDESC(JNUM),'</description>',&
               & '<sum unit="seconds">',ICALLS,'</sum>',&
               & '<sumcall unit="ms">',ZSUM,'</sumcall>',&
               & '<average unit="ms">',ZAVE,'</average>',&
               & '<cpuaverage unit="ms">',ZAVETCPU,'</cpuaverage>',&
               & '<vave unit="ms">',ZAVEVCPU,'</vave>',&
               & '<stddev unit="ms">',ZSTDDEV,'</stddev>',&
               & '<max unit="ms">',ZMAX,'</max>',&
               & '<sumb unit="seconds">',ZSUMB,'</sumb>',&
               & '<frac unit="percent">',ZFRAC,'</frac>',&
               & '</num>'
             ENDIF
        ENDIF
      ENDIF
    ENDDO
    IF( LDETAILED_STATS .AND. JROC <= NPRNT_STATS ) THEN
      WRITE(KULOUT,*) ''
      WRITE(KULOUT,'((A,2F8.1))')&
       &'CPU-TIME AND VECTOR CPU-TIME AS PERCENT OF TOTAL ',&
       &TTCPUSUM(0)/TIMESUM(0)*100.0_JPRD,TVCPUSUM(0)/TIMESUM(0)*100.0_JPRD
      IF(LXML_STATS)THEN
        WRITE(IXMLLUN,'((A,F8.1,A,/,A,F8.1,A))')&
         &'<cpufraction>',&
         &TTCPUSUM(0)/TIMESUM(0)*100.0_JPRD,&
         &'</cpufraction>',&
         &'<cpuvectorfraction>',&
         &TVCPUSUM(0)/TIMESUM(0)*100.0_JPRD,&
         &'</cpuvectorfraction>'
      ENDIF


      IF(ZT_SUM > 0.0_JPRD) THEN
        WRITE(KULOUT,'(A,F10.1,A,F6.2,A)')'SUMMED TIME IN COMMUNICATIONS   = '&
         & ,ZT_SUM, ' SECONDS ',ZT_SUM/TIMESUM(0)*100.0_JPRD,&
         &' PERCENT OF TOTAL'
        IF(LXML_STATS)THEN
          WRITE(IXMLLUN,'(A,F10.1,A,/,A,F6.2,A)')'<zcom unit="seconds">',&
           &ZT_SUM,'</zcom>',&
           &'<fraczcom unit="percent">',ZT_SUM/TIMESUM(0)*100.0_JPRD,&
           &'</fraczcom>'
        ENDIF
      ENDIF
      IF(ZT_SUM2 > 0.0_JPRD) THEN
        WRITE(KULOUT,'(A,F10.1,A,F6.2,A)')'SUMMED TIME IN PARALLEL REGIONS = '&
         & ,ZT_SUM2, ' SECONDS ',ZT_SUM2/TIMESUM(0)*100.0_JPRD,&
         &' PERCENT OF TOTAL'
        IF(LXML_STATS)THEN
          WRITE(IXMLLUN,'(A,F10.1,A,/,A,F6.2,A)') &
           &'<parallelztime unit="seconds">',&
           &ZT_SUM2, '</parallelztime>',&
           &'<fracparallelztime unit="percent">',ZT_SUM2/TIMESUM(0)*100.0_JPRD,&
           &'</fracparallelztime>'
        ENDIF
      ENDIF
      IF(ZT_SUMIO > 0.0_JPRD) THEN
        WRITE(KULOUT,'(A,F10.1,A,F6.2,A)')'SUMMED TIME IN I/O SECTIONS     = '&
         & ,ZT_SUMIO, ' SECONDS ',ZT_SUMIO/TIMESUM(0)*100.0_JPRD,&
         &' PERCENT OF TOTAL'
        IF(LXML_STATS)THEN
          WRITE(IXMLLUN,'(A,F10.1,A,/,A,F6.2,A)')'<ioztime unit="seconds">',&
           &ZT_SUMIO, '</ioztime>',&
           &'<fracioztime unit="percent">',ZT_SUMIO/TIMESUM(0)*100.0_JPRD,&
           &'</fracioztime>'
        ENDIF
      ENDIF
      IF(ZT_SUM3 > 0.0_JPRD) THEN
        WRITE(KULOUT,'(A,F10.1,A,F6.2,A)')'SUMMED TIME IN SERIAL SECTIONS  = '&
        & ,ZT_SUM3, ' SECONDS ',ZT_SUM3/TIMESUM(0)*100.0_JPRD,&
         &' PERCENT OF TOTAL'
        IF(LXML_STATS)THEN
          WRITE(IXMLLUN,'(A,F10.1,A,/,A,F6.2,A)')'<serialztime unit="seconds">',&
           & ZT_SUM3,'</serialztime>',&
           &'<fracserialztime unit="percent">',&
           &ZT_SUM3/TIMESUM(0)*100.0_JPRD,&
           &'</fracserialztime>'
        ENDIF
      ENDIF
      IF(ZT_SUM4 > 0.0_JPRD) THEN
        WRITE(KULOUT,'(A,F10.1,A,F6.2,A)')'SUMMED TIME IN BARRIERS         = '&
         & ,ZT_SUM4, ' SECONDS ',ZT_SUM4/TIMESUM(0)*100.0_JPRD,&
         &' PERCENT OF TOTAL'
        IF(LXML_STATS)THEN
          WRITE(IXMLLUN,'(A,F10.1,A,/,A,F6.2,A)')&
           &'<barrierztime unit="seconds">',&
           &ZT_SUM4,'</barrierztime>',&
           & '<fracbarrierztime unit="percent">',&
           & ZT_SUM4/TIMESUM(0)*100.0_JPRD,'</fracbarrierztime>'
        ENDIF
      ENDIF
      IF(ZT_SUM5 > 0.0_JPRD) THEN
        WRITE(KULOUT,'(A,F10.1,A,F6.2,A)')'SUMMED TIME IN MIXED SECTIONS   = '&
         & ,ZT_SUM5, ' SECONDS ',ZT_SUM5/TIMESUM(0)*100.0_JPRD,&
         &' PERCENT OF TOTAL'
        WRITE(IXMLLUN,'(A,F10.1,A,/,A,F6.2,A)')&
         &'<mixedztime unit="seconds">',&
         &ZT_SUM5,'</mixedztime>',&
         & '<fracmixedztime unit="percent">',&
         & ZT_SUM5/TIMESUM(0)*100.0_JPRD,'</fracmixedztime>'
      ENDIF
      IF(LSTATS_COMMS.AND.LSTATS_OMP)THEN
        WRITE(KULOUT,'(A,F8.2)')'FRACTION OF TOTAL TIME ACCOUNTED FOR ',&
         & (ZT_SUM+ZT_SUM2+ZT_SUMIO+ZT_SUM3+ZT_SUM4+ZT_SUM5)/TIMESUM(0)*100.0_JPRD
        WRITE(KULOUT,'(A,F8.2)')'FRACTION OF TOTAL TIME ACCOUNTED FOR INCLUDING SUMB ',&
         & (ZT_SUM+ZT_SUM2+ZT_SUMIO+ZT_SUM3+ZT_SUM4+ZT_SUM5+ZT_SUMB)/TIMESUM(0)*100.0_JPRD
        WRITE(KULOUT,'(" ")')
        IF(LXML_STATS)THEN
          WRITE(IXMLLUN,'(A,F8.2,A)')'<fractotal unit="percent">',&
           &(ZT_SUM+ZT_SUM2+ZT_SUMIO+ZT_SUM3+ZT_SUM4+ZT_SUM5)/TIMESUM(0)*100.0_JPRD,&
           &'</fractotal>'
        ENDIF
      ENDIF
    ENDIF
    IF( LDETAILED_STATS .AND. JROC < 3 ) THEN
      IF(LXML_STATS)THEN
        WRITE(IXMLLUN,'(A)')'</timing>'
      ENDIF
    ENDIF
    IF( LSTATS_MPL .AND. JROC <= NPRNT_STATS ) THEN
      WRITE(KULOUT,'(/,A,I4,/)') 'COMMUNICATIONS STATISTICS:TASK=',JROC
      WRITE(KULOUT,'(A)') &
       &' NUM     ROUTINE                              '//&
       &'  SUM(s)   SENDS  AVG(kb) TOTAL(MB) MB/s '//&
       &'  RECVS  AVG(kb) TOTAL(MB) MB/s '

      DO JNUM=501,1000
        IF((NUMSEND(JNUM) /= 0 .OR. NUMRECV(JNUM) /= 0 ) &
           .AND. TIMESUM(JNUM) > 0.0_JPRD) THEN
           SENDRATE=SENDBYTES(JNUM)*1.E-6_JPRD/TIMESUM(JNUM)
           RECVRATE=RECVBYTES(JNUM)*1.E-6_JPRD/TIMESUM(JNUM)
           IF(NUMSEND(JNUM) /= 0) THEN
             AVGSENDLEN=SENDBYTES(JNUM)*1.E-3_JPRD/NUMSEND(JNUM)
           ELSE
             AVGSENDLEN=0.0_JPRD
           ENDIF
           IF(NUMRECV(JNUM) /= 0) THEN
             AVGRECVLEN=RECVBYTES(JNUM)*1.E-3_JPRD/NUMRECV(JNUM)
           ELSE
             AVGRECVLEN=0.0_JPRD
           ENDIF
           WRITE(KULOUT,'(I6,1X,A40,f6.1,2(I8,3F8.1))') &
            &  JNUM,CCDESC(JNUM),TIMESUM(JNUM),&
            &  NUMSEND(JNUM),AVGSENDLEN,SENDBYTES(JNUM)*1.E-6_JPRD, SENDRATE, &
            &  NUMRECV(JNUM),AVGRECVLEN,RECVBYTES(JNUM)*1.E-6_JPRD, RECVRATE
        ENDIF
      ENDDO
      WRITE(KULOUT,'(/,A,I4,/)') 'UNKNOWN COMMUNICATIONS STATISTICS:TASK=', JROC
      WRITE(KULOUT,'(A)') &
       &' NUM     BEFORE ROUTINE                        '//&
       &'    SENDS TOTAL(MB) '//&
       &'RECVS TOTAL(MB)  '
      DO JNUM=501,1000
        IF(UNKNOWN_NUMSEND(JNUM) /= 0 .OR. UNKNOWN_NUMRECV(JNUM) /= 0 ) THEN
           WRITE(KULOUT,'(I6,1X,A40,2(I8,F8.1))') &
            &  JNUM,CCDESC(JNUM),&
            &  UNKNOWN_NUMSEND(JNUM),UNKNOWN_SENDBYTES(JNUM)*1.E-6_JPRD, &
            &  UNKNOWN_NUMRECV(JNUM),UNKNOWN_RECVBYTES(JNUM)*1.E-6_JPRD
        ENDIF
      ENDDO
      WRITE(KULOUT,'(7x,"TOTAL",35x,2(I8,F8.1),//)') &
       & SUM(UNKNOWN_NUMSEND(:)),SUM(UNKNOWN_SENDBYTES(:))*1.E-6_JPRD , &
       & SUM(UNKNOWN_NUMRECV(:)),SUM(UNKNOWN_RECVBYTES(:))*1.E-6_JPRD

     ENDIF

     IF(LSTATS_MPL) THEN
      DO JNUM=501,1000
         TOTSENDBYTES(JNUM) = TOTSENDBYTES(JNUM) + SENDBYTES(JNUM)
         TOTRECVBYTES(JNUM) = TOTRECVBYTES(JNUM) + RECVBYTES(JNUM)
         IF(SENDBYTES(JNUM).GT.0.0_JPRD.OR. &
         &  RECVBYTES(JNUM).GT.0.0_JPRD) THEN
           MAXCOMMTIME(JNUM)  = MAX(MAXCOMMTIME(JNUM),TIMESUM(JNUM))
         ENDIF
      ENDDO
     ENDIF

  ENDDO
  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A)')'<timing_all_tasks>'
  ENDIF
  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(A)') 'STATS FOR ALL TASKS'
  WRITE(KULOUT,'(A)') &
  &' NUM ROUTINE                                     CALLS      MEAN(ms)       MAX(ms)   FRAC(%)  UNBAL(%)'
  ZTOTUNBAL = 0.0_JPRD
  DO JNUM=0,500
    IF(NCALLS(JNUM) > 1) THEN
      ICALLS = NCALLS(JNUM)/2
      ZMEAN = ZAVEAVE(JNUM)/NPROC_STATS
      ZMAX  = ZAVEMAX(JNUM)
      ZMEANT = ZSUMTOT(JNUM)/NPROC_STATS
      ZMAXT  = ZSUMMAX(JNUM)
      IF(ZMEANT .NE. 0._JPRD)THEN
        ZUNBAL= (ZMAXT-ZMEANT)/ZTOTAL*100._JPRD
      ELSE
        ZUNBAL=0._JPRD
      ENDIF
      ZFRAC=ZFRACMAX(JNUM)
      WRITE(KULOUT,'(I4,1X,A40,1X,I8,2(1X,F13.3),2(1X,F9.2))')&
       &JNUM,CCDESC(JNUM),ICALLS,ZMEAN,ZMAX,ZFRAC,ZUNBAL

      IF(LXML_STATS)THEN
        WRITE(IXMLLUN,'(A,I4,A,/,A,A40,A,/,A,I8,A,2(A,F13.3,A,/),2(A,F9.2,A,/),A)')&
         &'<item id="',JNUM,'">',&
         &'<description>',CCDESC(JNUM),'</description>',&
         &'<calls>',ICALLS,'</calls>',&
         &'<mean unit="ms">',ZMEAN,'</mean>',&
         &'<max unit="ms">',ZMAX,'</max>',&
         &'<fraction unit="percent">',ZFRAC,'</fraction>',&
         &'<unbalanced unit="percent">',ZUNBAL,'</unbalanced>','</item>'
      ENDIF
    ENDIF
  ENDDO
  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A)')'</timing_all_tasks>'
  ENDIF

IF(LSTATS_COMMS)THEN
  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A)')'<timing_communications>'
  ENDIF
  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(A)') 'STATS FOR COMMUNICATIONS'
  WRITE(KULOUT,'(A)')  &
  &' NUM ROUTINE                                     CALLS    MEAN(ms)  MAX(ms)   FRAC(%)  UNBAL(%)'
  ZT_SUM=0._JPRD
  DO JNUM=500,JPMAXSTAT
    IF((CCTYPE(JNUM).EQ."MPL".OR.CCTYPE(JNUM).EQ."BAR".OR.CCTYPE(JNUM).EQ."GBR".OR.CCTYPE(JNUM).EQ."GB2") &
     & .AND.NCALLS(JNUM) > 1) THEN
      ICALLS = NCALLS(JNUM)/2
      ZMEAN = ZAVEAVE(JNUM)/NPROC_STATS
      ZMAX  = ZAVEMAX(JNUM)
      ZMEANT = ZSUMTOT(JNUM)/NPROC_STATS
      ZMAXT  = ZSUMMAX(JNUM)
      IF(ZMEANT .NE. 0._JPRD)THEN
        ZUNBAL= (ZMAXT-ZMEANT)/ZTOTAL*100._JPRD
      ELSE
        ZUNBAL=0._JPRD
      ENDIF
      ZFRAC=ZFRACMAX(JNUM)
      ZTOTUNBAL = ZTOTUNBAL+(ZMAXT-ZMEANT)
      WRITE(KULOUT,'(I4,1X,A40,1X,I8,2(1X,F9.1),2(1X,F9.2))')&
       &JNUM,CCDESC(JNUM),ICALLS,ZMEAN,ZMAX,ZFRAC,ZUNBAL
      IF(LXML_STATS)THEN
        WRITE(IXMLLUN,'(A,I4,A,/,A,A40,A,/,A,I8,A,/,2(A,F9.1,A,/),2(A,F9.2,A,/),A)')&
         &'<comitem id="',JNUM,'">',&
         &'<description>',CCDESC(JNUM),'</description>',&
         &'<calls>',ICALLS,'</calls>',&
         &'<mean unit="ms">',ZMEAN,'</mean>',&
         &'<max unit="ms">',ZMAX,'</max>',&
         &'<fraction unit="percent">',ZFRAC,'</fraction>',&
         &'<unbalanced unit="percent">',ZUNBAL,'</unbalanced>','</comitem>'
      ENDIF
      ZT_SUM=ZT_SUM+ZMEANT
    ENDIF
  ENDDO

  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(A,F10.1,A)')'SUMMED TIME IN COMMUNICATIONS   = ',ZT_SUM, ' SECONDS '
  IF(LSTATS_MPL) THEN
    WRITE(KULOUT,'(/,A,/)') 'TOTAL COMMUNICATIONS VOLUMES AND BANDWIDTH'
    WRITE(KULOUT,'(A)') &
     &'   NUM   ROUTINE                              '//&
     &'  SUM(s)  SEND(GB)  RECV(GB)    GB/s'
    DO JNUM=501,1000
       IF((TOTSENDBYTES(JNUM).GT.0.0_JPRD.OR.TOTRECVBYTES(JNUM).GT.0.0_JPRD) &
          .AND. MAXCOMMTIME(JNUM) > 0.0_JPRD) THEN
          WRITE(KULOUT,'(I6,1X,A40,f6.1,2F10.1,F8.1)') &
           &  JNUM,CCDESC(JNUM),MAXCOMMTIME(JNUM),&
           &  TOTSENDBYTES(JNUM)*1.E-9_JPRD, &
           &  TOTRECVBYTES(JNUM)*1.E-9_JPRD , &
           &  (TOTSENDBYTES(JNUM)*1.E-9_JPRD)/MAXCOMMTIME(JNUM)
       ENDIF
    ENDDO
    TOTSENDBYTESSUM = SUM(TOTSENDBYTES)
    TOTRECVBYTESSUM = SUM(TOTRECVBYTES)
    IF( TOTSENDBYTESSUM.GT.0.0_JPRD ) THEN
      WRITE(KULOUT,'(/,A,42x,f6.1,2F10.1,F8.1)') &
       &   'TOTAL', &
       &   SUM(MAXCOMMTIME) , &
       &   TOTSENDBYTESSUM*1.E-9_JPRD, &
       &   TOTRECVBYTESSUM*1.E-9_JPRD, &
       &   (TOTSENDBYTESSUM*1.E-9_JPRD)/SUM(MAXCOMMTIME)
     ENDIF
  ENDIF

  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A,F10.1,A)')'<zcom unit="seconds">',ZT_SUM, '</zcom>'
  ENDIF
  WRITE(KULOUT,*) ''
  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A)')'</timing_communications>'
  ENDIF

ENDIF
IF(LSTATS_OMP)THEN
  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A)')'<timing_parallel>'
  ENDIF
  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(A)') 'STATS FOR PARALLEL REGIONS'
  WRITE(KULOUT,'(A)')  &
  &' NUM ROUTINE                                     CALLS    MEAN(ms)  MAX(ms)   FRAC(%)  UNBAL(%)'
  ZT_SUM=0._JPRD
  DO JNUM=500,JPMAXSTAT
    IF(CCTYPE(JNUM).EQ."OMP".AND.NCALLS(JNUM) > 1) THEN
      ICALLS = NCALLS(JNUM)/2
      ZMEAN = ZAVEAVE(JNUM)/NPROC_STATS
      ZMAX  = ZAVEMAX(JNUM)
      ZMEANT = ZSUMTOT(JNUM)/NPROC_STATS
      ZMAXT  = ZSUMMAX(JNUM)
      IF(ZMEANT .NE. 0._JPRD)THEN
        ZUNBAL= (ZMAXT-ZMEANT)/ZTOTAL*100._JPRD
      ELSE
        ZUNBAL=0._JPRD
      ENDIF
      ZFRAC=ZFRACMAX(JNUM)
      ZTOTUNBAL = ZTOTUNBAL+(ZMAXT-ZMEANT)
      WRITE(KULOUT,'(I4,1X,A40,1X,I8,2(1X,F9.1),2(1X,F9.2))')&
       &JNUM,CCDESC(JNUM),ICALLS,ZMEAN,ZMAX,ZFRAC,ZUNBAL
      IF(LXML_STATS)THEN
        WRITE(IXMLLUN,'(A,I4,A,/,A,A40,A,/,A,I8,A,/,2(A,F9.1,A,/),2(A,F9.2,A,/),A)')&
         &'<parallelitem id="',JNUM,'">',&
         &'<description>',CCDESC(JNUM),'</description>',&
         &'<calls>',ICALLS,'</calls>',&
         &'<mean unit="seconds">',ZMEAN,'</mean>',&
         &'<max unit="seconds">',ZMAX,'</max>',&
         &'<fraction unit="percent">',ZFRAC,'</fraction>',&
         &'<unbalanced unit="percent">',ZUNBAL,'</unbalanced>',&
         &'</parallelitem>'
      ENDIF

      ZT_SUM=ZT_SUM+ZMEANT
    ENDIF
  ENDDO

  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(A,F10.1,A)')'SUMMED TIME IN PARALLEL REGIONS = ',ZT_SUM, ' SECONDS '
  WRITE(KULOUT,*) ''

  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A,F10.1,A)')'<zpar unit="seconds">',ZT_SUM, '</zpar>'
  ENDIF

  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(A)') 'STATS FOR I/O REGIONS'
  WRITE(KULOUT,'(A)')  &
  &' NUM ROUTINE                                     CALLS    MEAN(ms)  MAX(ms)   FRAC(%)  UNBAL(%)'
  ZT_SUM=0._JPRD
  DO JNUM=500,JPMAXSTAT
    IF(CCTYPE(JNUM).EQ."IO-".AND.NCALLS(JNUM) > 1) THEN
      ICALLS = NCALLS(JNUM)/2
      ZMEAN = ZAVEAVE(JNUM)/NPROC_STATS
      ZMAX  = ZAVEMAX(JNUM)
      ZMEANT = ZSUMTOT(JNUM)/NPROC_STATS
      ZMAXT  = ZSUMMAX(JNUM)
      IF(ZMEANT .NE. 0._JPRD)THEN
        ZUNBAL= (ZMAXT-ZMEANT)/ZTOTAL*100._JPRD
      ELSE
        ZUNBAL=0._JPRD
      ENDIF
      ZFRAC=ZFRACMAX(JNUM)
      ZTOTUNBAL = ZTOTUNBAL+(ZMAXT-ZMEANT)
      WRITE(KULOUT,'(I4,1X,A40,1X,I8,2(1X,F9.1),2(1X,F9.2))')&
       &JNUM,CCDESC(JNUM),ICALLS,ZMEAN,ZMAX,ZFRAC,ZUNBAL
      IF(LXML_STATS)THEN
        WRITE(IXMLLUN,'(A,I4,A,/,A,A40,A,/,A,I8,A,/,2(A,F9.1,A,/),2(A,F9.2,A,/),A)')&
         &'<para_io_item id="',JNUM,'">',&
         &'<description>',CCDESC(JNUM),'</description>',&
         &'<calls>',ICALLS,'</calls>',&
         &'<mean unit="seconds">',ZMEAN,'</mean>','<max unit="seconds">',&
         & ZMAX,'</max>',&
         &'<fraction unit="percent">',ZFRAC,'</fraction>',&
         &'<unbalanced unit="percent">',ZUNBAL,'</unbalanced>',&
         &'</para_io_item>'
      ENDIF

      ZT_SUM=ZT_SUM+ZMEANT
    ENDIF
  ENDDO

  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(A,F10.1,A)')'SUMMED TIME IN I/O REGIONS = ',&
       &ZT_SUM, ' SECONDS '
  WRITE(KULOUT,*) ''

  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A,F10.1,A)')'<zio unit="seconds">',ZT_SUM,'</zio>'
  ENDIF

  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(A)') 'STATS FOR SERIAL(no OMP) REGIONS'
  WRITE(KULOUT,'(A)')  &
  &' NUM ROUTINE                                     CALLS    MEAN(ms)  MAX(ms)   FRAC(%)  UNBAL(%)'
  ZT_SUM=0._JPRD
  DO JNUM=500,JPMAXSTAT
    IF(CCTYPE(JNUM).EQ."SER".AND.NCALLS(JNUM) > 1) THEN
      ICALLS = NCALLS(JNUM)/2
      ZMEAN = ZAVEAVE(JNUM)/NPROC_STATS
      ZMAX  = ZAVEMAX(JNUM)
      ZMEANT = ZSUMTOT(JNUM)/NPROC_STATS
      ZMAXT  = ZSUMMAX(JNUM)
      IF(ZMEANT .NE. 0._JPRD)THEN
        ZUNBAL= (ZMAXT-ZMEANT)/ZTOTAL*100._JPRD
      ELSE
        ZUNBAL=0._JPRD
      ENDIF
      ZFRAC=ZFRACMAX(JNUM)
      ZTOTUNBAL = ZTOTUNBAL+(ZMAXT-ZMEANT)
      WRITE(KULOUT,'(I4,1X,A40,1X,I8,2(1X,F9.1),2(1X,F9.2))')&
       &JNUM,CCDESC(JNUM),ICALLS,ZMEAN,ZMAX,ZFRAC,ZUNBAL
      IF(LXML_STATS)THEN
        WRITE(IXMLLUN,'(A,I4,A,A,A40,A,A,I8,A,2(A,F9.1,A),2(A,F9.2,A,/),A)')&
         &'<serialitem id="',JNUM,'">',&
         &'<description>',CCDESC(JNUM),'</description>',&
         &'<calls>',ICALLS,'</calls>',&
         &'<mean unit="ms">',ZMEAN,'</mean>','<max unit="ms">',ZMAX,'</max>',&
         &'<fraction unit="percent">',ZFRAC,'</fraction>',&
         &'<unbalanced unit="percent">',ZUNBAL,'</unbalanced>','</serialitem>'
      ENDIF

      ZT_SUM=ZT_SUM+ZMEANT
    ENDIF
  ENDDO

  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(A,F10.1,A)')'SUMMED TIME IN SERIAL REGIONS = ',ZT_SUM, ' SECONDS '
  WRITE(KULOUT,*) ''

  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A,F10.1,A)')'<zserial unit="seconds">',&
     &ZT_SUM, '</zserial>'
  ENDIF

  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(A)') 'STATS FOR MIXED SECTIONS'
  WRITE(KULOUT,'(A)')  &
  &' NUM ROUTINE                                     CALLS    MEAN(ms)  MAX(ms)   FRAC(%)  UNBAL(%)'
  ZT_SUM=0._JPRD
  DO JNUM=500,JPMAXSTAT
    IF(CCTYPE(JNUM).EQ."MXD".AND.NCALLS(JNUM) > 1) THEN
      ICALLS = NCALLS(JNUM)/2
      ZMEAN = ZAVEAVE(JNUM)/NPROC_STATS
      ZMAX  = ZAVEMAX(JNUM)
      ZMEANT = ZSUMTOT(JNUM)/NPROC_STATS
      ZMAXT  = ZSUMMAX(JNUM)
      IF(ZMEANT .NE. 0._JPRD)THEN
        ZUNBAL= (ZMAXT-ZMEANT)/ZTOTAL*100._JPRD
      ELSE
        ZUNBAL=0._JPRD
      ENDIF
      ZFRAC=ZFRACMAX(JNUM)
      ZTOTUNBAL = ZTOTUNBAL+(ZMAXT-ZMEANT)
      WRITE(KULOUT,'(I4,1X,A40,1X,I8,2(1X,F9.1),2(1X,F9.2))')&
       &JNUM,CCDESC(JNUM),ICALLS,ZMEAN,ZMAX,ZFRAC,ZUNBAL
      IF(LXML_STATS)THEN
        WRITE(IXMLLUN,'(A,I4,A,A,A40,A,A,I8,A,2(A,F9.1,A),2(A,F9.2,A,/),A)')&
         &'<mixeditem id="',JNUM,'">',&
         &'<description>',CCDESC(JNUM),'</description>',&
         &'<calls>',ICALLS,'</calls>',&
         &'<mean unit="ms">',ZMEAN,'</mean>','<max unit="ms">',ZMAX,'</max>',&
         &'<fraction unit="percent">',ZFRAC,'</fraction>',&
         &'<unbalanced unit="percent">',ZUNBAL,'</unbalanced>','</mixeditem>'
      ENDIF

      ZT_SUM=ZT_SUM+ZMEANT
    ENDIF
  ENDDO

  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(A,F10.1,A)')'SUMMED TIME IN MIXED SECTIONS = ',ZT_SUM, ' SECONDS '
  WRITE(KULOUT,*) ''

  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A,F10.1,A)')'<zmixed unit="seconds">',&
     &ZT_SUM, '</zmixed>'
  ENDIF

  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A)')'</timing_parallel>'
  ENDIF

ENDIF

  WRITE(KULOUT,'(A,F10.1,A,F4.1,A)')&
   &'TOTAL MEASURED IMBALANCE =',ZTOTUNBAL,&
   &' SECONDS, ',ZTOTUNBAL/ZTOTAL*100._JPRD,' PERCENT'
ELSE
  ZTOTAL=TIMESUM(0)
  ZTOTCPU = TTCPUSUM(0)
  ZTOTVCPU = TVCPUSUM(0)
ENDIF

IF ( MYPROC_STATS == 1) THEN
  WRITE(KULOUT,'(3(A,F11.3)/)')'TOTAL WALLCLOCK TIME ',ZTOTAL,&
   &' CPU TIME',ZTOTCPU,' VECTOR TIME ',ZTOTVCPU
  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(3(A,F11.3,A,/)/)')'<totalwallclocktime>',ZTOTAL,&
     &'</totalwallclocktime>',&
     &'<cputime>',ZTOTCPU,'</cputime>',&
     & '<vectortime>',ZTOTVCPU,'</vectortime>'
  ENDIF
ENDIF
IF( LDETAILED_STATS )THEN
  ITAG = JPTAGSTAT
  ZDELAY_MAX=0.0_JPRD
  ZMPL(:)=0.0_JPRD
  ZBAR(:)=0.0_JPRD
  ZGBR(:)=0.0_JPRD
  ZGB2(:)=0.0_JPRD
  ZOMP(:)=0.0_JPRD
  ZIO (:)=0.0_JPRD
  ZSER(:)=0.0_JPRD
  ZMXD(:)=0.0_JPRD
  DO JROC=1,NPROC_STATS
    IF( JROC > 1 )THEN
      IF( MYPROC_STATS == JROC )THEN
        ISEND=1
        CALL MPL_SEND(NDELAY_INDEX,KDEST=NPRCIDS_STATS(ISEND), &
         & KTAG=ITAG,CDSTRING='GSTATS_PRINT:')
        IF( NDELAY_INDEX > 0 )THEN
          CALL MPL_SEND(NDELAY_COUNTER(1:NDELAY_INDEX),KDEST=NPRCIDS_STATS(ISEND), &
           & KTAG=ITAG+1,CDSTRING='GSTATS_PRINT:')
          CALL MPL_SEND(TDELAY_VALUE(1:NDELAY_INDEX),KDEST=NPRCIDS_STATS(ISEND), &
           & KTAG=ITAG+2,CDSTRING='GSTATS_PRINT:')
          DO JDELAY=1,NDELAY_INDEX
            CLTEMP((JDELAY-1)*10+1:JDELAY*10)=CDELAY_TIME(JDELAY)
          ENDDO
          CALL MPL_SEND(CLTEMP(1:NDELAY_INDEX*10),KDEST=NPRCIDS_STATS(ISEND), &
           & KTAG=ITAG+3,CDSTRING='GSTATS_PRINT:')
        ENDIF
      ENDIF
      IF( MYPROC_STATS == 1 )THEN
        CALL MPL_RECV(NDELAY_INDEX,KSOURCE=NPRCIDS_STATS(JROC), &
         & KTAG=ITAG,CDSTRING='GSTATS_PRINT:')
        IF( NDELAY_INDEX > 0 )THEN
          CALL MPL_RECV(NDELAY_COUNTER(1:NDELAY_INDEX),KSOURCE=NPRCIDS_STATS(JROC), &
           & KTAG=ITAG+1,CDSTRING='GSTATS_PRINT:')
          CALL MPL_RECV(TDELAY_VALUE(1:NDELAY_INDEX),KSOURCE=NPRCIDS_STATS(JROC), &
           & KTAG=ITAG+2,CDSTRING='GSTATS_PRINT:')
          CALL MPL_RECV(CLTEMP(1:NDELAY_INDEX*10),KSOURCE=NPRCIDS_STATS(JROC), &
            & KTAG=ITAG+3,CDSTRING='GSTATS_PRINT:')
          DO JDELAY=1,NDELAY_INDEX
            CDELAY_TIME(JDELAY)=CLTEMP((JDELAY-1)*10+1:JDELAY*10)
          ENDDO
        ENDIF
      ENDIF
    ENDIF
    IF( MYPROC_STATS == 1 .AND. NDELAY_INDEX > 0 )THEN
      WRITE(KULOUT,'("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")')
      WRITE(KULOUT,'("TASK=",I6," NUMBER OF UNEXPECTED DELAYS=",I4)') JROC,NDELAY_INDEX
      IF( NDELAY_INDEX == JPMAXDELAYS )THEN
        WRITE(KULOUT,'(" NOTE THAT THE MAXIMUM NUMBER OF DELAYS HAS BEEN REACHED =",I6)')JPMAXDELAYS
      ENDIF
      ZDELAY=SUM(TDELAY_VALUE(1:NDELAY_INDEX))
      WRITE(KULOUT,'("TOTAL UNEXPECTED DELAY TIME (SECS) =",F9.1)') ZDELAY
      ZDELAY_MAX=MAX(ZDELAY_MAX,ZDELAY)
      DO JDELAY=1,NDELAY_INDEX
        WRITE(KULOUT,'(A,":",A,":",A,1X,I4,1X,A3,1X,A40,1X,F6.1)')&
         &CDELAY_TIME(JDELAY)(1:2),CDELAY_TIME(JDELAY)(3:4),CDELAY_TIME(JDELAY)(5:6),&
         &NDELAY_COUNTER(JDELAY),CCTYPE(NDELAY_COUNTER(JDELAY)),&
         &CCDESC(NDELAY_COUNTER(JDELAY)),TDELAY_VALUE(JDELAY)
         IF( CCTYPE(NDELAY_COUNTER(JDELAY)) .EQ. 'MPL' ) ZMPL(JROC)=ZMPL(JROC)+TDELAY_VALUE(JDELAY)
         IF( CCTYPE(NDELAY_COUNTER(JDELAY)) .EQ. 'BAR' ) ZBAR(JROC)=ZBAR(JROC)+TDELAY_VALUE(JDELAY)
         IF( CCTYPE(NDELAY_COUNTER(JDELAY)) .EQ. 'GBR' ) ZGBR(JROC)=ZGBR(JROC)+TDELAY_VALUE(JDELAY)
         IF( CCTYPE(NDELAY_COUNTER(JDELAY)) .EQ. 'GB2' ) ZGB2(JROC)=ZGB2(JROC)+TDELAY_VALUE(JDELAY)
         IF( CCTYPE(NDELAY_COUNTER(JDELAY)) .EQ. 'OMP' ) ZOMP(JROC)=ZOMP(JROC)+TDELAY_VALUE(JDELAY)
         IF( CCTYPE(NDELAY_COUNTER(JDELAY)) .EQ. 'IO-' ) ZIO (JROC)=ZIO (JROC)+TDELAY_VALUE(JDELAY)
         IF( CCTYPE(NDELAY_COUNTER(JDELAY)) .EQ. 'SER' ) ZSER(JROC)=ZSER(JROC)+TDELAY_VALUE(JDELAY)
         IF( CCTYPE(NDELAY_COUNTER(JDELAY)) .EQ. 'MXD' ) ZMXD(JROC)=ZMXD(JROC)+TDELAY_VALUE(JDELAY)
      ENDDO
      WRITE(KULOUT,'(" ")')
      WRITE(KULOUT,'("TASK=",I6," UNEXPECTED DELAYS SORTED BY COUNTER")') JROC
      DO JNUM=500,JPMAXSTAT
        IDELAY=0
        ZDELAY=0.0_JPRD
        DO JDELAY=1,NDELAY_INDEX
          IF( NDELAY_COUNTER(JDELAY) == JNUM )THEN
            IDELAY=IDELAY+1
            ZDELAY=ZDELAY+TDELAY_VALUE(JDELAY)
          ENDIF
        ENDDO
        IF( IDELAY /= 0 )THEN
          WRITE(KULOUT,'(I4,1X,A3,1X,A40,1X,I4,3X,F6.1)')&
           &JNUM,CCTYPE(JNUM),CCDESC(JNUM),IDELAY,ZDELAY
        ENDIF
      ENDDO
      WRITE(KULOUT,'(" ")')
      WRITE(KULOUT,'(" ")')
    ENDIF
    IF (NPROC_STATS > 1) THEN
      CALL MPL_BARRIER(CDSTRING='GSTATS_PRINT')
    ENDIF
  ENDDO
  IF( MYPROC_STATS == 1 )THEN
    WRITE(KULOUT,'("MAXIMUM TOTAL UNEXPECTED DELAY TIME (SECS) =",F9.1)') ZDELAY_MAX
    WRITE(KULOUT,'(" ")')
    WRITE(KULOUT,'(" ")')
    WRITE(KULOUT,'("  TASK   ","     MPL   ","     BAR   ","     GBR   ","     GB2   ","     OMP   ",&
     &"     IO-   ","     SER   ","     MXD   ")')
    DO JROC=1,NPROC_STATS
      WRITE(KULOUT,'(I6,8(2X,F9.1))') JROC,ZMPL(JROC),ZBAR(JROC),ZGBR(JROC),ZGB2(JROC),&
      &ZOMP(JROC),ZIO (JROC),ZSER(JROC),ZMXD(JROC)
    ENDDO
    WRITE(KULOUT,'(" ")')
    WRITE(KULOUT,'(" ")')
  ENDIF
ENDIF

!   Trace stats

IF (LTRACE_STATS) THEN
  WRITE(KULOUT,'(A)') '=== TRACE OF CALLS TO GSTATS'
  IF (NCALLS_TOTAL > NTRACE_STATS) THEN
    WRITE(KULOUT,'(A,2I10)') ' ONLY PART OF TRACE STORED AS BUFFER TO SMALL ',&
     & NCALLS_TOTAL,NTRACE_STATS
  ENDIF
  WRITE(KULOUT,'(A)') '==='
  CLACTION(0)='ON'
  CLACTION(1)='OFF'
  CLACTION(2)='SUSPEND'
  CLACTION(3)='RESUME'
  DO JCALL=1,MIN(NCALLS_TOTAL,NTRACE_STATS)
    ICALLER = MOD(NCALL_TRACE(JCALL),(JPMAXSTAT+1))
    IACTION   = NCALL_TRACE(JCALL)/(JPMAXSTAT+1)
    IF (IACTION == 0) THEN
      ZTIMELCALL(ICALLER) = TIME_TRACE(JCALL)
      ZTHISTIME(ICALLER) = 0.0_JPRD
    ELSEIF (IACTION == 2) THEN
      ZTHISTIME(ICALLER) = TIME_TRACE(JCALL)-ZTIMELCALL(ICALLER)
    ELSEIF (IACTION == 3) THEN
      ZTIMELCALL(ICALLER) = TIME_TRACE(JCALL)
    ENDIF
    IF (IACTION == 1) THEN
      WRITE(KULOUT,'(1X,F10.3,1X,A,1X,A,1X,F10.3)') &
       &TIME_TRACE(JCALL),CCDESC(ICALLER),CLACTION(IACTION),&
       &ZTHISTIME(ICALLER)+(TIME_TRACE(JCALL)-ZTIMELCALL(ICALLER))
    ELSE
      WRITE(KULOUT,'(1X,F10.3,1X,A,1X,A)') TIME_TRACE(JCALL),CCDESC(ICALLER),&
       & CLACTION(IACTION)
    ENDIF
  ENDDO
ENDIF

IF(LSTATS .AND. MYPROC_STATS == 1) THEN
  PAVEAVE(0:KLEN) = ZAVEAVE(0:KLEN)
ELSE
  PAVEAVE(0:KLEN) = 0.0_JPRD
ENDIF

WRITE(KULOUT,'(/A)')'===-=== END   OF TIMING STATISTICS ===-==='


IF(LSTATS_MEM)THEN
  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A)')'<memory>'
  ENDIF
  WRITE(KULOUT,*) ''
  WRITE(KULOUT,*) 'STATS FOR MEMORY'
  WRITE(KULOUT,*)  &
 &' NUM ROUTINE               CALLS  CALL    MAXINCR   TOTINCR   MININCR'
  WRITE(KULOUT,*)  &
 &'                                   NO      (KB)      (KB)      (KB)'
  DO JNUM=0,JPMAXSTAT
    IF(NCALLS(JNUM) > 1) THEN
      ICALLS = NCALLS(JNUM)/2
      IMEM=NTMEM(JNUM,1)
      INUM=NTMEM(JNUM,3)/2
      JMEM=NTMEM(JNUM,4)
      WRITE(KULOUT,'(I4,1X,A20,1X,I8,1X,I6,3(1X,I9))')&
       &JNUM,CCDESC(JNUM),ICALLS,INUM,IMEM,JMEM,NTMEM(JNUM,5)

      IF(LXML_STATS)THEN
        WRITE(IXMLLUN,'(A,I4,A,/,A,A20,A,/,A,I8,A,/,A,I6,A,/,3(A,I9,A,/))')&
         &'<memitem id="',JNUM,'"/>',&
         &'<description>',CCDESC(JNUM),'</description>',&
         &'<calls>',ICALLS,'</calls>',&
         &'<callnum>',INUM,'</callnum>','<maxincr unit="kb">',IMEM,'</maxincr>',&
         &'<totincr unit="kb">',JMEM,'</totincr>',&
         &'<minincr unit="kb">',NTMEM(JNUM,5),'</minincr>'
      ENDIF
    ENDIF
  ENDDO

  WRITE(KULOUT,*) ''
  WRITE(KULOUT,'(/A)')'===-=== END   OF MEMORY STATISTICS ===-==='
  WRITE(KULOUT,*) ''
  IF(LXML_STATS)THEN
    WRITE(IXMLLUN,'(A)')'</memory>'
  ENDIF
ENDIF
IF(LXML_STATS .AND. MYPROC_STATS==1)THEN
  WRITE(IXMLLUN,'(A)')'</gstats>'
  CLOSE(IXMLLUN)
ENDIF
RETURN
END SUBROUTINE GSTATS_PRINT
