#LyX 2.1 created this file. For more info see http://www.lyx.org/
\lyxformat 474
\begin_document
\begin_header
\textclass extarticle
\begin_preamble
\usepackage{minted}
\definecolor{bg}{rgb}{0.95,0.95,0.95}
\end_preamble
\use_default_options true
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman default
\font_sans default
\font_typewriter default
\font_math auto
\font_default_family rmdefault
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100
\font_tt_scale 100
\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command default
\paperfontsize 11
\spacing single
\use_hyperref true
\pdf_bookmarks true
\pdf_bookmarksnumbered false
\pdf_bookmarksopen false
\pdf_bookmarksopenlevel 1
\pdf_breaklinks false
\pdf_pdfborder false
\pdf_colorlinks false
\pdf_backref false
\pdf_pdfusetitle true
\papersize default
\use_geometry true
\use_package amsmath 1
\use_package amssymb 1
\use_package cancel 1
\use_package esint 1
\use_package mathdots 0
\use_package mathtools 1
\use_package mhchem 1
\use_package stackrel 1
\use_package stmaryrd 1
\use_package undertilde 1
\cite_engine basic
\cite_engine_type default
\biblio_style plain
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date false
\justification true
\use_refstyle 0
\index Index
\shortcut idx
\color #008000
\end_index
\leftmargin 1cm
\topmargin 2cm
\rightmargin 12mm
\bottommargin 2cm
\secnumdepth 3
\tocdepth 3
\paragraph_separation indent
\paragraph_indentation default
\quotes_language english
\papercolumns 1
\papersides 1
\paperpagestyle default
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Title
Parallel and Large-scale Simulation Enhancements to CGNS
\end_layout

\begin_layout Author
M.
 Scot Breitenfeld, The HDF Group, brtnfld@hdfgroup.org
\end_layout

\begin_layout Section
Overview of changes introduced in the HDF5_Parallel branch
\end_layout

\begin_layout Standard
Many of the changes discussed in the following sections address the currently
 (as of version 3.2.1) underperforming parallel capabilities of the CGNS library.
 For example, the CGNS function cgp_open, which opens a CGNS file for processing
, has substantially increasing execution time as the number of processes
 is increased, Fig.
 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:Time-for-completion"

\end_inset

 (trunk).
 The current improvement for cgp_open is substantial at 100-1000 times faster
 (branch) than the previous implementation (trunk).
 In fact, for runs with the largest number of processes (>1024) the batch
 job had a time limit of 5 minutes and not all the processes had completed
 cgp_open before this limit was reached.
 Obviously, the previous implementation of cgp_open is a lot worse than
 reported in the figure.
\end_layout

\begin_layout Standard
\begin_inset Float figure
placement h
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename figs/cgp_open.png
	scale 30

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:Time-for-completion"

\end_inset

Time for completion of cgp_open in write mode for the original implementation
 (trunk) and the current implementation (branch), the error bars correspond
 to the minimum and maximum time over all the processes that had completed
 before the batch job time limit was reached.
 
\end_layout

\end_inset


\end_layout

\begin_layout Plain Layout

\end_layout

\end_inset


\end_layout

\begin_layout Standard
Section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:General-behavior-changes"

\end_inset

 lists changes to CGNS that could effect the end user and introduces new
 functions and specifications.
 Fortran programmers should take notice of Section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:New-Fortran-changes"

\end_inset

 which highlights changes introduced for better interoperability with the
 C CGNS library.
 Section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Parallel-installation-instructio"

\end_inset

 gives example installation guides for GPFS and Lustre hardware.
 Items listed in blue affect compatibility of older codes when using CGNS
 v3.2.2.
 Known problems are highlighted in red.
 The branch can be checked out from sourceforge at,
\end_layout

\begin_layout Standard
\begin_inset space \space{}
\end_inset


\end_layout

\begin_layout Standard
\align center
http://svn.code.sf.net/p/cgns/code/cgns/branches/HDF5_Parallel
\end_layout

\begin_layout Section
\begin_inset CommandInset label
LatexCommand label
name "sub:General-behavior-changes"

\end_inset

General behavior changes and new recommendations for parallel performance
\end_layout

\begin_layout Itemize
The flush functions should not be used.
 Writing and reading immediately avoids IO contention occurring when flush
 is being used.
\end_layout

\begin_layout Itemize
The parallel routines are meant for parallel file systems (GPFS or Lustre).
\end_layout

\begin_layout Itemize

\color blue
The default parallel input/output mode was changed from CGP_INDEPENDENT
 to CGP_COLLECTIVE.
\end_layout

\begin_layout Itemize

\color blue
A new function was added for passing MPI info to the CGNS library.
 
\end_layout

\begin_layout Standard
\align center

\series bold
C
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{c}     
\end_layout

\begin_layout Plain Layout

    int cgp_mpi_info(MPI_Info info)
\end_layout

\begin_layout Plain Layout


\backslash
end{minted} 
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\align center

\series bold
Fortran
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{fortran}     
\end_layout

\begin_layout Plain Layout

     CALL cgp_mpi_info_f(comm_info, ierr)
\end_layout

\begin_layout Plain Layout

        INTEGER :: comm_info
\end_layout

\begin_layout Plain Layout

        INTEGER :: ierr
\end_layout

\begin_layout Plain Layout


\backslash
end{minted} 
\end_layout

\end_inset


\end_layout

\begin_layout Itemize
Functions for parallel reading and writing multi-component datasets using
 a single call was introduced.
 The new APIs use new capabilities are tentatively to be introduced in version
 1.10.0 of the HDF5 library.
 The new APIs pack multiple datasets into a single buffer and the underlying
 MPI IO completes the IO request using just one call.
 The availability of the new functions in the HDF5 library is checked at
 compile time.
 The current limitation (due to MPI) is that the size of the sum of the
 datasets must be less than 2GB.
 Example usage can be found in benchmark_hdf5.c and benchmark_hdf5_f90.F90
 in ptests.
\end_layout

\begin_layout Standard
\align center

\series bold
C
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{c}
\end_layout

\begin_layout Plain Layout

int cgp_coord_multi_read_data(int fn, int B, int Z, int *C, 
\end_layout

\begin_layout Plain Layout

                              const cgsize_t *rmin, const cgsize_t *rmax,
\end_layout

\begin_layout Plain Layout

                              void *coordsX,  void *coordsY,  void *coordsZ);
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{c}  
\end_layout

\begin_layout Plain Layout

int cgp_coord_multi_write_data(int fn, int B, int Z, int *C, 
\end_layout

\begin_layout Plain Layout

                               const cgsize_t *rmin, const cgsize_t *rmax,
\end_layout

\begin_layout Plain Layout

                               const void *coordsX, const void *coordsY,
 const void *coordsZ);
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{c}
\end_layout

\begin_layout Plain Layout

 int cgp_field_multi_read_data(int fn, int B, int Z, int S, int *F,
\end_layout

\begin_layout Plain Layout

                               const cgsize_t *rmin, const cgsize_t *rmax,
\end_layout

\begin_layout Plain Layout

                               int nsets, ...);
\end_layout

\begin_layout Plain Layout

/* ...
 nsets of variable arguments, *solution_array, corresponding to the order
 given by F */
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{c}
\end_layout

\begin_layout Plain Layout

int cgp_field_multi_write_data(int fn, int B, int Z, int S, int *F,
\end_layout

\begin_layout Plain Layout

                               const cgsize_t *rmin, const cgsize_t *rmax,
\end_layout

\begin_layout Plain Layout

                               int nsets, ...);
\end_layout

\begin_layout Plain Layout

/* ...
 nsets of variable arguments, *solution_array, corresponding to the order
 given by F */
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{c}
\end_layout

\begin_layout Plain Layout

int cgp_array_multi_write_data(int fn, int *A, const cgsize_t *rmin, const
 cgsize_t *rmax,
\end_layout

\begin_layout Plain Layout

                               int nsets, ...);
\end_layout

\begin_layout Plain Layout

/* ...
 nsets of variable arguments, *field_array, corresponding to the order given
 by F */
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{c}
\end_layout

\begin_layout Plain Layout

int cgp_array_multi_read_data(int fn, int *A, const cgsize_t *rmin,const
 cgsize_t *rmax,
\end_layout

\begin_layout Plain Layout

                              int nsets, ...);
\end_layout

\begin_layout Plain Layout

/* ...
 nsets of variable arguments, *field_array, corresponding to the order given
 by F */
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\align center

\series bold
Fortran
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{fortran}
\end_layout

\begin_layout Plain Layout

CALL cgp_coord_multi_read_data_f(fn, B, Z, C, rmin, rmax, coordsX, coordsY,
 coordsZ, ier)
\end_layout

\begin_layout Plain Layout

   INTEGER :: fn
\end_layout

\begin_layout Plain Layout

   INTEGER :: B
\end_layout

\begin_layout Plain Layout

   INTEGER :: Z
\end_layout

\begin_layout Plain Layout

   INTEGER :: C
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmin
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmax
\end_layout

\begin_layout Plain Layout

   REAL :: coordsX, coordsY, coordsZ
\end_layout

\begin_layout Plain Layout

   INTEGER :: ier
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{fortran}
\end_layout

\begin_layout Plain Layout

CALL cgp_coord_multi_write_data_f(fn, B, Z, C, rmin, rmax, coordsX, coordsY,
 coordsZ, ier)
\end_layout

\begin_layout Plain Layout

   INTEGER :: fn
\end_layout

\begin_layout Plain Layout

   INTEGER :: B
\end_layout

\begin_layout Plain Layout

   INTEGER :: Z
\end_layout

\begin_layout Plain Layout

   INTEGER :: C
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmin
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmax
\end_layout

\begin_layout Plain Layout

   REAL :: coordsX, coordsY, coordsZ
\end_layout

\begin_layout Plain Layout

   INTEGER :: ier
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{fortran}
\end_layout

\begin_layout Plain Layout

CALL cgp_field_multi_write_data_f(fn, B, Z, S, F, rmin, rmax, ier, nsets,
 ...)
\end_layout

\begin_layout Plain Layout

   INTEGER :: fn
\end_layout

\begin_layout Plain Layout

   INTEGER :: B
\end_layout

\begin_layout Plain Layout

   INTEGER :: Z
\end_layout

\begin_layout Plain Layout

   INTEGER :: C
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmin
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmax
\end_layout

\begin_layout Plain Layout

   INTEGER :: ier
\end_layout

\begin_layout Plain Layout

   INTEGER :: nsets
\end_layout

\begin_layout Plain Layout

   ...
 REAL, DIMENSION(*) :: field_array ! entered nsets times 
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{fortran}
\end_layout

\begin_layout Plain Layout

CALL cgp_field_multi_read_data_f(fn, B, Z, S, F, rmin, rmax, ier, nsets,
 ...)
\end_layout

\begin_layout Plain Layout

   INTEGER :: fn
\end_layout

\begin_layout Plain Layout

   INTEGER :: B
\end_layout

\begin_layout Plain Layout

   INTEGER :: Z
\end_layout

\begin_layout Plain Layout

   INTEGER :: C
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmin
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmax
\end_layout

\begin_layout Plain Layout

   INTEGER :: ier
\end_layout

\begin_layout Plain Layout

   INTEGER :: nsets
\end_layout

\begin_layout Plain Layout

   ...
 REAL, DIMENSION(*) :: field_array ! entered nsets times 
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{fortran}
\end_layout

\begin_layout Plain Layout

CALL cgp_array_multi_write_data_f(fn, B, Z, S, F, rmin, rmax, ier, nsets,
 ...)
\end_layout

\begin_layout Plain Layout

   INTEGER :: fn
\end_layout

\begin_layout Plain Layout

   INTEGER :: B
\end_layout

\begin_layout Plain Layout

   INTEGER :: Z
\end_layout

\begin_layout Plain Layout

   INTEGER :: C
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmin
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmax
\end_layout

\begin_layout Plain Layout

   INTEGER :: ier
\end_layout

\begin_layout Plain Layout

   INTEGER :: nsets
\end_layout

\begin_layout Plain Layout

   ...
 REAL, DIMENSION(*) :: data_array ! entered nsets times 
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{fortran}
\end_layout

\begin_layout Plain Layout

CALL cgp_array_multi_read_data_f(fn, B, Z, S, F, rmin, rmax, ier, nsets,
 ...)
\end_layout

\begin_layout Plain Layout

   INTEGER :: fn
\end_layout

\begin_layout Plain Layout

   INTEGER :: B
\end_layout

\begin_layout Plain Layout

   INTEGER :: Z
\end_layout

\begin_layout Plain Layout

   INTEGER :: C
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmin
\end_layout

\begin_layout Plain Layout

   INTEGER(CG_SIZE_T) :: rmax
\end_layout

\begin_layout Plain Layout

   INTEGER :: ier
\end_layout

\begin_layout Plain Layout

   INTEGER :: nsets
\end_layout

\begin_layout Plain Layout

   ...
 REAL, DIMENSION(*) :: data_array ! entered nsets times 
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
New C changes
\end_layout

\begin_layout Itemize
A new parallel example benchmark program, benchmark_hdf5.c, was added to
 directory ptests.
\end_layout

\begin_layout Itemize
The cgp_*_read/write_dataset APIs now excepts non-allocated arrays, or NULL,
 as valid parameters for the datasets.
 Additionally, the dimensional arrays
\shape italic
, rmin
\shape default
 and 
\shape italic
rmax
\shape default
, can also be NULL.
 If the data array is NULL and the dimensional arrays are not NULL, then
 the validity of the dimensional arrays, 
\shape italic
rmin
\shape default
 and 
\shape italic
rmax
\shape default
, is not check.
 For collective parallel IO, this is used as a mechanism to indicated that
 processes with NULL API parameters will not write any data to the file.
 
\end_layout

\begin_layout Subsubsection
Internal library changes
\end_layout

\begin_layout Enumerate
Fixed issue with autotools putting a blank 
\begin_inset Quotes eld
\end_inset

-l
\begin_inset Quotes erd
\end_inset

 in 
\begin_inset Quotes eld
\end_inset

MPILIBS =
\begin_inset Quotes erd
\end_inset

 when compiling library using using mpi.
\end_layout

\begin_layout Enumerate
Replaced the hid_t to double (and vice-versa) utilities 
\shape italic
to_HDF_ID
\shape default
 and 
\shape italic
to_ADF_ID
\shape default
 from a type cast to a function which uses memcpy for the conversion.
 This is need to for the upcomming release of HDF5 1.10 where hid_t was changed
 from a 32 bit integer to a 64 bit integer.
\end_layout

\begin_layout Subsection
\begin_inset CommandInset label
LatexCommand label
name "sub:New-Fortran-changes"

\end_inset

New Fortran changes
\end_layout

\begin_layout Standard
All users are 
\series bold
\color black
strongly
\series default
\color inherit
 encouraged to use a Fortran 2003 standard compliant compiler.
 Using a Fortran 2003 compiler guarantees interoperability with the C APIs
 via the ISO_C_BINDING module.
 Many changes were added to the CGNS library in order to take full advantage
 of the interoperability offered by the ISO_C_BINDING module.
 
\end_layout

\begin_layout Enumerate
Configure was changed to check if the Fortran compiler is Fortran 2003 compliant.
 If it is then the features of ISO_C_BINDING will be used.
 
\end_layout

\begin_layout Enumerate
The predefined CGNS constant parameters data types were changed from INTEGER
 to ENUM, BIND(C) for better C interoperability.
 The users should use the predefined constants whenever possible and not
 the numerical value represented by the constants.
 
\color blue
A variable expecting an enum value returned from a Fortran API should be
 declared, INTEGER(cgenum_t).
\end_layout

\begin_layout Enumerate

\shape italic
\color blue
INCLUDE 
\begin_inset Quotes eld
\end_inset

cgslib_h
\begin_inset Quotes erd
\end_inset


\shape default
 was changed in favor of using a module, USE CGNS.
\end_layout

\begin_deeper
\begin_layout Enumerate
This allows defining a KIND type for integers instead of the current way
 of using the preprocessor dependent 
\shape italic
cgsize_t
\shape default
.
\end_layout

\end_deeper
\begin_layout Enumerate
\begin_inset CommandInset label
LatexCommand label
name "enu:int64"

\end_inset

The user should be sure to declare the arguments declared 
\shape italic
int
\shape default
 in the C APIs as INTEGER in Fortran.
 The ONLY Fortran arguments declared as type 
\shape italic
cgsize_t
\shape default
 should be the arguments which are also declared
\shape italic
 cgsize_t
\shape default
 in the C APIs.
 This is very important when building with option 
\shape italic
--enable-64bit
\shape default
.
 The test programs were updated in order to conform to this convention.
\end_layout

\begin_layout Enumerate
Assuming the rules in step 
\begin_inset CommandInset ref
LatexCommand ref
reference "enu:int64"

\end_inset

 were followed, users should not need to use parameter CG_BUILD_64BIT since
 Fortran's 
\shape italic
cgsize_t
\shape default
 is now guaranteed to match C's 
\shape italic
cgsize_t
\shape default
.
 
\end_layout

\begin_layout Enumerate

\color red
Fortran programs defining CGNS data types with a default INTEGER size of
 8 bytes also then need to compile the CGNS library with the default INTEGER
 size of 8 bytes
\color inherit
.
 This is independent of whether or not 
\shape italic
--enable-64bit
\shape default
 is being used.
 For clarification, using 
\shape italic
--enable-64bit
\shape default
 allows for data types (i.e.
 those declared as 
\shape italic
cgsize_t
\shape default
) to be able to store values which are too large to be stored as 4 byte
 integers (i.e.
 numbers greater than 2,147,483,647).
 It is not necessary, or advisable (since it waste memory), to have CGNS
 INTEGER types (types declared 
\shape italic
int
\shape default
 in C) to be 8 bytes; the variables declared as 
\shape italic
cgsize_t
\shape default
 will automatically handle data types that can not be stored as 4 byte integers
 when 
\shape italic
--enable-64bit
\shape default
 is being used.
 If the CGNS library was not compiled with a default INTEGER of 8 bytes,
 but the calling program was, then all integers passed to CGNS with C type
 
\shape italic
int
\shape default
 should be declared INTEGER(C_INT).
 
\end_layout

\begin_deeper
\begin_layout Enumerate
CGNS developer's note: A new C data type, cgint_f, was introduced to be
 interpretable with the C type 
\shape italic
int
\shape default
.
 In order to allow for default 8 byte integers in Fortran: 
\end_layout

\begin_deeper
\begin_layout Enumerate
The C API wrappers in cg_ftoc.c were changed from 
\shape italic
cgsize_t
\shape default
 to 
\shape italic
cgint_f
\shape default
 everywhere the C argument is declared as an 
\shape italic
int
\shape default
 in C.
\end_layout

\begin_layout Enumerate
Configure detects what size the default integer is in Fortran and finds
 the corresponding size in C in order to set the correct size of 
\shape italic
cgint_f
\shape default
.
\end_layout

\end_deeper
\end_deeper
\begin_layout Enumerate
Two new benchmarking programs were introduced in directory ptests:
\end_layout

\begin_deeper
\begin_layout Enumerate
benchmarking_hdf5_f90.F90 uses the conventional Fortran wrappers.
\end_layout

\begin_layout Enumerate
benchmarking_hdf5_f03.F90 calls the C APIs directly, no Fortran wrappers
 are used.
\end_layout

\end_deeper
\begin_layout Enumerate
A new Fortran API was added for determining the CGNS data type of a variable
 which is interoperable with the C data type.
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{fortran}
\end_layout

\begin_layout Plain Layout

Function cg_get_type(var)
\end_layout

\begin_layout Plain Layout

   type, INTENT(IN) :: var
\end_layout

\begin_layout Plain Layout

   INTEGER(KIND(enumvar)) :: cg_get_type
\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
An example of using the new function to automatically specify the CGNS type
 corresponding to the Fortran data type is, 
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{minted}[bgcolor=bg]{fortran}
\end_layout

\begin_layout Plain Layout

   INTEGER, DIMENSION(1:10) :: Array_i
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

   CALL cgp_array_write_f("ArrayI",cg_get_type(Array_i(1)),1,INT(nijk(1),cgsize_
t),Ai, err) 
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
end{minted}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
Unfinished Fortran Features
\end_layout

\begin_layout Enumerate
Default double precision for reals in Fortran leads to a mismatch in the
 C APIs, which expect a float.
\end_layout

\begin_layout Section
\begin_inset CommandInset label
LatexCommand label
name "sec:Parallel-installation-instructio"

\end_inset

Parallel installation instructions
\end_layout

\begin_layout Standard
Two parallel files systems were investigated: GPFS (mira, Argonne National
 Laboratory) and Lustre (Pleiades NASA).
 The following descriptions were for those systems, but the overall procedure
 should be similar on different machines of the same type.
 Example build scripts for these systems can be found in src/SampleScripts
 of the CGNS source code.
 They include scripts for building zlib, hdf5 (assuming the user does not
 already have them installed system wide) and a script for building CGNS.
 All the scripts use autotools; 
\color red
cmake remains untested
\color inherit
.
 The next few examples assume all the needed packages are in ${HOME}/packages
 and all the build scripts are placed in ${HOME}/packages.
 This information can also be found in the README.txt in the scripts directory.
\end_layout

\begin_layout Subsection
Building on IBM Blue Gene (GPFS)
\end_layout

\begin_layout Enumerate
Building zlib from source: Download and extract the zlib source: http://www.zlib.n
et/
\end_layout

\begin_deeper
\begin_layout Enumerate
cd into the top level zlib source directory.
 
\end_layout

\begin_layout Enumerate
modify and run the script: ../build_zlib
\end_layout

\end_deeper
\begin_layout Enumerate
Building hdf5 from source
\end_layout

\begin_deeper
\begin_layout Enumerate
From the top level of the hdf5 library, change the ${HOME}/packages to where
 zlib was installed in STEP 1.
 
\end_layout

\begin_layout Enumerate
../build_hdf5 --without-pthread --disable-shared --enable-parallel --enable-produc
tion 
\backslash
 --enable-fortran --enable-fortran2003 
\backslash
 --disable-stream-vfd --disable-direct-vfd 
\backslash
 --with-zlib=${HOME}/packages/zlib-1.2.8/lib --prefix=${HOME}/packages/phdf5-trunk
\end_layout

\begin_layout Standard
where prefix is set for where the hdf5 library will get installed.
 There should be no need to modify the script.
\end_layout

\end_deeper
\begin_layout Enumerate
Building cgns from source:
\end_layout

\begin_deeper
\begin_layout Enumerate
cd into the cgns/src directory 
\end_layout

\begin_layout Enumerate
modify and run: <pathto>/build_cgns 
\end_layout

\begin_layout Enumerate
make
\end_layout

\begin_layout Enumerate
To make the tests: cd ptests; make;make test
\end_layout

\end_deeper
\begin_layout Enumerate
Important parameters for good performance on GPFS:
\end_layout

\begin_deeper
\begin_layout Enumerate
The environment variable BGLOCKLESSMPIO_F_TYPE=0x47504653 should be set.
 For example, this can be set in a batch job using qsub --env BGLOCKLESSMPIO_F_T
YPE=0x47504653
\end_layout

\end_deeper
\begin_layout Subsection
Building on SGI (Lustre)
\end_layout

\begin_layout Enumerate
Building zlib from source: Download and extract the zlib source: http://www.zlib.n
et/
\end_layout

\begin_deeper
\begin_layout Enumerate
cd into the top level zlib source directory.
 
\end_layout

\begin_layout Enumerate
modify and run the script: ../build_zlib
\end_layout

\end_deeper
\begin_layout Enumerate
Building hdf5 from source:
\end_layout

\begin_deeper
\begin_layout Enumerate
From the top level of the hdf5 library, change the ${HOME}/packages to where
 zlib was installed in STEP 1.
 
\end_layout

\begin_layout Enumerate
../build_hdf5
\end_layout

\end_deeper
\begin_layout Enumerate
Building cgns from source:
\end_layout

\begin_deeper
\begin_layout Enumerate
cd into the cgns/src directory 
\end_layout

\begin_layout Enumerate
modify and run: <pathto>/build_cgns 
\end_layout

\begin_layout Enumerate
make
\end_layout

\begin_layout Enumerate
To make the tests: cd ptests; make;make test
\end_layout

\end_deeper
\begin_layout Enumerate
Important parameters for good performance:
\end_layout

\begin_deeper
\begin_layout Enumerate
The Lustre parameters have not been fully tested.
\end_layout

\begin_layout Enumerate
On Pleiades, lfs setstripe -c 64 -s 0 /nobackupp8/<dir>, has shown good
 performance.
\end_layout

\end_deeper
\begin_layout Subsection
Parallel performance results
\end_layout

\begin_layout Standard
The following results are for the 
\shape italic
benchmark_hdf5*
\shape default
 programs found in the 
\shape italic
ptests
\shape default
 directory.
 The benchmark simulates writing and then reading a ~33.5 million 6-node
 pentahedra elements mesh with ~201 million nodes.
 The benchmark results from 
\shape italic
benchmark_hdf5.c
\shape default
 show improvement in cgp_open for up to 32,768 processes, Fig.
 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:Results-for-benchmark_hdf5.c"

\end_inset

 over the previous implementation.
 A comparison of the maximum IO bandwidth speeds for reading and writing
 CGNS data compared to the practical maximum IO bandwidth on cetus (ANL)
 shows the extremely fast writing of the element connectivity table, Fig.
 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:IO-bandwidth-speeds"

\end_inset

.
 A comparison of 
\shape italic
benchmark_hdf5_f90.F90
\shape default
 and 
\shape italic
benchmark_hdf5_f03.F90
\shape default
 also shows negligible performance differences when using the Fortran 90
 wrapper routines (
\shape italic
benchmark_hdf5_f90.F90)
\shape default
 and when calling the C CGNS APIs directly (
\shape italic
benchmark_hdf5_f03.F90)
\shape default
, Fig.
 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:Results-for-benchmark_hdf5_f03.F"

\end_inset

.
 Additionally, negligible difference in performance and scaling exists when
 calling CGNS from Fortran and C, Fig.
 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:Comparision-of-Fortran"

\end_inset

.
\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status collapsed

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename figs/C_gpfs.png
	lyxscale 50
	scale 30

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:Results-for-benchmark_hdf5.c"

\end_inset

Results for benchmark_hdf5.c on GPFS (cetus, ANL).
\end_layout

\end_inset


\end_layout

\begin_layout Plain Layout

\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename figs/Bandwidth_cgns.png
	lyxscale 50
	scale 35

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:IO-bandwidth-speeds"

\end_inset

IO bandwidth speeds for benchmark_hdf5.c on GPFS (cetus, ANL) compared to
 the practical maximum IO bandwidth speeds (shaded area).
\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename figs/C_gpfs.png
	lyxscale 50
	scale 30

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:Results-for-benchmark_hdf5_f03.F"

\end_inset

Results for 
\shape italic
benchmark_hdf5_f03.F90
\shape default
 (shown in color) and 
\shape italic
benchmark_hdf5_f90.F90
\shape default
 (shown in grey) on GPFS (cetus, ANL).
\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename figs/f03_C_gpfs.png
	lyxscale 50
	scale 30

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:Comparision-of-Fortran"

\end_inset

Comparison of Fortran (
\shape italic
benchmark_hdf5_f03.F90, 
\shape default
solid lines) and C (
\shape italic
benchmark_hdf5
\shape default
.c, dashed lines) on GPFS (cetus, ANL).
\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
The benchmark results from 
\shape italic
benchmark_hdf5.c
\shape default
 on the Lustre file system (Pleiades, NASA) is presented in Fig.
 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:Comparision-of-Fortran-1"

\end_inset

.
 The system defaults for the Lustre file system were used along with a strip
 count of 64 and a strip size of 4 MB.
 Overall the Lustre file system appears to be faster at reading a writing,
 but the GPFS scaled better for the benchmark.
\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename figs/Lustre.png
	lyxscale 50
	scale 30

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:Comparision-of-Fortran-1"

\end_inset

Results for 
\shape italic
benchmark_hdf5.c
\shape default
 on Lustre file system (Pleiades, NASA).
\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Section*
Acknowledgments
\end_layout

\begin_layout Standard
This work was funded by NASA through the CGNS project, contract #NNL14AB41T.
 Additionally, the computational studies done on cetus and mira at Argonne
 National Laboratory used allocation time through the ExaHDF5 project, DOE
 contract #DE-AC02-05CH11231.
 Computational studies on the Lustre file system was done on Pleiades through
 NASA's High-End Computer Capabilities Project.
 
\end_layout

\end_body
\end_document
