#LyX 2.0 created this file. For more info see http://www.lyx.org/
\lyxformat 413
\begin_document
\begin_header
\textclass scrreprt
\begin_preamble
\usepackage[dvipsnames]{xcolor}
\usepackage{listings}
\end_preamble
\options abstracton
\use_default_options true
\begin_modules
customHeadersFooters
enumitem
fixltx2e
\end_modules
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman default
\font_sans default
\font_typewriter default
\font_default_family rmdefault
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100
\font_tt_scale 100

\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command default
\paperfontsize 10
\spacing single
\use_hyperref true
\pdf_title "blkreplay"
\pdf_author "Thomas Schöbel-Theuer"
\pdf_bookmarks true
\pdf_bookmarksnumbered false
\pdf_bookmarksopen false
\pdf_bookmarksopenlevel 1
\pdf_breaklinks true
\pdf_pdfborder true
\pdf_colorlinks true
\pdf_backref false
\pdf_pdfusetitle true
\papersize a4paper
\use_geometry true
\use_amsmath 1
\use_esint 1
\use_mhchem 1
\use_mathdots 1
\cite_engine basic
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date false
\use_refstyle 1
\index Index
\shortcut idx
\color #008000
\end_index
\leftmargin 3.7cm
\topmargin 2.7cm
\rightmargin 2.8cm
\bottommargin 2.3cm
\secnumdepth 3
\tocdepth 3
\paragraph_separation indent
\paragraph_indentation default
\quotes_language english
\papercolumns 1
\papersides 2
\paperpagestyle headings
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Title

\family typewriter
blkreplay
\family default
 and Sonar Diagrams
\end_layout

\begin_layout Subtitle
A manual for system administrators,
\begin_inset Newline newline
\end_inset

kernel developers,
\begin_inset Newline newline
\end_inset

hardware technicians,
\begin_inset Newline newline
\end_inset

 and experts in IO systems
\end_layout

\begin_layout Author
Thomas Schöbel-Theuer (
\family typewriter
tst@1und1.de
\family default
)
\end_layout

\begin_layout Date
Version 1.0.0
\end_layout

\begin_layout Standard
\noindent
Copyright (C) 2012 Thomas Schöbel-Theuer / 1&1 Internet AG (see 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

http://www.1und1.de
\end_layout

\end_inset

 shortly called 1&1 in the following).
\end_layout

\begin_layout Standard
Permission is granted to copy, distribute and/or modify this document under
 the terms of the GNU Free Documentation License, Version 1.3 or any later
 version published by the Free Software Foundation; with no Invariant Sections,
 no Front-Cover Texts, and no Back-Cover Texts.
 A copy of the license is included in the section entitled 
\begin_inset Quotes eld
\end_inset


\begin_inset CommandInset ref
LatexCommand nameref
reference "chap:GNU-FDL"

\end_inset


\begin_inset Quotes erd
\end_inset

.
\end_layout

\begin_layout Abstract

\family typewriter
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
sloppy
\end_layout

\end_inset

 blkreplay
\family default
 is a GPL'ed toolkit, driving the block layer of Linux (or other Unix-like
 OSes) while measuring latency and throughput of IO operations for later
 visualization (so-called 
\begin_inset Quotes eld
\end_inset

sonar diagrams
\begin_inset Quotes erd
\end_inset

 and others).
\end_layout

\begin_layout Abstract

\family typewriter
blkreplay
\family default
 comes with a modular and extensible 
\series bold
test suite
\series default
, automating large projects for testing and/or benchmarking.
\end_layout

\begin_layout Abstract

\family typewriter
blkreplay
\family default
 can be used to 
\series bold
test physical hardware
\series default
, e.g.
 compare different brands of hard disks, or RAID controllers / their settings
 / RAID rebuild performance degradation, or to evaluate the effect of SSD
 caching, or to compare different block level transports like iSCSI vs Fibrechan
nel (over different kinds of storage networks).
\end_layout

\begin_layout Abstract
It can compare 
\series bold
virtual hardware
\series default
 (like 
\family typewriter
vmware
\family default
 or 
\family typewriter
XenServer
\family default
 block devices, or any type of block-level 
\series bold
storage virtualization
\series default
) to each other or to physical hardware, provided the test setup is handled
 very carefully
\begin_inset Foot
status open

\begin_layout Plain Layout
Otherwise you may get 
\emph on
useless fake results
\emph default
 measuring the 
\emph on
cache
\emph default
 performance or even 
\emph on
sparse accesses to holes
\emph default
 instead of the real hardware performance.
 Such fake results may differ from real results by 
\emph on
factors
\emph default
, er even by 
\emph on
orders of magnitude
\emph default
.
 
\family typewriter
blkreplay
\family default
 comes with thorough descriptions teaching you how to avoid the most common
 pitfalls.
\end_layout

\end_inset

.
\end_layout

\begin_layout Abstract

\family typewriter
blkreplay
\family default
 can compare 
\series bold
commercial storage
\series default
 systems from vendors like EMC, Net\SpecialChar \-
App, IBM, Hitachi, etc to each other
 or to cheap off-the-shelf hardware (in order to determine the price/performance
 ratio), provided the same care
\begin_inset Foot
status open

\begin_layout Plain Layout
Notice that most commercial storage systems 
\series bold
\emph on
are
\series default
\emph default
 in fact nothing but virtualized storage, so the above warnings about possible
 
\emph on
fake results
\emph default
 apply.
\end_layout

\end_inset

 is taken.
\end_layout

\begin_layout Abstract
Furthermore, it can be used for tests of the 
\series bold
Linux kernel
\series default
, e.g.
 for testing device drivers, comparing IO schedulers at different load patterns,
 determining the overhead of Linux 
\family typewriter
dm
\family default
 targets or the impact of network problems to DRBD, and much more.
\end_layout

\begin_layout Abstract
In addition to artificial loads like random read-write sweeps and various
 kinds of 
\emph on
overload tests
\emph default
, it can also replay 
\series bold
natural loads
\series default
 which have been recorded by 
\family typewriter
blktrace
\family default
 at heavily-loaded production servers at big data centers.
 
\family typewriter
blkreplay
\family default
 comes with a
\series bold
 large collection of natural loads
\series default
 from a wide spectrum of applications (such as web servers, databases, dedicated
 servers, etc) which have been released to the public by 1&1 under GPL.
 Some of these natural loads have recorded the real-life disk access behaviour
 from servers serving thousands of customers in parallel.
 Static analysis of the workingset behaviour of such natural loads is implemente
d.
\end_layout

\begin_layout Abstract
At 1&1, 
\family typewriter
blkreplay
\family default
 has even been used as a tool for root cause analysis of incidents: for
 example, high load peaks presumably stemming from traffic jam (or other
 sources of overload) were recorded at production sites in real time by
 
\family typewriter
blktrace
\family default
, and later replayed in the laboratory (without causing customer impact)
 seeking for the cause of trouble, or improving the safety margins by choice
 of better hardware.
\end_layout

\begin_layout Abstract
For experts in IO subsystems, visualization techniques like 
\begin_inset Quotes eld
\end_inset

sonar diagrams
\begin_inset Quotes erd
\end_inset

 can reveal (parts of) the internal structure of complex IO systems, such
 as cache hierarchies or other hierarchical storage systems.
\end_layout

\begin_layout Abstract
As a community project under GPL, 
\family typewriter
blkreplay
\family default
 is open to contributions from hardware vendors, other data centers, the
 kernel hacker community, etc.
\end_layout

\begin_layout Standard
\begin_inset CommandInset toc
LatexCommand tableofcontents

\end_inset


\end_layout

\begin_layout Chapter
Why Synthetic Benchmarks suck
\end_layout

\begin_layout Standard
There are a lot of benchmark tools around, like 
\family typewriter
iozone
\family default
, 
\family typewriter
iometer
\family default
, 
\family typewriter
iperf
\family default
, 
\family typewriter
bonnie
\family default
(
\family typewriter
++
\family default
) and many others.
\end_layout

\begin_layout Standard
What do they have in 
\emph on
common
\emph default

\begin_inset Foot
status open

\begin_layout Plain Layout
Of course, the mentioned performance measuring tools are targeted at inspection
 of different components of an OS, such as network, filesystem layer, and
 block IO layer.
 Here the question is a about 
\emph on
commonality
\emph default
, not differences.
\end_layout

\end_inset

?
\end_layout

\begin_layout Quotation
Simply, most of them generate an 
\series bold
artificial load
\series default
 onto your system.
\end_layout

\begin_layout Standard
Artificial loads, as opposed to 
\series bold
natural loads
\series default
, have a main disadvantage: they cannot answer questions like 
\begin_inset Quotes eld
\end_inset

will my application Z run on system A reliably?
\begin_inset Quotes erd
\end_inset


\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
<lyxtabular version="3" rows="3" columns="3">
<features tabularvalignment="middle">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
question
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
artificial
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
natural
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Is system A better than B for application Z?
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Formula $partly$
\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Formula $\sqrt{}$
\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Does application Z run on system A?
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Formula $\sqrt{}$
\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Standard
\noindent
It seems that some people 
\emph on
believe
\emph default
 that synthetic benchmarks can be used even at the position of the dash
 in the above table.
 These people are wrong.
\end_layout

\begin_layout Standard
Experiences at big data centers at 1&1 show that sometimes the differences
 between results from artificial benchmarks and real-world application behaviour
 are very large.
 We found cases where artificial benchmarks (adjusting parameters like IOPS)
 suggested that a particular application 
\emph on
should
\emph default
 run on a particular hardware system, but the real application 
\emph on
didn't
\emph default
: after deployment, a systematic series of incidents 
\emph on
disproved
\emph default

\begin_inset Foot
status open

\begin_layout Plain Layout
Sometimes we got results in the other direction: artificial benchmarks suggested
 that a particular application would 
\emph on
not
\emph default
 run, but in reality it 
\emph on
did
\emph default
 run.
\end_layout

\end_inset

 the validity of the former benchmark results for the 
\emph on
originally intended statement
\emph default
.
 The failed prediction from the artificial benchmarks led to a 
\series bold
failed invest
\series default
.
\end_layout

\begin_layout Standard
What can we do about that?
\end_layout

\begin_layout Standard
Obviously, parameters like IOPS (even when enriched with attributes like
 
\begin_inset Quotes eld
\end_inset

average IOPS
\begin_inset Quotes erd
\end_inset

 or 
\begin_inset Quotes eld
\end_inset

peak IOPS
\begin_inset Quotes erd
\end_inset

) are 
\emph on
not
\emph default
 representative for description of the real-world behaviour of applications.
 Attempts to describe real-world behaviour in mathematical terms of analytic
 functions have been already made in the 1970's; they failed.
 All such models can 
\emph on
try
\emph default
 to describe an 
\emph on
approximation
\emph default
 of real-world behaviour, if enough knowledge about the application 
\emph on
would
\emph default
 be available.
\end_layout

\begin_layout Standard
So why trying to deal with tools that never can fully describe real-world
 application behaviour, when there 
\emph on
exist
\emph default
 tools which definitely 
\emph on
can
\emph default
 do?
\end_layout

\begin_layout Standard
One of them is 
\family typewriter
blkreplay
\family default
.
\end_layout

\begin_layout Chapter
How 
\family typewriter
blkreplay
\family default
 works
\end_layout

\begin_layout Section
Principle
\end_layout

\begin_layout Standard
In some sense, 
\family typewriter
blkreplay
\family default
 is just the opposite of the well-known Linux kernel tool 
\family typewriter
blktrace
\family default
: recordings made by 
\family typewriter
blktrace
\family default
 are simply replayed on 
\emph on
another
\emph default
 block device.
\end_layout

\begin_layout Standard
A 
\family typewriter
blktrace
\family default
 record of an IO request contains the following information:
\end_layout

\begin_layout Enumerate
timestamp of the IO request (nanosecond resolution)
\end_layout

\begin_layout Enumerate
position of the IO request (sector#)
\end_layout

\begin_layout Enumerate
length of the IO request (#sectors)
\end_layout

\begin_layout Enumerate
direction: R[ead] or W[write]
\end_layout

\begin_layout Standard
Notice that 
\family typewriter
blktrace
\family default
 records do 
\emph on
not
\emph default
 contain any data.
 Therefore, 
\family typewriter
blkreplay
\family default
 must later generate some 
\emph on
fake data
\emph default
 in order to repeat the timely and positionly behaviour of the original
 recording.
 By default, NULL blocks enriched with some internal header information
 are generated.
 The internal headers may be used for 
\emph on
verification
\emph default
 of the correctness of IO semantics, either by immediate re-read after each
 write, or in a separate verify pass after the end of an ordinary 
\family typewriter
blkreplay
\family default
 run.
\end_layout

\begin_layout Standard
Notice that these NULL blocks will (together with the internal header informatio
n) 
\series bold
destroy
\series default
 any previous content (such as filesystem data) on your block device!
\end_layout

\begin_layout Standard
Therefore, 
\emph on
never
\emph default
 use 
\family typewriter
blkreplay
\family default
 on production systems.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset


\emph on
 Always
\emph default
 use 
\family typewriter
blkreplay
\family default
 in the laboratory, always on devices which don't contain any valuable data!
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Running 
\family typewriter
blkreplay
\family default
 
\emph on
in parallel
\emph default
 to mounted filesystems on the same device
\begin_inset Foot
status open

\begin_layout Plain Layout
Although a single Linux kernel instance tries to probihit such a disaster,
 there are cases where you can 
\begin_inset Quotes eld
\end_inset

achieve
\begin_inset Quotes erd
\end_inset

 that effect.
 Examples are iSCSI connections to the same iSCSI target in parallel.
\end_layout

\end_inset

 will certainly destroy your data and almost certainly crash your kernel.
 Always run at most a single 
\family typewriter
blkreplay
\family default
 instance on a single device!
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 In general, 
\family typewriter
blkreplay
\family default
 is a tool only for 
\emph on
experienced
\emph default
 technicians who know what they do.
 They should be at least at a senior level.
\end_layout

\begin_layout Section
Architecture of 
\family typewriter
blkreplay
\end_layout

\begin_layout Standard
The main challenge for 
\family typewriter
blkreplay
\family default
 is to generate 
\series bold
sufficient IO parallelism
\series default
.
\end_layout

\begin_layout Standard
Ordinary production systems like web servers are serving many thousands
 of HTTP requests per second, which may lead to an IO parallelism at the
 block level of several hundred outstanding IO requests at the same time.
\end_layout

\begin_layout Standard
In order to simulate such a behaviour in the lab, there are principally
 two alternatives:
\end_layout

\begin_layout Enumerate
use the 
\family typewriter
aio
\family default
 interface of the kernel to fire off a large number of IO requests in parallel.
\end_layout

\begin_layout Enumerate
use a sufficiently large number of kernel threads or processes in parallel,
 where each of them fires up only at most one IO request at the same time
 (blocking IO).
\end_layout

\begin_layout Standard
The current version of 
\family typewriter
blkreplay
\family default
 supports only method 2; method 1 is planned for a future release.
\end_layout

\begin_layout Standard
Method 2 is motivated by a typical Apache behaviour: it is almost a 
\begin_inset Quotes eld
\end_inset

fork bomb
\begin_inset Quotes erd
\end_inset

, in particular under high connection rates and slow block devices.
 A high number of ordinary Unix processes is doing conventional 
\family typewriter
read()
\family default
 / 
\family typewriter
write()
\family default
 operations in parallel.
\end_layout

\begin_layout Standard
For easy portability even to historic Unix flavours, 
\family typewriter
blkreplay
\family default
 uses ordinary Unix processes generated by simple 
\family typewriter
fork()
\family default
s and communicating via anonymous pipes, in favour to a shared-memory 
\family typewriter
pthreads
\family default
 model.
 However, future versions may also support 
\family typewriter
pthreads
\family default
.
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename architecture.fig
	width 75col%

\end_inset


\end_layout

\begin_layout Section
Mode of Operation
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 It is 
\series bold
essential
\series default
 that you understand the concepts described in this chapter.
 Otherwise you may produce 
\series bold
useless fake results
\series default
, deviating from valid measurements by 
\emph on
factors
\emph default
, or even 
\emph on
orders of magnitude
\emph default
.
\end_layout

\begin_layout Standard
It is crucial to understand the 
\emph on
operating environment
\emph default
 where 
\family typewriter
blkreplay
\family default
 is running.
 Please take a look at the following picture:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename io-layers.fig
	width 50text%

\end_inset


\end_layout

\begin_layout Standard
\noindent
In the live system, 
\family typewriter
blktrace
\family default
 will 
\begin_inset Quotes eld
\end_inset

listen
\begin_inset Quotes erd
\end_inset

 to the events occurring at point A, and will record them.
 In this example, we have a complex system, running virtual machines inside
 a hypervisor.
\end_layout

\begin_layout Standard
Notice that all IO requests of the application will not only go through
 the filesystem, but also through the buffer / page cache.
 The in-memory cache will usually serve most IO requests from the application,
 without causing physical IO at the block layer (so-called 
\begin_inset Quotes eld
\end_inset

cache hits
\begin_inset Quotes erd
\end_inset

).
\end_layout

\begin_layout Standard
On some well-tuned production servers, it is no problem to achieve cache
 hit rates of 99% or more, leading to a kind of 
\begin_inset Quotes eld
\end_inset

gear ratio
\begin_inset Quotes erd
\end_inset

 of 1:100, or even 1:1000 (in long-term runs).
 Of course, there also exist heavy workloads running on thin servers, where
 sometimes less than 1:10 can be achieved.
 Even in that case, there will be 
\emph on
always
\emph default
 some cache hits, for example caused by metadata requests from the filesystem.
 In practice, the cache hit rate will never go down to 0%.
 Notice that these inevitable cache effects are 
\emph on
already included
\emph default
 in any 
\family typewriter
blktrace
\family default
!
\end_layout

\begin_layout Standard
Now look at the situation in the laboratory.
 The application and the filesystem is no longer present, but its 
\emph on
effects
\emph default
 are simulated by 
\family typewriter
blkreplay
\family default
.
 Due to the architecture of the Linux kernel, all IO requests will continue
 to run through the buffer cache
\begin_inset Foot
status open

\begin_layout Plain Layout
Several commercial Unices used a concept called 
\begin_inset Quotes eld
\end_inset

raw device
\begin_inset Quotes erd
\end_inset

 which circumvented their buffer cache.
 In contrast, the internal structures of Linux device driver are internally
 interwoven with the page cache in a rather sticky way.
 Instead of 
\begin_inset Quotes eld
\end_inset

raw devices
\begin_inset Quotes erd
\end_inset

, Linux uses the concept of 
\begin_inset Quotes eld
\end_inset

direct IO
\begin_inset Quotes erd
\end_inset

, which 
\emph on
tries
\emph default
 to minimize any caching effects.
\end_layout

\end_inset

.
\end_layout

\begin_layout Standard
It should be clear by the very nature of our experiment, that at measuring
 point B 
\emph on
exactly
\emph default
 the same events 
\emph on
should
\emph default
 happen as had been formerly observed / recorded at point A.
\end_layout

\begin_layout Standard
Thus, the page / buffer cache of the laboratory system 
\emph on
must
\emph default
 be switched off.
 Otherwise, a 
\begin_inset Quotes eld
\end_inset

gear ratio
\begin_inset Quotes erd
\end_inset

 of 1:10 (or let it be only 1:1.1) would lead to distortions of the measurement
 results.
 In order to switch off the page / buffer cache, 
\family typewriter
blkreplay
\family default
 uses 
\family typewriter
O_DIRECT
\family default
 mode as offered by the Linux kernel.
\end_layout

\begin_layout Standard
Notice that even by these measures, there may remain some subtle differences
 between the operations occurring at point A and point B.
 The block layer of the Linux kernel does some optimizations, for example
 it tries to 
\emph on
coalesce
\emph default
 adjacent requests, or to 
\emph on
split
\emph default
 some requests, depending on the capabilities of the hardware.
 Usually, these are minor modifications, occurring at less than 1% of all
 requests.
 However, keep in mind (and check) these effects.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 The 
\series bold
elevator strategy
\series default
 (aka IO scheduler) of the Linux block layer is a bigger influence factor.
 It can 
\emph on
reorder
\emph default
 requests, and it can even add 
\emph on
artificial latencies
\emph default
.
 For example, 
\family typewriter
CFQ
\family default
 adds some speculative latencies in some cases to increase the chances for
 request mergers.
 Selecting the 
\begin_inset Quotes eld
\end_inset

wrong
\begin_inset Foot
status open

\begin_layout Plain Layout
Many sources from the internet claim that 
\family typewriter
CFQ
\family default
 is the 
\begin_inset Quotes eld
\end_inset

best
\begin_inset Quotes erd
\end_inset

 IO scheduler.
 While this is often true for typical workstation load pattern (and interactive
 user expectations), our experience at 1&1 is different with regards to
 
\emph on
server
\emph default
 loads.
 Check yourself! Run some 
\family typewriter
blkreplay
\family default
 comparisons between different IO schedulers.
 Hint: there may be even non-linear dependencies from the lower level, whether
 you have some RAID with BBU cache, or not.
\end_layout

\end_inset


\begin_inset Quotes erd
\end_inset

 scheduler may lead to larger distortions, even to seemingly 
\begin_inset Quotes eld
\end_inset

bad
\begin_inset Quotes erd
\end_inset

 behaviour (which is 
\emph on
not
\emph default
 the fault of bad hardware).
 In order to really get (almost) the same behaviour at points A and B, you
 
\emph on
must
\emph default
 select the 
\family typewriter
NOOP
\family default
 or at least the 
\family typewriter
DEADLINE
\family default
 scheduler.
 See 
\family typewriter
/sys/dev/block/*/queue/scheduler
\family default
.
\end_layout

\begin_layout Section
Overlapping of IO Requests
\end_layout

\begin_layout Standard
In general, there are two kinds of overlapping between IO requests in real
 production systems (at the time when a 
\family typewriter
blktrace
\family default
 record is made):
\end_layout

\begin_layout Enumerate
timely
\end_layout

\begin_layout Enumerate
positionly
\end_layout

\begin_layout Standard
Both kinds form a two-dimensional space:
\end_layout

\begin_layout Standard
\align center
\begin_inset Tabular
<lyxtabular version="3" rows="3" columns="3">
<features tabularvalignment="middle">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
overlapping
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
timely yes
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
timely no
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
positionly yes
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Formula $\sqrt{}$
\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
positionly no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Formula $\sqrt{}$
\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Formula $\sqrt{}$
\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Standard
\noindent
Pure timely overlapping (without positionly overlapping) is a frequent case
 in almost any IO system (usually called 
\begin_inset Quotes eld
\end_inset

IO parallelism
\begin_inset Quotes erd
\end_inset

 in folklore).
 In opposite, purely positionly overlapping (without timely overlapping)
 is also a frequent case, for example when the same sector is re-read after
 a while, or re-written when the contents of a file changes frequently.
 Completely unrelated requests (neither timely nor positionly overlapping)
 are probably the most frequent case in most practical load scenarios at
 production sites.
 There is no problem with any of them, indicated by the checkmarks.
\end_layout

\begin_layout Standard
However, what about both kinds of overlapping at the same time?
\end_layout

\begin_layout Standard
The case of 
\emph on
both
\emph default
 timely and positionly overlapping (simultanously) of IO operations is called
 
\emph on
damaged IO
\emph default
.
\end_layout

\begin_layout Standard
In ordinary OS kernels, damaged IO usually 
\emph on
never
\emph default
 occurs.
 Here are the reasons:
\end_layout

\begin_layout Standard
IO requests are usually generated by in-kernel memory caches like buffer
 caches or page caches.
 Even in case of databases using Direct IO when submitting requests to the
 block layer, their internal database buffer cache works similarly to in-kernel
 caches.
 It simply makes 
\emph on
no sense
\emph default
 to write to the same sector twice at the same time, because the result
 will be undefined.
 A similar argument holds for reads in parallel to writes to/from the same
 sector.
\end_layout

\begin_layout Standard
In theory, concurrent reads from the same sector would be possible without
 causing harm to data integrity on the block device.
 In operating system caches, this would introduce 
\emph on
copies
\emph default
 of the same data into the buffer or page cache, violating its internal
 
\emph on
uniqueness
\emph default
 properties stating that any sector is cached at most once.
 Consequently, this case also never appears 
\emph on
in practice
\emph default
 at 
\emph on
real-life systems(!)
\emph default
.
\end_layout

\begin_layout Standard
However, in terms of 
\series bold
correctness of storage semantics
\series default
 the case of damaged IO involving read/read to the same sector is 
\emph on
allowed
\emph default
.
 Probably you already know the following table from database textbooks and
 others:
\end_layout

\begin_layout Standard
\align center
\begin_inset Tabular
<lyxtabular version="3" rows="3" columns="3">
<features tabularvalignment="middle">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
conflict?
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
read
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
write
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
read
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
write
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Standard
\noindent
Some block IO systems like DRBD show some misbehaviour in case of concurrent
 writes to the same sector, or in some cases they even fail.
 Some DRBD versions
\begin_inset Foot
status open

\begin_layout Plain Layout
At present, this seems to be an undocumented behaviour observed by the author.
 Even if DRBD's behaviour may change in the future: damaged IO is a bad
 idea by itself.
 It would be unfair to blame DRBD for 
\begin_inset Quotes eld
\end_inset

psychologically unexpected
\begin_inset Quotes erd
\end_inset

 behaviour under illegal load patterns, which should never occur.
 In general, making code rubust against damaged IO could decrease performance
 during ordinary operation.
 Thus damaged IO should be avoided at its 
\emph on
source
\emph default
.
\end_layout

\end_inset

 will at least delay further IO requests for several milliseconds, lowering
 IO bandwidth or even leading to temporary hangs.
\end_layout

\begin_layout Standard
So, damaged IO 
\emph on
should
\emph default
 be avoided under all circumstances.
 Failing to do so may result in a disaster; in general, some IO devices
 like elder tape drives may even be corrupted as a whole.
\end_layout

\begin_layout Standard
While avoidance of damaged IO is automatically guaranteed in real production
 systems by the buffer / cache page of the Linux kernel (or other components
 like database memory buffers), our tool 
\family typewriter
blkreplay
\family default
 must be designed very carefully not to step into that pitfall.
\end_layout

\begin_layout Standard
Why is there a risk that 
\family typewriter
blkreplay
\family default
 could (accidentally) start some damaged IO?
\end_layout

\begin_layout Standard
Well, replay of the original timing of requests is not always possible.
 Even if 
\family typewriter
blkreplay
\family default
 
\emph on
tries
\emph default
 to start IO requests in the same timely pattern as at the original site,
 a very slow device (or a heavily overloaded device) may delay an IO operation
 for a very long time.
 In overload scenarios, or in case of iSCSI network hangs, it is possible
 that some IO requests may take 5 minutes to complete (or even more, or
 even 
\emph on
never
\emph default
 complete in case of fatal errors).
 In such cases, it is not unlikely that a new write to the same sector is
 started before the old one has completed.
\end_layout

\begin_layout Standard
In order to avoid damaged IO, 
\family typewriter
blkreplay
\family default
 uses some in-memory hash tables to detect any (potential) conflicts between
 IO requests.
\end_layout

\begin_layout Standard
In order to deal with (potential) problems caused by damaged IO, we use
 the following options in 
\family typewriter
blkreplay
\family default
 to control its behaviour at 
\emph on
replay
\emph default
 time:
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--strong=0
\family default
 For the sake of conflict detection, only write/write conflicts will count.
 This mode ensures that the 
\begin_inset Quotes eld
\end_inset

storage semantics
\begin_inset Quotes erd
\end_inset

 is obeyed with respect to the 
\emph on
written data
\emph default

\begin_inset Foot
status open

\begin_layout Plain Layout
This allows to check the end result via 
\family typewriter
verify
\family default
 modes for correctness.
\end_layout

\end_inset

, but it allows reads to permute with writes (which is 
\begin_inset Quotes eld
\end_inset

wrong
\begin_inset Quotes erd
\end_inset

 in 
\emph on
strong
\emph default
 sense, hence the name).
 In essence, the following conflict table is used internally for conflict
 detection:
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset space \hfill{}
\end_inset


\begin_inset Tabular
<lyxtabular version="3" rows="3" columns="3">
<features tabularvalignment="middle">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
conflict?
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
read
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
write
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
read
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
no
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
write
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--strong=1
\family default
 (default) Use the standard conflict table as known from the literature.
 Only read/read is treated as non-conflicting:
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset space \hfill{}
\end_inset


\begin_inset Tabular
<lyxtabular version="3" rows="3" columns="3">
<features tabularvalignment="middle">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
conflict?
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
read
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
write
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
read
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
write
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--strong=2
\family default
 
\emph on
All
\emph default
 damaged IO is treated as conflicting, even read/read.
 This may be useful for simulating the behaviour of 
\emph on
real
\emph default
 OS caches.
 This is equivalent to the following table:
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset space \hfill{}
\end_inset


\begin_inset Tabular
<lyxtabular version="3" rows="3" columns="3">
<features tabularvalignment="middle">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<column alignment="center" valignment="top" width="0">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
conflict?
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
read
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
write
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
read
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
write
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Standard
Indepently from the correctness criterion, the following operation modes
 may be selected.
 They determine the kind of reaction in case of detected conflicts:
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--with-conflicts
\family default
 No countermeasures against damaged IO are taken.
 Consequently, is does not matter which 
\family typewriter
--strong=
\family default
 mode you have selected before.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--with-drop
\family default
 Whenever a new request is conflicting with an old (already issued) request,
 it is simply dropped.
 This has no side effects, other than that some reads and/or writes may
 be missing (depending on 
\family typewriter
--strong=
\family default
 mode).
 This has the lowest overhead of all conflict-avoiding methods.
 Depending on properties of the load, the number of dropped requests may
 be rather high.
 Please check the tail of the result file.
 In the statistics section, you will find the number of dropped requests.
 If that number is higher than, say, 5%, you should consider one of the
 following options.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--with-partial
\family default
 (default) Any conflicting requests (as determined by 
\family typewriter
--strong=
\family default
) are pushed back to an internal pushback list and kept there until the
 conflict is gone.
 Pushed back requests are immediately submitted as soon as the conflict
 has gone away.
 This results in a reordering of the 
\emph on
affected
\emph default
 requests, while 
\emph on
trying
\emph default
 to replay unaffected ones at original timestamps.
 This leads to a partial ordering of requests, which may be 
\emph on
very different
\emph default
 from the original ordering, and thus may 
\begin_inset Quotes eld
\end_inset

violate
\begin_inset Quotes erd
\end_inset

 the 
\begin_inset Quotes eld
\end_inset

original storage semantics
\begin_inset Quotes erd
\end_inset

 if it would make a difference on the replay system.
\begin_inset Newline newline
\end_inset

This mode leads to a 2-class society, where ordinary requests are processed
 faster than conflicting ones.
 Further details may be found in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Influence-of-Request"

\end_inset

.
\begin_inset Newline newline
\end_inset

Attention! as a side effect, this mode may 
\emph on
increase
\emph default
 the actual IO parallelism to a larger number than configured via the 
\family typewriter
--threads=
\family default
 parameter, because some request slots (and in turn, also some threads)
 must
\begin_inset Foot
status open

\begin_layout Plain Layout
Otherwise a reactivated request from the pushback list could have to wait
 for an ordinary request slot to become free, which would result in an artificia
l delay.
 Experiments have show that such kinds of distortions can be serious.
\end_layout

\end_inset

 be reserved 
\emph on
in advance
\emph default
 for pushed-back requests.
 When conflicts are gone and the system tries to 
\begin_inset Quotes eld
\end_inset

catch up
\begin_inset Quotes erd
\end_inset

, additional IO requests may be submitted.
\begin_inset Newline newline
\end_inset

In practice, this can happen in particular with some Windows loads, where
 a lot of writes seem to be repeated.
 Frequently, pushed back requests (colored differently for better distinction
 from ordinary requests) are the main contributors to delays.
 Check them!
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--with-ordering
\family default
 The main process spreads its IO requests to the worker processes in a round-rob
in fashion over their anonymous pipes.
 Whenever conflicting requests (as determined by 
\family typewriter
--strong=
\family default
) are detected, this spreading process will stop until the conflict has
 gone away.
 In consequence, the original ordering of requests will be preserved as
 much as possible.
 As a side effect, 
\emph on
all following
\emph default
 IO requests are also delayed, even if they don't conflict with anything
 else.
 This can lead to 
\emph on
artificial delays
\emph default
.
\begin_inset Newline newline
\end_inset

In general, this mode is the 
\begin_inset Quotes eld
\end_inset

most robust
\begin_inset Quotes erd
\end_inset

 one.
\end_layout

\begin_layout Standard
In many cases, you will prefer 
\family typewriter
--with-partial
\family default
, which is the default.
 It delivers almost the same throughput as 
\family typewriter
--with-conflicts
\family default
 without the downsides of 
\family typewriter
--with-drop
\family default
, while minimizing artificial delays
\begin_inset Foot
status open

\begin_layout Plain Layout
In extreme cases, pushed-back requests (visualized in stronger colors) can
 form some kind of 
\begin_inset Quotes eld
\end_inset

transitive queues
\begin_inset Quotes erd
\end_inset

, e.g.
 when pushback requests depend on other pushback requests transitively.
 In such a case, their delays can sum up over a longer time, independently
 from 
\begin_inset Quotes eld
\end_inset

normal
\begin_inset Quotes erd
\end_inset

 requests (clearly visible in the delay diagram).
 Although this is often a property of the load, you can 
\emph on
try(!)
\emph default
 to minimize such an effect by setting the expert option 
\family typewriter
ahead_limit
\family default
 to values lower than 1s (however too low values will destroy throughput
 at all).
 Dangerous!
\end_layout

\end_inset

.
\end_layout

\begin_layout Standard
However, 
\family typewriter
--with-ordering
\family default
 is also useful in many scenarios.
 Practical experience from many hardware tests shows that the artificial
 delays caused by 
\family typewriter
--with-ordering
\family default
 seem to be an 
\emph on
advantage
\emph default
.
 Whenever such stalls occur more than seldomly, they act as an 
\series bold
indicator
\series default
 for massive IO problems of the test candidate.
 
\family typewriter
--with-ordering
\family default
 is often a kind of 
\begin_inset Quotes eld
\end_inset

detector
\begin_inset Quotes erd
\end_inset

 for hardware problems, since it visualizes any problems in an eye-catching
 way.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 When you start a lot of threads (typically 
\begin_inset Formula $\geq256$
\end_inset

), 
\family typewriter
--with-ordering
\family default
 may yield 
\emph on
better(!)
\emph default
 throughput than 
\family typewriter
--with-partial
\family default
.
 In such cases, the reason is 
\emph on
counter-intuitive
\emph default
, as explained in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Influence-of-Number"

\end_inset

: artificial delays caused by 
\family typewriter
--with-ordering
\family default
 will 
\emph on
decrease
\emph default
 the 
\emph on
average
\emph default
 request queue length actually occuring at runtime, which will in turn 
\emph on
increase
\emph default
 the average throughput, depending on counter-intuitive properties of your
 test candidate.
 A way to find out is just to run the same benchmarks with 
\family typewriter
--threads=16
\family default
 or 
\family typewriter
--threads=32
\family default
.
 In case 
\family typewriter
--with-partial
\family default
 is now better than 
\family typewriter
--with-ordering
\family default
, you likely detected that problem.
 Don't draw wrong conclusions from such counter-intuitive effects!
\end_layout

\begin_layout Section
Verification of Storage Semantics
\end_layout

\begin_layout Standard
In order to allow verification of the sector headers and their timestamps
 / version stamps, 
\family typewriter
blkreplay
\family default
 needs some temporary storage where information can be kept for a longer
 time than just during IO.
 Two temporary files are used: 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/tmp/verify_table
\end_layout

\end_inset


\family default
 and 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/tmp/completion_table
\end_layout

\end_inset


\family default
.
 Both are sparse files, containing a simple (sparse) array of sequence numbers,
 indexed by the position (sector#).
 Whenever a write is started, a new sequence number is recorded in 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/tmp/verify_table
\end_layout

\end_inset


\family default
.
 Whenever that write is completed, the same number is recorded in 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/tmp/completion_table
\end_layout

\end_inset


\family default
.
 At any time, both tables keep track of the current progress of write operations.
 
\end_layout

\begin_layout Standard
Whenever the sequence numbers at the same position (sector#) are different
 between both files, we know that a write operation has not yet completed.
 If they are the same, we know which sequence number should appear in the
 header of the corresponding sector.
\end_layout

\begin_layout Standard
There are following variants of verification modes:
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--with-verify
\family default
 Whenever a sector is read (by a regular read request) which has been written
 before (by an ordinary write request), the read data is checked against
 the sequence numbers from the tables.
 Any mismatches are reported by the string 
\family typewriter
VERIFY ERROR
\family default
 in the result file.
 Thus, 
\family typewriter
zgrep 
\begin_inset Quotes eld
\end_inset

VERIFY ERROR
\begin_inset Quotes erd
\end_inset

 *.replay.gz
\family default
 will show them to you.
 In addition, the statistics section at the end of the output file will
 contain some valuable information.
\begin_inset Newline newline
\end_inset

Warning! this mode can only reveal errors in the storage semantics if written
 blocks are re-read somewhen.
 When sectors are just written, but never read, this mode will not detect
 anything.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--with-final-verify
\family default
 In addition to 
\family typewriter
--with-verify
\family default
, a separate pass will be started at the end of a 
\family typewriter
blkreplay
\family default
 run.
 All sectors which have been touched before, will be checked.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--with-paranoia
\family default
 In addition, any written sector will be immediately re-read during operation.
 This doubles the IO rate and leads to extremely high distortions of measurement
 results.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 All verify modes will create temporary tables in 
\family typewriter
/tmp/
\family default
.
 Although the temporary files are sparse, they can use up a significant
 amount of storage (depending on the load).
 The additional IOs form a 
\emph on
sequential bottleneck
\emph default
, and therefore can slowdown 
\family typewriter
blkreplay
\family default
 considerably.
 Please use the verify modes only for 
\emph on
validation
\emph default
, but not for measurements / determining performance!
\end_layout

\begin_layout Chapter
How to use 
\family typewriter
blkreplay
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Running 
\family typewriter
blkreplay
\family default
 naïvely without reading this chapter may easily lead to 
\series bold
completely worthless fake results
\series default
 which would be only useful for production of bullshit!
\end_layout

\begin_layout Standard
In science, it would be unethical to produce such bullshit willingly or
 even deliberately.
\end_layout

\begin_layout Standard
In industry, usage of such bullshit (even inadvertendly) may easily lead
 to failed invests up to millions of Euros / Dollars (depending on the applicati
on and the size of your datacenter).
\end_layout

\begin_layout Standard
Most of the following advice will also apply to other benchmark tools like
 
\family typewriter
iometer
\family default
.
\end_layout

\begin_layout Section
How to Avoid Common Pitfalls
\begin_inset CommandInset label
LatexCommand label
name "sec:How-to-Avoid"

\end_inset


\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Don't skip this section! Read it 
\emph on
completely
\emph default
, even if you are impatient or under time pressure!
\end_layout

\begin_layout Standard
Modern IO subsystems often use some kind of 
\series bold
storage virtualization
\series default
 interally.
 More often than you can dream of, concepts from storage virtualization
 are used (interally) in places where you don't expect them.
\end_layout

\begin_layout Standard
Example: seemingly 
\begin_inset Quotes eld
\end_inset

simple
\begin_inset Quotes erd
\end_inset

 SSDs or even some USB sticks(!) show some of the behaviours described next.
\end_layout

\begin_layout Subsection
Pitfalls from Storage Virtualization
\begin_inset CommandInset label
LatexCommand label
name "sub:Pitfalls-from-Storage"

\end_inset


\end_layout

\begin_layout Standard
The basic idea of storage virtualization is some kind of 
\begin_inset Quotes eld
\end_inset

translation
\begin_inset Quotes erd
\end_inset

 (or 
\begin_inset Quotes eld
\end_inset

mapping
\begin_inset Quotes erd
\end_inset

) between 
\emph on
virtual storage addresses
\emph default
 and 
\emph on

\begin_inset Quotes eld
\end_inset

physical
\begin_inset Quotes erd
\end_inset

 storage addresses
\emph default
.
\end_layout

\begin_layout Standard
In many
\begin_inset Foot
status open

\begin_layout Plain Layout
A promient exception is classical LVM as implemented by the Linux kernel:
 unless you use LVM snapshots or other advanced features, it uses almost
 static mappings, and it carries almost no observable overhead in many practical
 scenarios.
\end_layout

\end_inset

 cases, the address translation / mapping is created on the fly, dynamically
 at runtime.
 In the following simplified
\begin_inset Foot
status open

\begin_layout Plain Layout
For simplicity, this example assumes that the address translation uses the
 same granularity as the benchmark, e.g.
 single sectors.
 We also don't discuss the internals of the mapping which can also have
 a drastic impact on measurement results.
\end_layout

\end_inset

 example, the timely order of accesses is marked by increasing numbers,
 while the type of IO request is indicated by colors (red = write, blue
 = read):
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename storage-virtualization.fig
	lyxscale 70
	width 80col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
We start with an empty logical address space (often called 
\begin_inset Quotes eld
\end_inset

logical volume
\begin_inset Quotes erd
\end_inset

, shortly LV), having a logical size of several terabytes.
 Here, 
\begin_inset Quotes eld
\end_inset

empty
\begin_inset Quotes erd
\end_inset

 means that initially 
\emph on
no
\emph default
 assignment between logical and physical sector numbers exists.
 Now we start a short benchmark (whether 
\family typewriter
blkreplay
\family default
 or others like 
\family typewriter
iometer
\family default
).
 When request #1 (a write) arrives, no physical location exists yet.
 Therefore, a new location must be assigned.
 In this example, the locations are always taken from the beginning of the
 physical address space.
\end_layout

\begin_layout Standard
Thus, all the physical sectors occurring in this example will be allocated
 in a very small and compact area at the start of the physical address space.
 Imagine the drawing not being true to scale: imagine a total size of several
 terabytes (
\begin_inset Formula $>10^{9}$
\end_inset

 sectors), and a 
\family typewriter
blkreplay
\family default
 benchmark touching only a few thousands of sectors.
 What will be the effect?
\end_layout

\begin_layout Enumerate
Only a 
\emph on
tiny
\emph default
 fraction of the physical space will be actually used, usually less than
 one per mille or even less than a millionth.
\begin_inset Newline newline
\end_inset

In contrast, real world applications as well as real customers tend to use
 up significant space with real data, usually more than 50% (and up to 100%).
\end_layout

\begin_layout Enumerate
Even if accesses to the LV are (pseudo-)random, accesses to the 
\begin_inset Quotes eld
\end_inset

physical hot area
\begin_inset Quotes erd
\end_inset

 will 
\emph on
not
\emph default
 remain random: as you can see, they are translated to (purely / almost)
 
\emph on
sequential
\emph default
 access patterns.
 If the physical addresses are residing on a mechanical hard disk, (almost)
 no seek operations will occur.
 Additionally, the physical operations are in 
\emph on
ascending
\emph default
 order, which is a classical use case for BBU-cached writes and/or readahead
 strategies.
 Notice that ascending sequential IO on hard disks is usually faster than
 random IO by a factor of 100 or even more (depending on hardware and further
 factors like RAID, between one and three orders of magnitude).
\begin_inset Newline newline
\end_inset

In contrast, real-world writes will be spreaded much more randomly over
 the physical partition, and there will be a significant amount of 
\emph on
in-place updates
\emph default
 in many practical use cases.
\end_layout

\begin_layout Enumerate
Even worse, read requests need not be mapped at all (indicated by shading
 in the drawing).
 When reading from an address where never anything had been written before,
 NULL blocks may be returned on-the-fly, without causing any physical access
 at all
\begin_inset Foot
status open

\begin_layout Plain Layout
A similar effect is known from holes in traditional Unix 
\emph on
sparse files
\emph default
.
\end_layout

\end_inset

.
 Notice that such 
\begin_inset Quotes eld
\end_inset

fake reads
\begin_inset Quotes erd
\end_inset

 can be faster than true read accesses by 
\emph on
several
\emph default
 orders of magnitude.
\end_layout

\begin_layout Enumerate
Even in case read requests are also mapped upon first reference
\begin_inset Foot
status open

\begin_layout Plain Layout
Several commercial storage boxes are known to do so.
 However, notice that this behaviour is not documented, and thus not guaranteed
 by the vendors.
 They sell you a blackbox.
 All you can do is to analyze such behaviour if you are curious about their
 internals.
 In their next firmware release, the behaviour may be already different
 without notice.
\end_layout

\end_inset

, perhaps leading to a physical IO (or perhaps not), the same arguments
 as for writes apply.
\end_layout

\begin_layout Enumerate
If you 
\emph on
repeat
\emph default
 the same 
\family typewriter
blkreplay
\family default
 benchmark once again, immediately after the first run, you will get another
 surprise: this time the mapping between logical and physical addresses
 already exists, thus you will likely get different results, usually drastically
 better, but seldomly slightly worse, depending on the vendor of the storage
 virtualization (and on many other factors such as the size of the logical
 volume).
 In scientific terminology: your experiment is not truly 
\emph on
repeatable
\emph default
.
\end_layout

\begin_layout Standard
What can you do about that?
\end_layout

\begin_layout Standard
There is no general solution for all cases.
 It depends on the 
\emph on
statement
\emph default
 you want to prove or disprove by usage of 
\family typewriter
blkreplay
\family default
.
\end_layout

\begin_layout Standard
The following is just an 
\emph on
approximation
\emph default
 if you want to reveal the real-life
\begin_inset Foot
status open

\begin_layout Plain Layout
In real life, customer data or enterprise data is stored on LVs.
 Thus benchmarks of empty LVs are 
\emph on
completely wrong
\emph default
 if you try to reveal real life behaviour.
\end_layout

\end_inset

 behaviour of virtualized storage systems:
\end_layout

\begin_layout Enumerate
Whenever you start a new run of a benchmark, you 
\series bold
\emph on
must
\emph default
 delete
\series default
 your old LVs, and create new ones.
 Otherwise, your old run will influence the new one in some way you cannot
 predict easily.
 Remember that the mapping table in the above example is a kind of 
\begin_inset Quotes eld
\end_inset

memory
\begin_inset Quotes erd
\end_inset

 which records not only the sector numbers occurring in your benchmark,
 but even their timely order.
 Make sure that this kind of 
\begin_inset Quotes eld
\end_inset

memory
\begin_inset Quotes erd
\end_inset

 is erased completely
\begin_inset Foot
status open

\begin_layout Plain Layout
Even delete your LVs if you believe that's unnecessary, because you have
 obeyed point 2 and have filled it with data to initialize the mapping:
 some storage systems make 
\emph on
re-assignments
\emph default
 of the mapping during your benchmarks.
 Because many commercial storage systems are blackboxes, you cannot immediately
 see that.
 Always keep in mind that usually ordinary benchmarks will only touch a
 
\emph on
tiny fraction
\emph default
 of all physical sectors, compared to real life!
\end_layout

\end_inset

 between any runs!
\end_layout

\begin_layout Enumerate
After each fresh creation of a logical volume, 
\series bold
fill it with data
\series default
.
 This is the only reliable
\begin_inset Foot
status open

\begin_layout Plain Layout
Some storage vendors have internal functions which 
\emph on
preallocate
\emph default
 the space for a LV.
 Don't use them! Don't trust any claims that this would be equivalent to
 filling with random data - we found cases where we could 
\emph on
disprove
\emph default
 such claims, where results differed even by 
\emph on
factors
\emph default
.
 Just fill your LVs with random data to be sure, even if this delays your
 measurements for some hours or even days.
 Keep in mind that later production systems will take weeks or even months
 to be filled with data before potential problems could show up, so don't
 hesitate to resemble such kind of effort in the laboratory.
\end_layout

\end_inset

 way.
 Best practice is to use tools like 
\family typewriter
wipe
\family default
, filling the whole
\begin_inset Foot
status open

\begin_layout Plain Layout
If you are consciously concerned that filling the 
\emph on
whole
\emph default
 LV might not catch your usecase where (say) only 50% of logical space is
 actually used, you 
\emph on
could
\emph default
 try to fill only 50% of the LV.
 However, be 
\emph on
sure
\emph default
 to fill any blocks which occur in the benchmarks.
 Otherwise, the effects described above will almost certainly lead to a
 
\emph on
higher
\emph default
 distortion of your results than just filling with 100%.
 Notice that some of the above effects deal with orders of magnitude, not
 just a few percent.
\end_layout

\end_inset

 LV with random data.
 Filling with NULL blocks is not recommended, because some blackbox storage
 systems might detect this easily and circumvent it by not creating a mapping
 at all (or even erasing an old mapping similar to 
\family typewriter
punch
\family default
 operations).
\end_layout

\begin_layout Enumerate

\emph on
Immediately
\emph default
 after filling with random data, start your benchmark 
\series bold
exactly once
\series default
.
 
\emph on
Never
\emph default
, really 
\emph on
never
\emph default
 kill a run of 
\family typewriter
blkreplay
\family default
 (or any other benchmark tool) and restart it.
 In case of any error or misbehaviour, you 
\emph on
must
\emph default
 start over with step 1!
\end_layout

\begin_layout Enumerate
There is a single exception if you really know what you are doing: immediately
 after the first run, you may restart the 
\emph on
same
\emph default
 benchmark once again, in order to reveal some hidden properties of the
 mapping.
 You 
\emph on
must
\emph default
 name your output files differently from the first run, and you 
\emph on
must not
\emph default
 confuse the meaning of the second run with the meaning of the first run.
\end_layout

\begin_layout Subsection
Pitfalls from Caches
\begin_inset CommandInset label
LatexCommand label
name "sub:Pitfalls-from-Caches"

\end_inset


\end_layout

\begin_layout Standard
It is easy to be caught by these pitfalls (even if you try to avoid them
 very hard), since caches occur very frequently in almost any type of storage
 system, and even in places where you don't expect them.
 In addition, some real-life loads have hidden properties you cannot see
 at first glance.
\end_layout

\begin_layout Subsubsection
Pitfalls from Cache Operation States
\end_layout

\begin_layout Standard
Many people believe that the most important case is 
\emph on
cold caches
\emph default
 versus 
\emph on
hot caches
\emph default
.
 Although this is not completely wrong, it is not always fully true.
 There is another more important property of cache states: 
\series bold
steady state
\series default
.
\end_layout

\begin_layout Standard
Steady state is not the same as hot state.
 When you start your system freshly, many caches will be of course empty
\begin_inset Foot
status open

\begin_layout Plain Layout
There are some exceptions: SSD caches may start in a hot state (caused by
 your previous benchmark run), and BBU caches at RAID controllers may also
 survive even power failure.
\end_layout

\end_inset

.
 An empty cache is always 
\begin_inset Quotes eld
\end_inset

cold
\begin_inset Quotes erd
\end_inset

.
 During operation, it will be filled with data.
 A cache is called 
\begin_inset Quotes eld
\end_inset

hot
\begin_inset Quotes erd
\end_inset

, if the 
\series bold
cache hit rate
\series default
 is 
\begin_inset Quotes eld
\end_inset

significantly high enough
\begin_inset Quotes erd
\end_inset

.
 What's that? All of these terms are rather vague and depend on the application.
 However, some caches may 
\emph on
never
\emph default
 reach 
\begin_inset Quotes eld
\end_inset

hot
\begin_inset Quotes erd
\end_inset

 state, for example when the cache design / architecture is 
\begin_inset Quotes eld
\end_inset

inefficient
\begin_inset Quotes erd
\end_inset

 (cf section 
\begin_inset CommandInset ref
LatexCommand ref
reference "A-cache-is"

\end_inset

).
 What then? The term 
\begin_inset Quotes eld
\end_inset

hot
\begin_inset Quotes erd
\end_inset

 is not the right one for describing that problem: when your cache never
 gets hots, your testing candidate will just fail your performance test,
 but your test as such will be valid: the result is just telling you that
 the cache is not tuned well enough for your application workload.
\end_layout

\begin_layout Standard
What is 
\begin_inset Quotes eld
\end_inset

steady state
\begin_inset Quotes erd
\end_inset

?
\end_layout

\begin_layout Standard
Intuitively, it just means that 
\begin_inset Quotes eld
\end_inset

nothing changes (fundamentally) any more
\begin_inset Quotes erd
\end_inset

.
 In practice, there is a simple rule of thumb: your benchmark should just
 run 
\series bold
long enough
\series default
 to get into steady state.
 In section 
\begin_inset CommandInset ref
LatexCommand ref
reference "A-cache-is"

\end_inset

, some theoretical methods are described how to compute the time 
\begin_inset Formula $\delta$
\end_inset

 until steady state is reached.
 In practice, just make a few experiments in order to determine steady state
 intuitively.
 Many people will get a good feeling for 
\begin_inset Quotes eld
\end_inset

steady state
\begin_inset Quotes erd
\end_inset

 after some experience.
\end_layout

\begin_layout Standard
Once you know when you have reached the 
\begin_inset Quotes eld
\end_inset

golden
\begin_inset Quotes erd
\end_inset

 steady state, you can do one of two things:
\end_layout

\begin_layout Enumerate
ignore all benchmark results from the time span before steady state has
 been reached.
\end_layout

\begin_layout Enumerate
let the whole benchmark run 
\emph on
at least
\emph default
 10x as long as the time to reach steady state.
 The longer, the better.
\end_layout

\begin_layout Subsubsection
Pitfalls from Cache Size
\end_layout

\begin_layout Standard
The following illustration explains a general problem of cache hierarchies.
 Our example demonstrates an ill-designed behaviour: the lower cache is
 
\emph on
smaller
\emph default
 than the upper one.
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename cache-toosmall.fig
	lyxscale 50
	width 75col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
Now assume that the LV has a size of several terabytes, and the workload
 is operating on a large part of that.
 On one hand, this is several orders of magnitude larger than the upper
 cache, but on the other hand typical application workloads will not access
 all sectors with the same uniform probability.
 In general, caches are only useful in two cases (non-exclusively):
\end_layout

\begin_layout Enumerate
the cache is strictly larger than the workload.
\end_layout

\begin_layout Enumerate
the workload contains unevenly distributed access frequencies.
\end_layout

\begin_layout Standard
Case 1 occurs only in special cases, such as in-memory databases (or workstation
 loads on large RAM machines).
 Case 2 appears more often.
\end_layout

\begin_layout Standard
Back to the above example: assume that only case 2 applies to the upper
 cache.
 Consequently, case 2 will also apply to the lower cache, because the lower
 cache is 
\emph on
strictly smaller
\emph default
 than the upper cache in this ill-designed example.
 Now assume we have an 
\emph on
inclusive
\emph default

\begin_inset Foot
status open

\begin_layout Plain Layout
In case of 
\emph on
exclusive
\emph default
 cache hierarchies, the whole picture can be 
\emph on
approximately(!)
\emph default
 replaced by a simplified one having only one cache level.
 The size of the new simplified cache is just the 
\emph on
sum
\emph default
 of the sizes of both original caches.
\end_layout

\end_inset

 cache hierarchy, and are using a standard cache eviction strategy like
 LRU.
 As a consequence from LRU (or any other strategy avoiding anomalies
\begin_inset Foot
status open

\begin_layout Plain Layout
Probably the best known anomaly is the famous FIFO anomaly, as explained
 in most text books about operating systems.
\end_layout

\end_inset

), any sector present in the lower cache will be also be present in the
 upper cache
\begin_inset Foot
status open

\begin_layout Plain Layout
This is just the 
\emph on
definition
\emph default
 of non-anomaly behaviour.
\end_layout

\end_inset

.
 As a further consequence, we see that the lower cache is superfluous: by
 removing it, the system could even become faster
\begin_inset Foot
status open

\begin_layout Plain Layout
Notice that there are exceptions.
 For example, internal memory caches present at hard disk drives are way
 too small to be able to contribute to classical hierarchical caching, but
 they can act as cylinder buffers for 
\emph on
local aggregation strategies
\emph default
 like readahead.
\end_layout

\end_inset

 due to less overhead.
\end_layout

\begin_layout Standard
What can we do about that?
\end_layout

\begin_layout Standard
Simply, just design your system in such a way that lower caches are always
 strictly larger than higher ones, by a factor of 
\begin_inset Formula $k$
\end_inset

.
 In order to be useful, 
\begin_inset Formula $k\geq2$
\end_inset

 should be used, but for really good performance 
\begin_inset Formula $k\geq10$
\end_inset

 should be selected.
\end_layout

\begin_layout Standard
Probably you already know this, and you think you don't violate it.
 However, it is possible you might violate it unwillingly.
 The standard case is a server from a data center, equipped with several
 gigabytes of RAM.
 Almost all of the main memory can be used by the buffer/page caches of
 the Linux kernel.
 Therefore, you already may have a rather large upper cache you didn't think
 about.
 As a consequence, caches at the block storage level (e.g.
 SSD caches etc) should be larger by an order of magnitude (in this case
 
\begin_inset Formula $\sim1$
\end_inset

TB or more).
 At the time of writing this paper, some commercial storage systems don't
 match this seemingly simple requirement.
\end_layout

\begin_layout Standard
There is another variant of this pitfall: records made by 
\family typewriter
blktrace
\family default
 are measuring the IO traffic 
\emph on
below
\emph default
 the buffer/page cache in most cases.
 Therefore, most (if not all) natural loads obtained by 
\family typewriter
blktrace
\family default
 contain effects
\begin_inset Foot
status open

\begin_layout Plain Layout
One of the more well-known effects of caches is called 
\emph on
cache inversion
\emph default
.
 At the time of writing this paper, Wikipedia didn't carry much about it.
 Consult a really good textbook or some research papers to learn more about
 it.
 
\end_layout

\end_inset

 of the caches of the original system.
 In some cases (e.g.
 published loads from the 
\family typewriter
blktrace
\family default
 project), you don't know the original RAM size.
 Even if you know, you often cannot tell how large the page cache 
\emph on
really
\emph default
 was at the time of recording: how much RAM was spent for other purposes
 like processes, how much for other filesystems / partitions?
\end_layout

\begin_layout Standard
Even if you knew all that: do you know the 
\emph on
workingset size
\emph default
 of your application workload? Read on\SpecialChar \ldots{}

\end_layout

\begin_layout Subsection
Pitfalls from Workingset Sizes
\begin_inset CommandInset label
LatexCommand label
name "sub:Pitfalls-from-Workingset"

\end_inset


\end_layout

\begin_layout Standard
The workingset theory has been developped by Denning in the late 1960s,
 and has been originally used for the description of the behaviour of hardware-M
MU-based paging / swapping systems and their strategies like LRU.
 It is also useful in other areas, such as block storage.
 Here is an adaptation of Denning's theory to our needs:
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
WS(t,\delta)=\big\{\text{{set\,\ of\,\ all\,\ sectors\,\ touched\,\ in\,\ the\,\ time\,\ interval\,[t-\delta,t]}}\big\}
\]

\end_inset


\end_layout

\begin_layout Standard
\noindent
Usually, 
\begin_inset Formula $\delta$
\end_inset

 is treated as a constant, called 
\emph on
window size
\emph default
.
 Then the workingset 
\begin_inset Formula $WS(t,\delta)$
\end_inset

 at some point in time 
\begin_inset Formula $t$
\end_inset

 is simply the set of sectors occurring in a 
\family typewriter
blktrace
\family default
 during a time window of 
\begin_inset Formula $\delta$
\end_inset

 seconds 
\emph on
before
\emph default

\begin_inset Foot
status open

\begin_layout Plain Layout
Notice: Dennings original theory used the time interval 
\begin_inset Formula $[t,t+\delta].$
\end_inset

We find our definition more handy for practical purposes, because we need
 no 
\begin_inset Quotes eld
\end_inset

lookahead
\begin_inset Quotes erd
\end_inset

 into the 
\begin_inset Quotes eld
\end_inset

future
\begin_inset Quotes erd
\end_inset

.
\end_layout

\end_inset

 the interesting point in time 
\begin_inset Formula $t$
\end_inset

.
 Notice that we have a 
\emph on
set
\emph default
 here: it makes no difference how often a particular sector occurs during
 the time window 
\begin_inset Formula $\delta,$
\end_inset

 it just matters 
\emph on
whether
\emph default
 that sector appears or not.
 Also, it makes no difference whether a particular sector is read or written
 (or both).
 Thus the workingset model is a 
\emph on
reduction
\emph default
 of the reality to a handy theoretical model, but a model which is known
 to 
\emph on
preserve
\emph default
 some relevant and very interesting properties of the reality.
\end_layout

\begin_layout Description
Example: for any two window sizes 
\begin_inset Formula $\delta_{1}<\delta_{2},$
\end_inset

 the condition 
\begin_inset Formula $WS(t,\delta_{1})\leq WS(t,\delta_{2})$
\end_inset

 holds at any point in time 
\begin_inset Formula $t$
\end_inset

.
 In other words: increasing the window size 
\begin_inset Formula $\delta$
\end_inset

 will never make the workingset smaller; the workingset can only 
\emph on
grow
\emph default
 if the window becomes larger.
\end_layout

\begin_layout Standard
The efficiency of block storage caches can be predicted by the workingset
 theory in a rather easy and intuitive way.
 We assume that accesses to the cache are 
\emph on
much faster
\emph default
 than accesses to the background storage, such that we can 
\emph on
neglect
\emph default
 the access times to the cache when compared to accesses to the background
 storage.
 Then we can model the following interesting property:
\end_layout

\begin_layout Quote
A cache is called to be designed 
\series bold
efficiently
\series default

\begin_inset CommandInset label
LatexCommand label
name "A-cache-is"

\end_inset

, iff at any point in time 
\begin_inset Formula $t$
\end_inset

 the following condition holds:
\begin_inset Newline newline
\end_inset

 
\begin_inset Formula $|WS(t,\delta)|\leq|\text{{cachesize}}|$
\end_inset

 for some 
\begin_inset Formula $\delta\geq(\text{{time\,\ to\,\ fill\,\ the\,\ cache\,\ once}})$
\end_inset

.
\end_layout

\begin_layout Standard
The potential painpoint is easy to see: just take 
\begin_inset Formula $\delta$
\end_inset

 as the time to fill the cache from the background storage, which is 
\begin_inset Formula $(\text{{accesstime\,\ to\,\ storage}})\star(\text{{cachesize}})$
\end_inset

.
 If there would exist a point in time 
\begin_inset Formula $t$
\end_inset

 where the workingset 
\begin_inset Formula $WS(t,\delta)$
\end_inset

 would be 
\emph on
larger
\emph default
 than could be transferred to/from the storage during that 
\emph on
same
\emph default
 time window 
\begin_inset Formula $\delta$
\end_inset

, the data transfers could become a 
\emph on
bottleneck
\emph default
 of the system.
 Vice versa: if the workingset during window 
\begin_inset Formula $\delta$
\end_inset

 would always 
\emph on
fit
\emph default
 into the cache, the cache will usually
\begin_inset Foot
status open

\begin_layout Plain Layout
Of course, this holds only if the workload contains some 
\emph on
repetitions
\emph default
 during the window 
\begin_inset Formula $\delta$
\end_inset

, and if the cache employs some 
\begin_inset Quotes eld
\end_inset

good
\begin_inset Quotes erd
\end_inset

 replacement strategy like LRU.
 The latter assures that 
\begin_inset Quotes eld
\end_inset

hot
\begin_inset Quotes erd
\end_inset

 pages from the workingset will be kept in the cache.
 In addition, keep in mind that neglecting the access times to the cache
 could be an 
\emph on
oversimplification
\emph default
 in many cases.
\end_layout

\end_inset

 speed up things.
\end_layout

\begin_layout Standard
Consequences:
\end_layout

\begin_layout Itemize
The 
\series bold
workingset behaviour of the application
\series default
 is 
\emph on
crucial
\emph default
 for any storage system.
 
\end_layout

\begin_layout Itemize
The above argumentation contains an 
\emph on
oversimplification
\emph default
: of course, accesses to the cache are not 
\begin_inset Quotes eld
\end_inset

indefinitly fast
\begin_inset Quotes erd
\end_inset

 as our above neglect assumed.
 Therefore, don't take the above inequalities as verbatim inequalities.
 Multiply some 
\emph on
factor
\emph default
 onto them.
 In practice, if you really want blastingly fast caches, make sure your
 cache is at least 
\series bold
one order of magnitude larger
\series default
 than the workingset size of your application.
\end_layout

\begin_layout Standard
In order to do that, you would need the know the workingset size of your
 
\emph on
application
\emph default
.
 Please keep in mind that this is not the same as the workingset size measured
 by 
\family typewriter
blktrace
\family default
:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename hierarchy.fig
	lyxscale 60
	width 30col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
As you can see, the measuring point of 
\family typewriter
blktrace
\family default
 sits 
\emph on
below
\emph default
 the buffer / page cache, therefore it does not 
\emph on
directly
\emph default
 measure the application behaviour (in addition to influences from the filesyste
m, like metadata updates etc).
 In practice, 
\family typewriter
blktrace
\family default
 measurements may differ from the 
\begin_inset Quotes eld
\end_inset

real
\begin_inset Quotes erd
\end_inset

 application workload of stateless webserver designs by 
\emph on
several
\emph default
 orders of magnitude.
 However, 
\emph on
all
\emph default
 modern OSes employ caches.
 Even if we were able to measure the true application workingset in some
 way, it would not be relevant for block storage systems, because it would
 be 
\emph on
impractical
\emph default
 to operate those systems without caches.
 We need the above picture for understanding the fundamental properties
 of 
\family typewriter
blktrace
\family default
 measurements, and for determining the window size 
\begin_inset Formula $\delta$
\end_inset

.
 If that is not possible, try to estimate it.
 As a very rough estimation, take 
\begin_inset Formula $\delta$
\end_inset

 as several minutes.
\end_layout

\begin_layout Standard
The 
\family typewriter
blkreplay
\family default
 suite comes with a small tool to visualize the workingset behaviour as
 measured by 
\family typewriter
blktrace
\family default
, which is currently the best approximation of the workingset behaviour
 of the application we can easily get access to.
 Following is an example, showing different window sizes 
\begin_inset Formula $\delta$
\end_inset

 in the same picture (where 000 means cumulation to 
\begin_inset Formula $\infty$
\end_inset

):
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/example1.g30.ws_log.png
	lyxscale 30
	width 100col%

\end_inset


\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Spend some time on it! Your replay needs to last 
\emph on
vastly longer
\emph default
 than the 
\begin_inset Formula $\delta_{\text{{old}}}$
\end_inset

 needed to descibe the steady state of the original buffer/page cache, as
 well as the 
\begin_inset Formula $\delta_{\text{{new}}}$
\end_inset

 to describe the steady state of your replay system.
 Otherwise, you can get fake results which differ from real practical performanc
e by factors, or even orders of magnitude.
 As a rough rule of thumb, any replay of natural loads should take at least
 one hour.
 Better, make a few measurements lasting 8 hours, or even 24 hours, and
 check whether the results differ more than expected (besides natural variations
).
\end_layout

\begin_layout Subsection
Pitfalls from Replay Device Sizes and Others
\begin_inset CommandInset label
LatexCommand label
name "sub:Pitfalls-from-Replay"

\end_inset


\end_layout

\begin_layout Standard
There is a simple intuitive rule: your replay device for 
\family typewriter
blkreplay
\family default
 should have (almost) the 
\emph on
same
\emph default
 size as the original device where 
\family typewriter
blktrace
\family default
 has taken its measurement from
\begin_inset Foot
status open

\begin_layout Plain Layout
If you don't know this, just make a test run with 
\family typewriter
blkreplay
\family default
 and check the output file.
 At the end, you will find a human-readable statistics showing the highest
 original block number occurring in the replay, as well as some other interestin
g numbers.
\end_layout

\end_inset

.
\end_layout

\begin_layout Standard
For easy checking, our output graphics will display the so-called 
\series bold
wraparound_factor
\series default
 in the lower right corner: it shows the ratio between the max block# occuring
 in the load, and the size of the replay device.
 Ideally, it should be near 
\begin_inset Formula $1.0$
\end_inset

.
 When it deviates from this by more than a factor of two (or less than one
 halve), it is printed in purple color to warn you.
\end_layout

\begin_layout Standard
Some people don't take this seriously, and some don't even believe that
 this can have a 
\emph on
tremendous
\emph default
 effect.
\end_layout

\begin_layout Standard
Practical experience at 1&1 tells that the above rule is 
\emph on
valid
\emph default
, and that results 
\emph on
will
\emph default
 almost certainly vary.
 The bias can be 
\emph on
considerable
\emph default
.
\end_layout

\begin_layout Standard
Example: the original 
\family typewriter
blktrace
\family default
 recording was taken from a production server equipped with 20 TB RAID.
 Since in the lab we had only a smaller system with 4 TB, 
\family typewriter
blktrace
\family default
 measurements were run despite the smaller size.
 Whenever 
\family typewriter
blkreplay
\family default
 tries to start an IO request outside the LV size, it just 
\emph on
remaps
\emph default
 the sector number modulo the (new) LV size.
 Therefore, results 
\emph on
appear
\emph default
 to be valid, since you cannot see any 
\begin_inset Quotes eld
\end_inset

big
\begin_inset Quotes erd
\end_inset

 holes or anomalies.
 You will find out the difference only if you compare to the 
\emph on
correct
\emph default
 setup.
 When repeating the 
\emph on
same
\emph default
 measurements with correct LV sizes of ~20 TB, there is a 
\emph on
significant
\emph default
 difference.
\end_layout

\begin_layout Standard
Some people think that such differences can be attributed solely to the
 natural differences in 
\emph on
spindle count
\emph default

\begin_inset Foot
status open

\begin_layout Plain Layout
When using the same hard disk model, a 20 TB RAID must contain 5 times as
 much spindles as an equivalent 4 TB system.
 It is clear that additional spindles can benefit random IO.
 But those effects are 
\series bold
non-linear
\series default
!
\end_layout

\end_inset

, and therefore it would not hurt if different models of hard disks were
 used.
 Although there 
\emph on
are
\emph default
 some effects by spindle count, that opinion is wrong.
 In order to disprove such a 
\begin_inset Quotes eld
\end_inset

theory
\begin_inset Quotes erd
\end_inset

, just build two different RAIDs with same spindle count, but fundamentally
\begin_inset Foot
status open

\begin_layout Plain Layout
Of course, there are a lot of disk drives (in the same 
\begin_inset Quotes eld
\end_inset

class
\begin_inset Quotes erd
\end_inset

) showing only minor differences.
 A 
\begin_inset Quotes eld
\end_inset

fundamental
\begin_inset Quotes erd
\end_inset

 difference is for example between a cheap SATA disk versus a smaller but
 faster 15k SAS disk.
 
\end_layout

\end_inset

 different disk drive models (resulting in different total capacity), and
 compare (under otherwise equal conditions).
\end_layout

\begin_layout Standard
In short, any of the following factors can influence the performance, independen
tly from each other (and in no particular order):
\end_layout

\begin_layout Itemize
Total capacity, just by itself.
 If you don't believe it, just create LVs with 1/10 size of the physical
 storage and compare (on the 
\emph on
same
\emph default
 hardware) with results from the 
\emph on
full
\emph default
 physical storage size.
 Of course, the original recording must stem from sufficiently large devices,
 otherwise your 
\begin_inset Quotes eld
\end_inset

disproof
\begin_inset Quotes erd
\end_inset

 will be false.
\end_layout

\begin_layout Itemize
Model/class of disk drives.
\end_layout

\begin_layout Itemize
Number of spindles.
\end_layout

\begin_layout Itemize
RAID level.
\end_layout

\begin_layout Itemize
Vendor / firmware version of the controller.
\end_layout

\begin_layout Itemize
Interconnect technology, such as SATA vs SAS.
\end_layout

\begin_layout Itemize
any caches in the hierarchy, such as different BBU cache sizes or different
 parameters.
\end_layout

\begin_layout Standard
...
 and probably many others.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 General rule: 
\emph on
never
\emph default
 expect any of these effects to be linear.
 Almost always, they are 
\series bold
non-linear
\series default

\begin_inset Foot
status open

\begin_layout Plain Layout
Non-linear effects cannot be combined with each other in a predictable way,
 at least in general.
\end_layout

\end_inset

.
 Anyone claiming something else must 
\series bold
prove
\series default
 it!
\end_layout

\begin_layout Standard
Consequence: anyone violating the above rule produces 
\series bold
invalid
\series default
 results, unless proven the opposite!
\end_layout

\begin_layout Subsection
Pitfalls from Parallelism in IO Systems
\end_layout

\begin_layout Subsubsection
Pitfalls from 
\emph on
missing
\emph default
 Parallelism
\end_layout

\begin_layout Standard
This pitfall is usually trapping less people, because some intuitive sense
 for the effects of IO parallelism is more widespread.
 Even if you are already aware of this pitfall, read on.
 There are some subtleties.
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename consolidation.fig
	lyxscale 50
	width 30text%

\end_inset


\end_layout

\begin_layout Standard
\noindent
A frequent use case is 
\emph on
storage consolidation
\emph default
.
 Up to 
\begin_inset Formula $k$
\end_inset

 CPU nodes are connected to some 
\begin_inset Quotes eld
\end_inset

central
\begin_inset Quotes erd
\end_inset

 storage via some kind of storage network.
\end_layout

\begin_layout Standard
You may want to evaluate such an architecture in advance with help of 
\family typewriter
blkreplay
\family default
, in order to avoid failed invests.
 The 
\family typewriter
blkreplay
\family default
 suite will help you, because its supervisor scripts like 
\family typewriter
tree-replay.sh
\family default
 are ready to run masses of 
\family typewriter
blkreplay
\family default
 instances in parallel to each other, and on different nodes.
\end_layout

\begin_layout Standard
For obvious reasons, you should determine the optimum consolidation factor
 
\begin_inset Formula $k$
\end_inset

 for each given hardware candidate.
 If you vary the factor 
\begin_inset Formula $k$
\end_inset

, you will get at least two effects:
\end_layout

\begin_layout Enumerate
With higher 
\begin_inset Formula $k$
\end_inset

, you need to build up more CPU nodes and more network connections.
 Otherwise you will neglect effects from IO parallelism (which don't scale
 linearly
\begin_inset Foot
status open

\begin_layout Plain Layout
There is no general way to predict the behaviour of an unknown system! Many
 non-linear effects show some kind of 
\begin_inset Quotes eld
\end_inset

binary
\begin_inset Quotes erd
\end_inset

 behaviour, suddenly collapsing at some point where you didn't expect it.
\end_layout

\end_inset

), at multiple places of the picture: at potential bottlenecks at the CPU
 nodes, at potential bottlenecks inside your storage network, and at lots
 of potential bottlenecks in your central storage system.
\begin_inset Newline newline
\end_inset

For example, it is completely wrong to just double the load on 
\begin_inset Formula $k/2$
\end_inset

 nodes.
 Experiences with some test candidates at 1&1 show that results can differ
 enormously from results from single loads, each on 
\begin_inset Formula $k$
\end_inset

 nodes.
\end_layout

\begin_layout Enumerate
The total capacity of the central storage will also vary with 
\begin_inset Formula $k$
\end_inset

.
 As known from section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Replay"

\end_inset

, this will not only influence results by itself, but in addition by further
 effects like spindle count, non-scaling of caches with factor 
\begin_inset Formula $k$
\end_inset

, and many others.
\end_layout

\begin_layout Subsubsection
Pitfalls from 
\emph on
too high
\emph default
 IO Parallelism
\end_layout

\begin_layout Standard
You can tune the number of 
\family typewriter
blkreplay
\family default
 threads via the parameter 
\family typewriter
threads=\SpecialChar \ldots{}

\family default
 (see appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:Config-File-Parameters"

\end_inset

).
 As a side effect, this will also influence the maximum number of outstanding
 IO requests which can be 
\begin_inset Quotes eld
\end_inset

on the fly
\begin_inset Quotes erd
\end_inset

 in parallel, together with the replay parallelism (number of 
\family typewriter
blkreplay
\family default
 instances running in parallel on the same physical system).
\end_layout

\begin_layout Standard
Many people believe that an increasing number of outstanding IO requests
 will improve overall throughput.
\end_layout

\begin_layout Standard
However, some devices / drivers / IO schedulers may respond in some 
\emph on
counter-productive
\emph default
 way when hammered with too many IO requests in parallel.
 Sometimes, your throughput can even 
\emph on
collapse
\emph default
 to less than 1/10 of the maximum.
 There are many possible reasons for this unexpected behaviour.
\end_layout

\begin_layout Standard
In most cases, you will notice collapsing effects only during 
\series bold
overload situations
\series default
.
 During normal operation, you will not notice anything, because the device
 can 
\emph on
catch up
\emph default
 with the IO demands.
 There are almost no queues, because the 
\emph on
average service time
\emph default
 is shorter than the average request rate.
\end_layout

\begin_layout Standard
As soon as the device gets overloaded, the situation will change rapidly.
 Masses of 
\family typewriter
blkreplay
\family default
 threads are trying to fire off their requests in time, starting to overlap
 now because the service time increases.
 Suddenly, some queuing will take place, somewhere.
 In mathematical theory, the queue lengths could even grow indefinitely
 whenever the average request rate is higher than the average service time.
 In practice, there will be some limits somewhere.
\end_layout

\begin_layout Standard
Imagine you are going shopping in a supermarket.
 When the cashier girl cannot catch up with the demand from the customers,
 their shopping trolleys will start to form a queue in front of her.
 Now assume the following behaviour: the longer the queue, the 
\emph on
slower
\emph default
 she will work.
 Imagine that! After a while, customer satisfaction will go down to zero,
 because the queue will get longer and longer.
 And the longer, the slower she will work.
 And so on.
 There is no escape, other than stopping to buy anything from there.
\end_layout

\begin_layout Standard
A similar behaviour can be observed in some IO systems.
 Once a queue has formed, there is almost no escape from a behaviour similar
 to 
\series bold
traffic jam
\series default
.
 If you cannot change the system, your only chance is to reduce load, or
 even to remove the load at all.
\end_layout

\begin_layout Standard
Sometimes it is even hard to trigger such behaviour.
 A systems appears to run smoothly for months, but suddenly it collapses.
\end_layout

\begin_layout Standard
If you have such a system (or cannot be sure to have one
\begin_inset Foot
status open

\begin_layout Plain Layout
To find out, we recommend the artifical loads 
\family typewriter
*bursts*.load.gz
\family default
 for creating anal kinds of overload.
\end_layout

\end_inset

) and want to simulate its behaviour in the laboratory, you should be carefully
 tuning the 
\family typewriter
threads
\family default
 parameter.
 In reality, the number of outstanding requests is often limited in some
 way.
 For example, a typical workstation load caused by a single user has often
 some intrinsically limited IO parallelism, similar to a limited number
 of shopping trolleys in the supermarket.
 On the other hand, some server loads (such as those caused by Apache) can
 
\family typewriter
fork()
\family default
 off a high number of threads.
 
\end_layout

\begin_layout Standard
So it is quite possible to observe some traffic jam behaviour in practice,
 depending on your application.
 It is quite possible that such behaviour is 
\series bold
relevant
\series default
 for IT operations, at least for 
\emph on
some
\emph default
 applications (but not for all).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Taking any of these 
\series bold
non-linear
\series default
 (and sometimes even 
\emph on
binary collapsing
\emph default
) effects not seriously and/or using a wrong number of threads / replay
 parallelism can easily lead to 
\series bold
invalid results
\series default
, and in turn to failed invests!
\end_layout

\begin_layout Standard
Hint: a frequent case are distributed systems by 
\emph on
themselves
\emph default
.
 They tend to produce 
\series bold
queues
\series default
 at times and in places where you don't expect them, and they tend to produce
 avalanche-like negative effects (self-amplifying), similar to suddenly
 appearing 
\series bold
traffic jams
\series default
 where you cannot determine one single reason in isolation.
\end_layout

\begin_layout Section
Recommended Setup and Usage
\end_layout

\begin_layout Subsection
Planning Phase
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Never try to plan a project without deep knowledge of the pitfalls described
 in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:How-to-Avoid"

\end_inset

.
 In addition, some experience with 
\family typewriter
blkreplay
\family default
 is helpful.
 In order to gain such experience, consider a 
\emph on
test project
\emph default
 just for playing around, and for getting familiar with the pitfalls.
\end_layout

\begin_layout Subsubsection
Describe the Scope of Project
\end_layout

\begin_layout Standard
Before starting, you should get conscious with yourself.
 
\emph on
What exactly
\emph default
 is the question you want to answer with help of 
\family typewriter
blkreplay
\family default
?
\end_layout

\begin_layout Standard
Write down the question both as shortly, as well as precisely as possible.
 Here are some examples:
\end_layout

\begin_layout Itemize
Compare hardware vendor 
\family typewriter
A
\family default
 with 
\family typewriter
B
\family default
 and 
\family typewriter
C
\family default
 for my production workload X.
\end_layout

\begin_layout Itemize
Compare hardware vendor 
\family typewriter
A
\family default
 with 
\family typewriter
B
\family default
 and 
\family typewriter
C
\family default
 in general.
\end_layout

\begin_layout Itemize
Debug kernel module 
\family typewriter
xxx
\family default
.
\end_layout

\begin_layout Itemize
Compare iSCSI with Fibrechannel for my production workload X.
\end_layout

\begin_layout Itemize
\SpecialChar \ldots{}

\end_layout

\begin_layout Standard
Next, write down a precise description of your intended 
\series bold
test environment
\series default
.
 Best practice is to name hardware vendors, models, all components (including
 intermediate gear like network switches), and so on.
\end_layout

\begin_layout Standard
Last step: describe all the parameters which know of, which 
\emph on
could
\emph default
 have an influence onto your test results.
 Example:
\end_layout

\begin_layout Standard
\noindent
\align center

\size small
\begin_inset Tabular
<lyxtabular version="3" rows="7" columns="2">
<features tabularvalignment="middle">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top" width="0">
<row>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size small
Parameter
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size small
Varying?
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size small
16 GB RAM in storage node
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size small
no
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size small
10GBit vs 1 GBit network speed
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size small
yes
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size small
Cheap 2TB SATA vs expensive 600GB raptor disk (model 
\begin_inset Quotes eld
\end_inset

tyrannosaurus rex
\begin_inset Quotes erd
\end_inset

)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size small
yes
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
RAID-Level 
\begin_inset Formula $\in\{1,5,6\}$
\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
RAID Stripesize 
\begin_inset Formula $\in\{16,32,64,128\}$
\end_inset

kB
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
yes
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size small
\SpecialChar \ldots{}

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size small
\SpecialChar \ldots{}

\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Subsubsection
Describe the Setup of your Experiment
\end_layout

\begin_layout Standard
In many cases, the parameters described in your table will make up a multi-dimen
sional problem space (cartesian product) which is too large to be explored
 exhaustively.
 As explained in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Replay"

\end_inset

, many of them will influence your results 
\emph on
non-linearly
\emph default
.
\end_layout

\begin_layout Standard
Thus, you will have to consider the following general strategies:
\end_layout

\begin_layout Itemize
You may fix some of the parameters to particular values.
 Although this saves time, you may miss an opportunity to find an optimal
 solution.
\end_layout

\begin_layout Itemize
You may select some/enough random samples from your multi-dimensional problem
 space and try them randomly (Monte Carlo methods).
\end_layout

\begin_layout Itemize
Stepwise refinement: explore the multidimensional space by varying exactly
 
\emph on
one
\emph default
 paramenter at once.
 This is slow, but you can be sure of the effects caused by this.
\end_layout

\begin_layout Subsubsection
Select 
\family typewriter
blkreplay
\family default
 Load
\begin_inset CommandInset label
LatexCommand label
name "sub:Select-blkreplay-Load"

\end_inset


\end_layout

\begin_layout Standard
Depending on your project, you should consider both artificial and natural
 loads, but not too many of them.
 Usually, more than three loads are impractical for an ambitious project
 (unless you want to compare masses of loads on the same reference hardware).
\end_layout

\begin_layout Standard
If your project tries to answer a question for some specific workload X,
 you should just record that workload if you can do so (see chapter 
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:How-to-use"

\end_inset

).
\end_layout

\begin_layout Standard
If you cannot obtain your real workload in advance, you have to select one
 from the 
\family typewriter
blkreplay
\family default
 project (or other sources) which comes as close as possible to your (intended)
 natural workload.
\end_layout

\begin_layout Standard
For the sake of static comparison of workloads, 
\family typewriter
cd
\family default
 to a directory containing your 
\family typewriter
*.load.gz
\family default
 files and issue the following command:
\end_layout

\begin_layout Quotation

\family typewriter
/path/to/graph.sh --static myname.load.gz
\end_layout

\begin_layout Standard
This will produce some 
\family typewriter
.png
\family default
 graphics, describing the throughput and workingset behaviour of your load
 (cf section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Workingset"

\end_inset

).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 You need enough space in 
\family typewriter
/tmp/
\family default
 (or in another 
\family typewriter
$TMPDIR
\family default
) for temporary intermediate files.
 If your 
\family typewriter
*.load.gz
\family default
 file is very large (several hours or even days), you may need several gigabytes.
 Please don't interrupt 
\family typewriter
graph.sh
\family default
 as it spawns lots of subprocesses and creates lots of temporary files.
 Currently, there are no checks for free space in 
\family typewriter
/tmp/
\family default
, so running out of space may produce wrong results 
\emph on
silently
\emph default
.
 As a countermeasure, run 
\family typewriter
watch df /tmp/
\family default
 in a separate window during your run of 
\family typewriter
graph.sh
\family default
.
\end_layout

\begin_layout Standard
Hint: some 
\family typewriter
blktrace
\family default
 recordings (but not all) contain some timing information about the original
 IO latencies as measured at the original site.
 Use 
\family typewriter
/path/to/graph.sh --dynamic myname.load.gz
\family default
 to create some additional graphics about them.
\end_layout

\begin_layout Subsubsection
Selection of Replay Duration
\end_layout

\begin_layout Standard
This is a hairy problem, as already described in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Workingset"

\end_inset

.
 Often, you cannot run 24h replays for many hundred times.
\end_layout

\begin_layout Standard
If you want to be sure that a particular load will run even under 
\emph on
worst-case
\emph default
 conditions, you should definitly select some appropriate time window around
 
\emph on
load peaks
\emph default
, measured both in throughput as well as in workingset size.
\end_layout

\begin_layout Standard
In addition, you can try the following strategies:
\end_layout

\begin_layout Itemize
For explorative phases in your project, such as determining the optimum
 in your parameter space, you can try to minimize running times as much
 as possible.
 But not too much.
 Otherwise you will be caught by the pitfalls described in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:How-to-Avoid"

\end_inset

.
\end_layout

\begin_layout Itemize
For final verification of your results, you should repeat benchmarks with
 a longer window (at least 8h or 24h).
\end_layout

\begin_layout Subsubsection
Total Project Time
\end_layout

\begin_layout Standard
Working with huge parameter spaces is not all you have to consider.
 Setup of different RAID levels, re-initialization after changes of stripe
 sizes, filling LVs with random data, etc, may take a very long time, in
 addition to the benchmark themselves.
 Don't forget that! Your only chance are nights and weekends, if you manage
 to run something unattendedly.
 But predictions are sometimes wrong.
 In addition, something may fail and then needs to be restarted.
 Calculate some spare time for that!
\end_layout

\begin_layout Standard
If there is a high time pressure in the project timeline, you probably will
 have to 
\emph on
rework
\emph default
 some parts of your project plan.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Planning is crucial! When you find any discrepancies, try to re-think your
 plan as a whole, not just some parts of it.
\end_layout

\begin_layout Subsection
Setup Phase
\end_layout

\begin_layout Subsubsection
Lab setup
\end_layout

\begin_layout Standard
Ensure that all your hard- and software components are ready in the lab
 and operational.
\end_layout

\begin_layout Standard
In addition, you need some workstation (or server) where the 
\family typewriter
blkreplay
\family default
 suite is checked out.
 Do the following steps:
\end_layout

\begin_layout Itemize
Ensure that 
\family typewriter
gcc
\family default
, 
\family typewriter
make
\family default
, 
\family typewriter
gnuplot
\family default
, and some standard tools like 
\family typewriter
grep
\family default
 / 
\family typewriter
gawk
\family default
 are installed.
 If you need multiple machine architectures (such as 
\family typewriter
x86_64
\family default
 and 
\family typewriter
i386
\family default
) in parallel, ensure that 
\family typewriter
gcc
\family default
 can cross-compile via flags 
\family typewriter
-m32
\family default
 / 
\family typewriter
-m64
\family default
 and that the appropriate libs are installed.
\end_layout

\begin_layout Itemize

\family typewriter
git clone https://github.com/schoebel/blkreplay
\end_layout

\begin_layout Itemize

\family typewriter
cd blkreplay
\end_layout

\begin_layout Itemize

\family typewriter
./configure
\end_layout

\begin_layout Itemize

\family typewriter
make
\end_layout

\begin_layout Itemize
Attention! 
\family typewriter
make install
\family default
 is not yet supported.
 Just leave everything in place.
 You can either put 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/path/to/blkreplay/scripts/
\end_layout

\end_inset


\family default
 into your 
\family typewriter
$PATH
\family default
, or call the scripts via hard path.
\end_layout

\begin_layout Standard
Ensure that all your test machines are reachable as 
\family typewriter
root
\family default
 via 
\family typewriter
ssh
\family default
 from your central workplace, without need for any password prompt.
 In order to achieve that, you should consider 
\family typewriter
ssh-agent
\family default
, in addition so some tweaking of 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

/etc/ssh/ssh_config
\end_layout

\end_inset

 (and probably 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

/etc/ssh/sshd_config
\end_layout

\end_inset

 on each of the target hosts).
\end_layout

\begin_layout Standard
On your workstation, you should have enough disk space to store your results.
 Create a subdirectory there for your project.
 Copy 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

/path/to/blkreplay/example-run/default-main.conf
\end_layout

\end_inset

 (and possibly other 
\family typewriter
*.conf
\family default
 files) to that new subdirectory, and finally 
\family typewriter
cd
\family default
 to it.
 For the rest of your life, you will be working there ;)
\end_layout

\begin_layout Standard
You can now either call 
\family typewriter
/path/to/blkreplay/scripts/
\emph on
something
\emph default
.sh
\family default
 as hard paths as indicated in the following examples, or you may put 
\family typewriter
/path/to/blkreplay/scripts/
\family default
 into your 
\family typewriter
$PATH
\family default
.
\end_layout

\begin_layout Standard
Customization of 
\family typewriter
default-main.conf
\family default
 is described in the following.
\end_layout

\begin_layout Subsubsection
Configuration Files
\end_layout

\begin_layout Standard
You should edit 
\family typewriter
default-main.conf
\family default
 to reflect the default setup for your project.
 If you want to run multiple variants of your default setup, you can do
 so by creating additional files like 
\family typewriter
\emph on
something
\emph default
.conf
\family default
 as well as a subdirectory 
\family typewriter
\emph on
something
\emph default
/
\family default
 (having the same name without suffix 
\family typewriter
.conf
\family default
).
 When you later start your benchmark, the values from 
\family typewriter
\emph on
something
\emph default
.conf
\family default
 will override those from 
\family typewriter
default-main.conf
\family default
.
 It is highly recommended to override only 
\emph on
one
\emph default
 parameter inside 
\family typewriter
\emph on
something
\emph default
.conf
\family default
, otherwise it may become difficult to reveal the real impact of changed
 parameters onto your test candidate.
\end_layout

\begin_layout Standard
In general, you may override 
\emph on
any
\emph default
 parameter from 
\family typewriter
default-main.conf
\family default
, even hostnames, or input files 
\family typewriter
*.load.gz
\family default
, or whatever.
\end_layout

\begin_layout Subsubsection
Meaning of the Config File Parameters
\end_layout

\begin_layout Standard
The meaning of the parameters is documented in the following places:
\end_layout

\begin_layout Enumerate
Comments inside 
\family typewriter
default-main.conf
\family default
 should provide enough information for experienced administrators, at least
 for a quick start, and should guide you through the most basic steps.
\end_layout

\begin_layout Enumerate
The same information is available in appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:Config-File-Parameters"

\end_inset

.
\end_layout

\begin_layout Enumerate
Last but not least: read the sources, if you are in doubt about anything.
\end_layout

\begin_layout Subsection
Benchmark Phase
\end_layout

\begin_layout Standard
The basic idea is simple: after customization of 
\family typewriter
default-main.conf
\family default
 (and probably other 
\family typewriter
default-*.conf
\family default
 files when using additional modules), you create a new subdirectory for
 each benchmark run.
\end_layout

\begin_layout Standard
Whenever you call 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/path/to/blkreplay/scripts/tree-replay.sh
\end_layout

\end_inset


\family default
 (without parameters), a whole bundle of benchmarks will be started, one
 for each 
\emph on
leaf
\emph default

\begin_inset Foot
status open

\begin_layout Plain Layout
A leaf has no further subtree inside it.
\end_layout

\end_inset

 subdirectory (starting from 
\family typewriter
cwd
\family default
), provided that for each (intermedidate) subdirectory name 
\family typewriter
\emph on
xxx
\family default
\emph default
 there exists some 
\family typewriter
\emph on
xxx
\emph default
.conf
\family default
 in the current working directory or in one of its parents.
 In the whole subdirectory structure, any directory ending with 
\family typewriter
.old
\family default
, or including the substring 
\family typewriter
ignore
\family default
, or containg a file 
\family typewriter
skip
\family default
 will be ignored.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
Example: you have created a subdirectory 
\family typewriter
./short/
\family default
 as well as two nested subdirectories 
\family typewriter
./short/model1/
\family default
 and 
\family typewriter
./short/model2/
\family default
.
 You further have prepared the config files 
\family typewriter
short.conf
\family default
, 
\family typewriter
model1.conf
\family default
 and 
\family typewriter
model2.conf
\family default
, existing in the current working directory or in some parent directory
 down the 
\family typewriter
../
\family default
 chain.
 Then exactly 
\emph on
two
\emph default
 benchmark runs will be started, namely in 
\family typewriter
./short/model1/
\family default
 and in 
\family typewriter
./short/model2/
\family default
.
 The benchmark running in 
\family typewriter
./short/model1/
\family default
 will include the following 
\family typewriter
*.conf
\family default
 files, in the following order: 
\family typewriter
default-main.conf
\family default
, 
\family typewriter
short.conf
\family default
, and finally 
\family typewriter
model1.conf
\family default
.
 Each of the specialized config files may override any previous setting,
 but it is highly recommended to change only one parameter at a time and
 to use short but expressive names.
 Notice: the intermediate directory 
\family typewriter
./short/
\family default
 is no leaf (since it contains some subdirectories), therefore no benchmark
 will be started inside it.
 Later, you just need to create 
\family typewriter
./long/
\family default
 as well as 
\family typewriter
./long/model1/
\family default
 and 
\family typewriter
./long/model2/
\family default
 and some 
\family typewriter
long.conf
\family default
 in order to repeat the same benchmarks with a longer 
\family typewriter
replay_duration
\family default
 setting.
\end_layout

\begin_layout Standard
Hint: using 
\begin_inset Quotes eld
\end_inset

intuitive
\begin_inset Quotes erd
\end_inset

 names like 
\family typewriter
short
\family default
 and 
\family typewriter
long
\family default
 bears some danger.
 A few years later, you will not remember what they exactly have meant.
 Looking into 
\family typewriter
*.conf
\family default
 will not help other people, for example if you publish your benchmark results
 somewhere.
 Therefore it may be wise to use 
\begin_inset Quotes eld
\end_inset

speaking
\begin_inset Quotes erd
\end_inset

 names like 
\family typewriter
duration_600
\family default
, at least if you have more than two variants.
 On the other hand, 
\begin_inset Quotes eld
\end_inset

intuitive
\begin_inset Quotes erd
\end_inset

 names are better for presentation to some less-deeply involved audience.
 Take some time for creating well-designed names for 
\family typewriter
*.conf
\family default
 and your directory hierarchy! Changing that names later is cumbersome.
 Better to design your names in advance in a systematic (but simple) way.
\end_layout

\begin_layout Standard
On large investigation projects, deeply nested structures may be necessary,
 involving different loads, different hardware, different hardware setup,
 etc.
 Not all of them are currently automated.
 You can use the generic module mechanism to extend the default scripts
 with further functionality, to push automation further.
\end_layout

\begin_layout Standard
However, not all setup tasks 
\emph on
can
\emph default
 be automated at all.
 Some of them like forcing physical RAID degradation must be started by
 physically removing a disk, which cannot be automated (other than buying
 extremely expensive robots).
 Therefore, you may include some human-readable dialogs inside your 
\family typewriter
*.conf
\family default
 files (in shell script syntax), or in some new modules you have written.
 In any case, it is advisable to write some script code to 
\emph on
check some preconditions
\emph default
 (such as RAID status) in order to prevent wrong measurements.
\end_layout

\begin_layout Standard
In general, 
\family typewriter
tree-replay.sh
\family default
 will never repeat any benchmark which has already completed (i.e.
 there exists an output file 
\family typewriter
*.replay.gz
\family default
 in that leaf directory).
 This allows an incremental style of working.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
Continued
\begin_inset space ~
\end_inset

example: normally, you can call 
\family typewriter
tree-replay.sh
\family default
 again, but nothing happens, because all leaf directories already contain
 some results.
 However, then you find a problem with the results in 
\family typewriter
./short/model1/
\family default
, so you want to repeat that benchmark without deleting your first (questionable
) results.
 Now you create a subdirectory 
\family typewriter
./short/model1/try1.old/
\family default
 and move your results there.
 As said above, any directory names ending with 
\family typewriter
.old
\family default
 or containing the substring 
\family typewriter
ignore
\family default
 are ignored, so the directory 
\family typewriter
./short/model1/
\family default
 will continue to count as a leaf (despite its newer subdirectory, which
 is just ignored).
 Since the 
\family typewriter
*.replay.gz
\family default
 files are now missing in 
\family typewriter
./short/model1/
\family default
 due to the 
\family typewriter
mv
\family default
, 
\family typewriter
tree-replay.sh
\family default
 will repeat that single benchmark there.
\end_layout

\begin_layout Standard
Here are some useful hints:
\end_layout

\begin_layout Itemize
You may skip any directory by creating a file 
\family typewriter
skip
\family default
 inside it.
 
\family typewriter
touch ./short/model1/skip
\family default
 will disable that directory.
 Later, you can remove that file in order to fire off that benchmark.
\end_layout

\begin_layout Itemize

\family typewriter
skip
\family default
 files are also working in intermediate directories like 
\family typewriter
./short/
\family default
, disabling the whole subtree in one step.
\end_layout

\begin_layout Itemize
Design your 
\family typewriter
*.conf
\family default
 files such that arbitrary combinations are 
\emph on
possible
\emph default
 (cartesian product).
 In contrast, your directory hierarchy need not (and, in many cases, 
\emph on
will not
\emph default
) exploit the 
\emph on
full
\emph default
 cartesian product.
\end_layout

\begin_layout Itemize
You may create a new leaf directory (somewhere in the subtree) even in parallel
 to an already running benchmark.
 Whenever the currently running benchmark has completed, 
\family typewriter
tree-replay.sh
\family default
 will re-scan the subdirectory structure, find any freshly created leaf
 directories, and determine which benchmark to start next.
 All leaf directory names which have not yet completed are sorted alphabetically
, and the first name according to ASCII sort order is taken first.
\end_layout

\begin_layout Itemize
When 10 or more variants could appear somewhere (even after a while), use
 leading zeros in any names like 
\family typewriter
v001
\family default
, 
\family typewriter
v002
\family default
 etc to ensure that ASCII sort order is the same as numerical order.
\end_layout

\begin_layout Itemize
Sometimes the ASCII sort order of names like 
\family typewriter
short
\family default
 vs 
\family typewriter
long
\family default
 is boring, because you want to run the 
\family typewriter
short
\family default
 benchmarks first.
 As a workaround in larger projects, add some numerical prefixes like in
 
\family typewriter
01_short
\family default
 vs 
\family typewriter
09_long
\family default
 (leaving some numerical space such that you can later add 
\family typewriter
05_medium
\family default
).
 As a side effect, this also improves the ASCII sort order of your later
 
\family typewriter
*.png
\family default
 graphics.
\end_layout

\begin_layout Itemize
You can rearrange the order in another way: just create an empty file 
\family typewriter
prio-nnn
\family default
 in a leaf directory, where 
\family typewriter
nnn
\family default
 is a number denoting a priority class.
 Priority classes are overriding the global ASCII sort order.
 Directories having the lowest priority class are run first, while directories
 without any class are run last.
 Inside of each class, the ASCII sort order is obeyed.
\end_layout

\begin_layout Itemize
When you design your 
\family typewriter
*.conf
\family default
 files systematically as a cartesian product, in theory it makes no difference
 whether you permute some directory components (e.g 
\family typewriter
./model1/short/
\family default
 instead of 
\family typewriter
./short/model1/
\family default
).
 However, in practice it influences the ASCII sort order (taking the 
\emph on
full
\emph default
 path) and therefore the order in which your benchmarks are run.
\end_layout

\begin_layout Itemize
When some benchmark fails, just delete the corresponding output files.
 On the next cycle, 
\family typewriter
tree-replay.sh
\family default
 will detect the missing files and just restart that benchmark (possibly
 among others).
\end_layout

\begin_layout Itemize

\family typewriter
\emph on
xxx
\emph default
.conf
\family default
 files can even reside in some parent directory of the current working directory.
 This way, you need not copy your 
\family typewriter
.conf
\family default
 files inside a complex directory hierarchy (even spanning multiple projects).
 However, only the 
\family typewriter
\emph on
xxx
\emph default
.conf
\family default
 files corresponding to directories reachable from the current working directory
 will be included.
 Example: if you go to a leaf of your subtree and start 
\family typewriter
tree-replay.sh
\family default
 there, no 
\family typewriter
*.conf
\family default
 file other than 
\family typewriter
default-*.conf
\family default
 will be included.
 This may produce different results than expected.
 Make sure you start your replays always in the same base directory!
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 It is easy to misconfigure almost anything by accident.
 Check each step you make.
 In particular, run tools like 
\family typewriter
\size footnotesize
top
\family default
\size default
, 
\family typewriter
\size footnotesize
xosview
\family default
\size default
, 
\family typewriter
\size footnotesize
iostat
\family default
\size default
, 
\family typewriter
\size footnotesize
watch df /tmp/
\family default
\size default
 etc on 
\emph on
all(!)
\emph default
 your involved machines in order to get a chance for noticing when anything
 goes wrong! 
\emph on
Never
\emph default
, really 
\emph on
never
\emph default
 run 
\family typewriter
\size footnotesize
tree-replay.sh
\family default
\size default
 
\series bold
blindly
\series default
!
\end_layout

\begin_layout Subsection
Visualization of Results
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 You need enough space in 
\family typewriter
/tmp/
\family default
 (or in another 
\family typewriter
$TMPDIR
\family default
) for temporary intermediate files.
 If your 
\family typewriter
blkreplay
\family default
 run was very long (several hours or even days), or if your replay had a
 high degree of parallelism, you may need several gigabytes, in extreme
 cases even several hundreds of gigabytes (as well as rather long running
 times -- please don't interrupt 
\family typewriter
graph.sh
\family default
 as it spawns lots of subprocesses and creates lots of temporary files).
 Currently, there are no checks for free space in 
\family typewriter
/tmp/
\family default
, so running out of space may produce wrong results 
\emph on
silently
\emph default
.
 As a countermeasure, run 
\family typewriter
watch df /tmp/
\family default
 in a separate window during your run of 
\family typewriter
graph.sh
\family default
.
\end_layout

\begin_layout Standard
If you have enabled the module 
\family typewriter
graph
\family default
 in the config file 
\family typewriter
default-graph.conf
\family default
, the following steps will be carried out automatically for you.
 Alternatively or in addition, the script 
\family typewriter
tree-graph.sh
\family default
 can be used to (re-)create all 
\family typewriter
*.png
\family default
 graphics in a whole directory hierarchy, analogously to 
\family typewriter
tree-replay.sh
\family default
 (and even sourcing the same 
\family typewriter
*.conf 
\family default
files in the same way).
 In case you need some individual graphics, read on for details.
\end_layout

\begin_layout Standard
After a run of 
\family typewriter
tree-replay.sh
\family default
, 
\family typewriter
cd
\family default
 to one of the subdirectories where your result files 
\family typewriter
*.replay.gz
\family default
 have been produced.
 There should be as many 
\family typewriter
*.replay.gz
\family default
 files as there was replay parallelism (on multiple devices in parallel).
 Check that.
 In addition, check that no errors are inside them, for example by typing:
\end_layout

\begin_layout Quotation

\family typewriter
zgrep ERROR *.replay.gz
\end_layout

\begin_layout Standard
If all is right, issue the following command:
\end_layout

\begin_layout Quotation

\family typewriter
/path/to/graph.sh *.replay.gz
\end_layout

\begin_layout Standard
This will produce 
\family typewriter
*.png
\family default
 files, which you can inspect with any graphical viewer like 
\family typewriter
eog
\family default
 / 
\family typewriter
konqueror
\family default
 etc, print via 
\family typewriter
lpr
\family default
, or even work on with graphical editors like 
\family typewriter
gimp
\family default
.
\end_layout

\begin_layout Standard
All files 
\family typewriter
*.replay.gz
\family default
 will be taken together, to form a single result from contemporary replays
 on several devices in parallel.
 This means: where possible, results from multiple replay devices will be
 merged together into a single graphics.
\end_layout

\begin_layout Description
Hint: If you want to view only a single particular device (or zoom into
 it), just call 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/path/to/graph.sh
\end_layout

\end_inset


\family default
 with a single argument.
\end_layout

\begin_layout Standard
As output, multiple kinds of graphics are produced.
 Each one starts with the same prefix, but has another suffix.
 For example, 
\family typewriter
yourname.g01.latency.realtime.png
\family default
 is a graphics file showing the measured latencies in realtime.
 The numbering part 
\family typewriter
.g01.

\family default
 etc is for sorting in the shell, such that the 
\begin_inset Quotes eld
\end_inset

most interesting
\begin_inset Foot
status open

\begin_layout Plain Layout
For many people; of course, there may be different needs.
 Feel free to rename your result files as you like.
\end_layout

\end_inset


\begin_inset Quotes erd
\end_inset

 graphics will come first.
\end_layout

\begin_layout Standard
By default, parameters matching 
\family typewriter
*.load.gz
\family default
 will produce graphics containing only 
\emph on
static
\emph default
 analyses.
 When matching 
\family typewriter
*.replay.gz
\family default
, only 
\emph on
dynamic
\emph default
 ones are produced.
 In order additionally switch on any (or both), just add one of the options
 
\family typewriter
--static
\family default
 or 
\family typewriter
--dynamic
\family default
.
\end_layout

\begin_layout Standard
There is one exception: the throughput graphics are produced always, independent
ly from 
\family typewriter
--static
\family default
 or 
\family typewriter
--dynamic
\family default
.
 In the following, the meaning of different suffixes occurring in the output
 filenames is described:
\end_layout

\begin_layout Description

\family typewriter
*.thrp.*.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The throughput is displayed on the y axis.
 There is an overview variant, where the overall requested throughput is
 depicted green, while the actual throughput (when running in dynamic mode)
 is orange, for comparison.
 When both are exactly identical (which occurs for example when you have
 an extremely fast disk or a very low-demanding load), don't be confused
 when the orange line completely covers the green one (a property of 
\family typewriter
gnuplot
\family default
).
 There are also more detailed variants where the read vs write throughput
 is distinct.
\end_layout

\begin_layout Subsubsection
Static Analysis (
\family typewriter
--static
\family default
)
\end_layout

\begin_layout Description

\family typewriter
*.ws_log.*.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

On the y axis, the workingset size (cf section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Workingset"

\end_inset

) is displayed in logarithmic scale.
 Multiple window sizes 
\begin_inset Formula $\delta$
\end_inset

 are displayed together in one graphics: 
\family typewriter
001
\family default
 means 
\begin_inset Formula $\delta=1s$
\end_inset

, 
\family typewriter
006
\family default
 means 6 seconds, 
\family typewriter
060
\family default
 means 60 seconds, and 
\family typewriter
000
\family default
 means 
\begin_inset Formula $\delta=\infty$
\end_inset

.
 The latter is nothing but cumulation of 
\emph on
all
\emph default
 occurring sector numbers into one set
\begin_inset Foot
status open

\begin_layout Plain Layout
The slope of the 
\family typewriter
000
\family default
 line is an indicator for the 
\begin_inset Quotes eld
\end_inset

repetitiveness
\begin_inset Quotes erd
\end_inset

 of the workload.
\end_layout

\end_inset

.
\end_layout

\begin_layout Description

\family typewriter
*.ws_lin.*.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

Like 
\family typewriter
ws_log
\family default
, but the y axis is in linear scale.
 May be useful for detecting more fine-grained behaviour in peaks.
\end_layout

\begin_layout Description

\family typewriter
*.sum_dist.*.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

On the y axis, the 
\emph on
distance
\emph default
 between the lowest and the highest sector number occurring in each workingset
 window is displayed.
 As a result, we can see something like a 
\begin_inset Quotes eld
\end_inset

total seek distance
\begin_inset Quotes erd
\end_inset

, as if a disk elevator strategy 
\emph on
would
\emph default
 sort all requests inside a workingset window according to sector numbers,
 in order to optimize throughput in favour of latency.
 For experts, this can reveal some interesting internal property of the
 workload.
\end_layout

\begin_layout Description

\family typewriter
*.avg_dist.*.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

Same as 
\family typewriter
sum_dist
\family default
, but the average distance is displayed (normalization against the workingset
 size).
 This results in something like an idealized 
\begin_inset Quotes eld
\end_inset

average seek distance
\begin_inset Quotes erd
\end_inset

 during each time window.
\end_layout

\begin_layout Description

\family typewriter
*.rqsize.*.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

Displays a histogram of the request sizes (#sectors) occurring in the load,
 in units of sectors.
\end_layout

\begin_layout Description

\family typewriter
*.rqpos.*.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

Displays a histogram of the request position (sector#) occurring in the
 load, in coarse units of whole GiB.
 The coarse units are necessary to avoid too much jitter in the y axis.
\end_layout

\begin_layout Description

\family typewriter
*.turns.*.png
\family default
 (both static and dynamic variants)
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

A turn is a request where the sector number is 
\emph on
lower
\emph default
 than the sector number of its predecessor.
 For a mechanical hard disk, a turn means that it has to 
\begin_inset Quotes eld
\end_inset

seek backwards
\begin_inset Quotes erd
\end_inset

, which is often more time-consuming than seeking forward.
 Imagine an old-fashioned magnetic tape (e.g.
 DAT), then you will get some feeling for the meaning of a turn.
 Turns are an interesting measure for the locality behaviour of the load
 (or the replay result).
 There is no freak that some IO schedulers try to minimize the number of
 turns (for example the famous sawtooth strategy).
 In this graphics, the y axis shows the 
\emph on
relative
\emph default
 number of turns occurring during a time window of 1 seconds (in percent,
 related to the total number of requests occurring during that time window).
 
\end_layout

\begin_layout Subsubsection
Dynamic Analysis (
\family typewriter
--dynamic
\family default
)
\end_layout

\begin_layout Description

\family typewriter
*.latency.*.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

On the y axis, latencies are displayed.
\end_layout

\begin_layout Description

\family typewriter
*.delay.*.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

On the y axis, the delays between the intended starting time and the real
 starting time are displayed.
\end_layout

\begin_layout Description

\family typewriter
*.thrp.*.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

On the y axis, the troughput (IOPS) is displayed.
\end_layout

\begin_layout Description

\family typewriter
*.flying.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The y axis shows the total number of requests which are currently 
\begin_inset Quotes eld
\end_inset

on the fly
\begin_inset Quotes erd
\end_inset

.
\begin_inset Newline newline
\end_inset

Examples: 
\family typewriter
*.latency.flying.png
\family default
 is derived from the latencies, and thus showing the number of requests
 currently submitted to the device, while 
\family typewriter
*.delay.flying.png
\family default
 is derived from the delay, showing the number of requests currently waiting
 in the logical 
\begin_inset Quotes eld
\end_inset

delay queue
\begin_inset Quotes erd
\end_inset

.
\end_layout

\begin_layout Description

\family typewriter
*.smooth.*
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

This internal infix indicates that the y axis has been smoothed, in order
 to be able to see anything in wildly jiggling data.
 Example: 
\family typewriter
*.smooth.latency.flying.png
\family default
.
\end_layout

\begin_layout Description

\family typewriter
*.realtime.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The x axis is ordered according to the real starting timestamps which have
 actually occurred during replay, possibly containing any delays.
 Notice: when multiple concurrent 
\family typewriter
*.replay.gz
\family default
 have been supplied as an argument, they are 
\emph on
merged
\emph default
 (since their timestamps are usually from the same range), similar to the
 effect of overlay slides.
\end_layout

\begin_layout Description

\family typewriter
*.setpoint.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The x axis is ordered according to the 
\emph on
intended
\emph default
 request submission time (starting point), i.e.
 when the request 
\emph on
should have
\emph default
 started (ignoring any delays).
 As before, multiple 
\family typewriter
*.replay.gz
\family default
 are merged.
\end_layout

\begin_layout Description

\family typewriter
*.completed.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The x axis is ordered by the completion time of the requests.
 Note that this may differ from the submission time.
\end_layout

\begin_layout Description

\family typewriter
*.points.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The x axis is ordered by 
\emph on
requests
\emph default
, not timestamps.
 As a result, requests on the x axis are 
\emph on
equidistant
\emph default
, even in case of heavy throughput differences.
 For 
\family typewriter
*.replay.gz
\family default
, the order is the 
\emph on
completion
\emph default
 order.
 This graphics is useful as a kind of 
\begin_inset Quotes eld
\end_inset

looking glass
\begin_inset Quotes erd
\end_inset

, to reveal more details from performance hot spots.
 Notice: multiple concurrent replays are 
\emph on
not
\emph default
 merged on the x axis (as is the case with 
\family typewriter
*.realtime.png
\family default
, 
\family typewriter
*.setpoint.png
\family default
 and 
\family typewriter
*.completed.png
\family default
).
 Instead, the runs are just concatenated (pasted together sequentially)
 in the same order as in the corresponding files.
\end_layout

\begin_layout Description

\family typewriter
*.bins.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The new y axis is now a 
\emph on
histogram
\emph default
 showing absolute frequency occurring at the former y axis, while the new
 x axis now carries the role of the former y axis.
\begin_inset Newline newline
\end_inset

Examples: 
\family typewriter
*.latency.bins.png
\family default
 shows the latencies on the x axis (while the former 
\family typewriter
*.latency.realtime.png
\family default
 had it on the y axis), while the new y axis now shows the absolute frequency
 of that latency (how often that latency occurs, independently from the
 former replay timestamps).
\end_layout

\begin_layout Description

\family typewriter
*.xy.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The x axis as well as the y axis are displaying non-timely data.
 This is useful for visualization of 
\emph on
corrolations
\emph default
.
 The following variants exist:
\end_layout

\begin_deeper
\begin_layout Description

\family typewriter
*.latency.xy.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

While latencies are displayed at the y axis, the x axis inidicates the number
 of flying requests (request queue length).
 Thus you can see the corrolation between request queue length and service
 time.
\end_layout

\begin_layout Description

\family typewriter
*.delay.xy.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

Similarly, but shows the corrolation between request queue length and delays
 (the time needed to 
\emph on
enter
\emph default
 the request queue at all).
\end_layout

\begin_layout Description

\family typewriter
*.latency.delay.xy.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

Immediate corrolation between latency and delay.
\end_layout

\end_deeper
\begin_layout Standard
Hint: the internal data format of 
\family typewriter
*.replay.gz
\family default
 is the same as 
\family typewriter
*.load.gz
\family default
.
 Thus you can use 
\family typewriter
/path/to/graph.sh --static *.replay.gz
\family default
 to additionally create the same static analysis graphics as described in
 section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Select-blkreplay-Load"

\end_inset

.
 In difference to analysis of 
\family typewriter
*.load.gz
\family default
, this time you may have selected a different time window, and you may have
 merged multiple replays together.
\end_layout

\begin_layout Section
Human Interpretation of Results
\end_layout

\begin_layout Standard
Many existing benchmark tools try to deliver a single number as a result,
 such as 
\begin_inset Formula $x$
\end_inset

 
\family typewriter
hyper-ops
\family default
 or 
\begin_inset Formula $x$
\end_inset

 
\family typewriter
hell-stones
\family default
.
 Such numbers cannot describe the reality, which has multiple dimensions
 and is much more complex.
 Sometimes, new dimensions are even discovered to have an influence.
\end_layout

\begin_layout Standard
By default, 
\family typewriter
blkreplay
\family default
 does not produce any (single) number, but graphics, showing the behaviour
 over time (or in other dimensions).
 Although some of those graphics like 
\family typewriter
*thrp*
\family default
 could be used for production of numbers, most of them serve as input to
 the human brain: the human 
\begin_inset Quotes eld
\end_inset

neural network
\begin_inset Quotes erd
\end_inset

 can be 
\emph on
trained
\emph default
 to detect 
\series bold
hidden properties
\series default
 in your measurement data, which cannot be (easily) detected by current
 AI technology.
\end_layout

\begin_layout Subsection
Sonar Diagrams
\end_layout

\begin_layout Standard
The following example shows the replay latencies of a natural load over
 real time.
 Note that the y axis is logarithmic scale (otherwise you would'nt see too
 much in the lower bands):
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/example1.g01.latency.realtime.png
	lyxscale 20
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
Reads are depicted as blue dots, while writes are colored red.
 In case of 
\family typewriter
--with-partial
\family default
, pushed-back requests are displayed additionally
\begin_inset Foot
status open

\begin_layout Plain Layout
Notice that the set of purple 
\begin_inset Quotes eld
\end_inset

pushes
\begin_inset Quotes erd
\end_inset

 is always a 
\emph on
subset
\emph default
 of the 
\begin_inset Quotes eld
\end_inset

writes
\begin_inset Quotes erd
\end_inset

.
 They are only visible because they are printed later (hiding the previous
 red dots).
\end_layout

\end_inset

 in purple.
 Together, they form some kind of 
\begin_inset Quotes eld
\end_inset

clouds
\begin_inset Quotes erd
\end_inset

, showing some 
\begin_inset Quotes eld
\end_inset

density
\begin_inset Quotes erd
\end_inset

 both in time scale, as well as in the 
\begin_inset Quotes eld
\end_inset

latency scale
\begin_inset Quotes erd
\end_inset

.
\end_layout

\begin_layout Standard
It requires some experience to interpret sonar diagrams.
\end_layout

\begin_layout Itemize
\noindent
As you can see, there are two main 
\series bold

\begin_inset Quotes eld
\end_inset

bands
\begin_inset Quotes erd
\end_inset


\series default
 optically visible to the human interpretation.
 These look similar to 
\begin_inset Quotes eld
\end_inset

fish swarms
\begin_inset Quotes erd
\end_inset

 occuring in sonar diagrams; hence the name 
\begin_inset Quotes eld
\end_inset

sonar diagram
\begin_inset Quotes erd
\end_inset

.
\end_layout

\begin_layout Itemize
Most obviously, 
\series bold
cache hierarchies
\series default
 can be identified visually, by looking at the 
\begin_inset Quotes eld
\end_inset

bands
\begin_inset Quotes erd
\end_inset

.
 In addition, the natural fluctuation of the load can be seen.
 Even better, it is quite interesting how the test candidate is responding
 to the fluctuation, even over time.
\end_layout

\begin_layout Itemize
Any differences between read and write behaviour will immediately catch
 your eye.
\end_layout

\begin_layout Itemize
The red 
\begin_inset Quotes eld
\end_inset

needles
\begin_inset Quotes erd
\end_inset

 are indications of queueing.
 In difference to a single 
\begin_inset Quotes eld
\end_inset

hang
\begin_inset Quotes erd
\end_inset

 which would just 
\begin_inset Quotes eld
\end_inset

fly above
\begin_inset Quotes erd
\end_inset

 other requests, intermediate latencies from 
\begin_inset Quotes eld
\end_inset

fellow requests
\begin_inset Quotes erd
\end_inset

 are also present, spanning almost the full range from 
\begin_inset Quotes eld
\end_inset

fast
\begin_inset Quotes erd
\end_inset

 requests to the queued ones.
\end_layout

\begin_layout Itemize
In case of the strongest needle, you can even see a 
\begin_inset Quotes eld
\end_inset

lash
\begin_inset Quotes erd
\end_inset

 behind it.
 There are no 
\begin_inset Quotes eld
\end_inset

faster
\begin_inset Quotes erd
\end_inset

 requests (not even reads) during this lash (i.e.
 the space below it appears empty - even as if there were an interruption
 in the time axis).
 This is an indication of a 
\emph on
global
\emph default
 queuing strategy.
\end_layout

\begin_layout Standard
The same latencies are now displayed as a histogram.
 A histogram is a non-timely survey of the absolute frequencies occurring
 in the previous graphics.
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/example1.g06.latency.bins.png
	lyxscale 20
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
You can identify the 
\begin_inset Quotes eld
\end_inset

bands
\begin_inset Quotes erd
\end_inset

 a second time by looking at the camel-like 
\begin_inset Quotes eld
\end_inset

humps
\begin_inset Quotes erd
\end_inset

.
 In addition to the sonar diagram, you can identify the 
\begin_inset Quotes eld
\end_inset

hight
\begin_inset Quotes erd
\end_inset

 and 
\begin_inset Quotes eld
\end_inset

thickness
\begin_inset Quotes erd
\end_inset

 of a hump more easily.
 Some very small additional humps can also be found, probably an indicator
 for further caches or probably for some scheduler timeouts.
\end_layout

\begin_layout Subsection
Delay Diagrams
\end_layout

\begin_layout Standard
As already explained, delays are the difference between the 
\emph on
intended/demanded
\emph default
 starting time of an IO requests, and its 
\emph on
real
\emph default
 starting time.
 Huge delays are almost always an indication that the system cannot 
\begin_inset Quotes eld
\end_inset

catch up
\begin_inset Quotes erd
\end_inset

 with the application demands.
 Here is an example:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/example1.g04.delay.setpoint.png
	lyxscale 20
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
Here are some interpretations:
\end_layout

\begin_layout Itemize
You will immediately notice the 
\begin_inset Quotes eld
\end_inset

runaway tail
\begin_inset Quotes erd
\end_inset

 after the purple indicator line, which is just a strong indication that
 overall throughput is too low.
\end_layout

\begin_layout Itemize
Even before, you can see some 
\begin_inset Quotes eld
\end_inset

needles
\begin_inset Quotes erd
\end_inset

 with a 
\begin_inset Quotes eld
\end_inset

lash
\begin_inset Quotes erd
\end_inset

 behind.
 Another indication, even occurring in a section with rather 
\begin_inset Quotes eld
\end_inset

low
\begin_inset Quotes erd
\end_inset

 throughput demands.
\end_layout

\begin_layout Standard
As before, there is also a histogram variant of delays:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/example1.g07.delay.bins.png
	lyxscale 20
	width 100col%

\end_inset


\end_layout

\begin_layout Subsection
Throughput Diagrams
\end_layout

\begin_layout Standard
\noindent
In order to compare this with the actual throughput, look at the following
 picture, where demanded throughput is colored green, while actually delivered
 throughput is orange:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/example1.g000.overview.thrp.png
	lyxscale 20
	width 100col%

\end_inset


\family typewriter
*.latency.flying
\end_layout

\begin_layout Standard
\noindent
This picture is a strong indication that single IOPS numbers are not sufficient
 for really describing the actual IO throughput behaviour of a system.
 Our timely plot can tell you a lot more about the system.
 More examples can be found in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Overload-with-(Derived)"

\end_inset

.
\end_layout

\begin_layout Subsection
Flying Diagrams
\end_layout

\begin_layout Standard
These are useful to inspect the actual request queue length over time.
 The following graphics exhibits the typical 
\begin_inset Quotes eld
\end_inset

jitter
\begin_inset Quotes erd
\end_inset

 of the queue length during replay of natural loads having a lot of conflicts,
 and using 
\family typewriter
--with-ordering
\family default
:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/example1.g08.latency.flying.png
	lyxscale 20
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
Notice: this is only the view of 
\family typewriter
blkreplay
\family default
.
 The physical device may be actually loaded with 
\emph on
less
\emph default
 requests in parallel, because intermediate layers (such as drivers, iSCSI,
 etc) may 
\series bold
limit
\series default
 the IO parallelism.
 When 
\family typewriter
blkreplay
\family default
 fires off a request, it need not hit the device immediately.
 There are a lot of opportunities to be queued for some time in another
 places.
\end_layout

\begin_layout Subsection
Corrolation Diagrams
\end_layout

\begin_layout Standard
Most of them are for advanced users and experts.
 They can visualize internal relationships between interesting quantities,
 such as observed request queue length versus observed latency:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/example1.g10.latency.xy.png
	lyxscale 20
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
Rough interpretation: under overload, write requests cannot be started in
 time, and when they are started, they will take a long time.
 Read requests are seemingly behaving better.
\end_layout

\begin_layout Section
Advanced Features
\end_layout

\begin_layout Subsection
Modules
\end_layout

\begin_layout Standard
The script 
\family typewriter
tree-replay.sh
\family default
 can be extended by plugins, aka modules.
 Some modules are delivered with the 
\family typewriter
blkreplay
\family default
 suite, but you can add your own ones (and you should consider submitting
 them to the opensource project 
\family typewriter
blkreplay
\family default
 at 
\family typewriter
github.com
\family default
 if they could be useful for a broader audience).
\end_layout

\begin_layout Standard
Here is a list of already existing modules:
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
iscsi_target_iet
\family default
 When enabled, your 
\family typewriter
iet
\family default
 iSCSI target will be automatically configured for you, automatically generating
 iSCSI 
\family typewriter
IQN
\family default
 names for you.
 Usually this module is used in combination with 
\family typewriter
iscsi_initiator
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
iscsi_initiator
\family default
 When enabled, your 
\family typewriter
blkreplay
\family default
 runs will automatically act on iSCSI 
\family typewriter
IQN
\family default
 identifiers instead of on device names, transparently creating iSCSI connection
s for you.
 It can be used standalone, for example to connect to commercial storage
 boxes.
 When combined with module 
\family typewriter
iscsi_target_
\family default
*, even the 
\family typewriter
IQN
\family default
 name generation will be automated transparently (in this case you just
 supply the device names present at the 
\family typewriter
iscsi_target
\family default
 host).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
recreate_lvm
\family default
 You just supply some physical devices (PVs) in 
\family typewriter
$replay_device_list
\family default
, and this module will transparently (re-)create an LVM volume group (VG)
 for you, together with some configurable number of logical volumes (LVs)
 inside it, optionally even with LVM striping.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
create_lv
\family default
 Here you can put in your code for re-creating logical volumes on an arbitrary
 storage manager.
 You 
\series bold
need
\series default
 this 
\series bold
absolutely
\series default
 whenever you are benchmarking virtualized storage; see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Storage"

\end_inset

.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
scheduler
\family default
 By default (when enabled), the IO scheduler will be set to 
\family typewriter
noop
\family default
 (or another value) on all relevant target machines.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
wipe
\family default
 When enabled, all your devices will be filled with random data before the
 benchmark starts.
 
\series bold
Always
\series default
 use this when benchmarking virtualized storage; see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Storage"

\end_inset

.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
bbu_megaraid
\family default
 When enabled, your LSI megaraid conrollers will be programmed to enable/disable
 BBU write caching.
 The tool 
\family typewriter
MegaCli
\family default
 must be installed.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
graph
\family default
 By default (when enabled), the script 
\family typewriter
graph.sh
\family default
 will be called automatically for you at the end of each 
\family typewriter
blkreplay
\family default
 run.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
pipe_*
\family default
 When enabled, these filtering modules modify the 
\family typewriter
*.load.gz
\family default
 input at runtime before it reaches 
\family typewriter
blkreplay
\family default
.
 Following pipe filters are available:
\end_layout

\begin_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
pipe_repeat
\family default
 Repeat the same input files forever, or until 
\family typewriter
$replay_duration
\family default
 is reached.
 This is useful if your 
\family typewriter
*.load.gz
\family default
 is too short for a longer replay.
 However keep in mind that the 
\emph on
workingset
\emph default
 will not increase, because exactly the 
\emph on
same
\emph default
 IO operations will be repeated after a while.
 In order to circumvent that, combine this module with 
\family typewriter
pipe_slip
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
pipe_slip
\family default
 Add some offset to the sector# after a bunch of operations.
 Use this in combination with 
\family typewriter
pipe_repeat
\family default
 to produce a 
\begin_inset Quotes eld
\end_inset

slowly moving
\begin_inset Quotes erd
\end_inset

 behaviour of the sector numbers, such that the next cycle will repeat the
 operations at a slightly higher position.
 Details see appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:File-default-pipe_slip.conf"

\end_inset

.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
pipe_subst
\family default
 Replace 
\family typewriter
R
\family default
[reads] with 
\family typewriter
W
\family default
[writes] or vice versa.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
pipe_spread
\family default
 Increase the region where the sector numbers will run about (corresponding
 to the size of a LV) by some factor.
 Details see appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:File-default-pipe_spread.conf"

\end_inset

.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
pipe_resize
\family default
 Increase the request size (#sectors) by some factor, or clip it to some
 bounds.
 Details see appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:File-default-pipe_resize.conf"

\end_inset

.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
pipe_cmd
\family default
 Insert arbitrary shell commands into the input pipeline.
\end_layout

\end_deeper
\begin_layout Standard
Detailed documentation of the module parameters can be found in 
\family typewriter
default-*.conf
\family default
, or in appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Module-Parameters"

\end_inset

.
\end_layout

\begin_layout Standard
In order to write your own module, just look at the existing ones and take
 them as a kind of 
\begin_inset Quotes eld
\end_inset

template
\begin_inset Quotes erd
\end_inset

.
 Modules are simply 
\family typewriter
/bin/bash
\family default
 code residing in 
\family typewriter
scripts/modules/
\family default
 and starting with a number
\begin_inset Foot
status open

\begin_layout Plain Layout
The number is used to determine the ASCII sort order, and thus the order
 in which modules are 
\family typewriter
source
\family default
d.
\end_layout

\end_inset

, followed by an underscore, and finally 
\family typewriter
\emph on
modulename
\emph default
.sh
\family default
.
 Any existing module is automatically 
\family typewriter
source
\family default
d by 
\family typewriter
tree-replay.sh
\family default
, provided that some config file 
\family typewriter
default-
\emph on
modulename
\emph default
.conf
\family default
 exists in your working directory (or in any parent directory down the 
\family typewriter
../
\family default
 path).
\end_layout

\begin_layout Standard
Modules are sourced only once at the start of 
\family typewriter
tree-replay.sh
\family default
.
 Thus they should only define some shell 
\family typewriter
function
\family default
s and change some list variables.
 The shell functions will be called later for each leaf directory.
 The names of your functions should obey the following conventions:
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
\emph on
modulename
\emph default
_prepare()
\family default
 is called during the preparation phase.
 Typically, you can set some variables here, initialize some controllers,
 setup your network, create some iSCSI connections, or the like.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
\emph on
modulename
\emph default
_setup()
\family default
 The setup phase starts after 
\emph on
all
\emph default
 prepare functions of all modules have been executed.
 For example, some device setup operations are only possible after some
 iSCSI connections have been already established.
 Therefore, the plugin methodology offers you a separate phase in order
 to guarantee some overall order on operations even from unknown foreign
 modules.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
\emph on
modulename
\emph default
_run()
\family default
 Typically, you won't do anything here, because the running phase is usually
 the task of the 
\family typewriter
main
\family default
 module.
 Only in very special cases, you would need to hook yourself into this phase.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
\emph on
modulename
\emph default
_cleanup()
\family default
 Typically, you will de-configure something here, remove some temporary
 files, check your results, create some statistics, etc.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
\emph on
modulename
\emph default
_finish()
\family default
 Typically, network connections etc will be shutdown only here, because
 some operations from the cleanup phase could be relying on them being active
 yet.
\end_layout

\begin_layout Standard
For each phase, there exists a variable 
\family typewriter
\emph on
phasename
\emph default
_list
\family default
 determining the order in which the functions will be called.
 In most cases, you will just 
\emph on
append
\emph default
 your function name to that list.
 In some cases, you may want to 
\emph on
pre
\emph default
pend instead.
 Look into the existing code of small modules like 
\family typewriter
graph
\family default
 to get an impression, and read the 
\family typewriter
main()
\family default
 function in the 
\family typewriter
main
\family default
 module.
\end_layout

\begin_layout Section
Lowlevel Details and Expert Usage
\end_layout

\begin_layout Standard
Ordinary users should skip this section.
\end_layout

\begin_layout Subsection
Internal Overhead
\end_layout

\begin_layout Standard
Some new SSDs are promising IO rates of more than 100.000 IOPS when operated
 over extremely fast IO channels.
 When you try to achieve such high rates, you may probably stumble over
 several difficulties.
\end_layout

\begin_layout Standard
For example, running too much threads / processes on the same file handle
 or on the same pipe can lead to serious lock contention in the Linux kernel,
 limiting overall throughput.
 Although some people might argue that 
\family typewriter
blkreplay
\family default
 itself could produce an 
\begin_inset Quotes eld
\end_inset

artificial
\begin_inset Quotes erd
\end_inset

 bottleneck, please keep in mind that you need some 
\emph on
real applications
\emph default
 which were capable of producing / consuming such extremely high IO traffic.
 Real applications usually don't use AIO, where such problems occur less
 frequently.
 No wonder that benchmark results published on the internet are often based
 on AIO, which is far from practice.
\end_layout

\begin_layout Standard
Anyway, here is some advice how to deal with extremely high IO traffic.
 For 
\begin_inset Quotes eld
\end_inset

normal
\begin_inset Quotes erd
\end_inset

 IO rates in the region of a few 10.000, the following is usually not needed.
 Otherwise, you should be prepared to look into internals of the kernel,
 compile your own high-speed custom kernels with some debugging options
 switched off / better tuning of schedulers etc, and possibly use tools
 like 
\family typewriter
oprofile
\family default
 to check what is really going on.
\end_layout

\begin_layout Standard
In order to determine the internal overhead of 
\family typewriter
blkreplay
\family default
, play around with the following parameters:
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--threads=\SpecialChar \ldots{}

\family default
 (or 
\family typewriter
threads=\SpecialChar \ldots{}

\family default
 as a shell variable in 
\family typewriter
*.conf
\family default
) The most important setting.
 Although counter-intuitive, too high numbers of threads may increase the
 overall overhead and thus reduce throughput.
 Be prepared to degrade this to rather low numbers in order to really achieve
 extremely high IO throughput.
 Usually, no more than 32 should be used.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--dry-run
\family default
 (or, as a shell variable, 
\family typewriter
dry_run
\family default
 with underscore instead of minus) Omit any 
\family typewriter
read()
\family default
 and 
\family typewriter
write()
\family default
 operations, but continue to use 
\family typewriter
lseek64()
\family default
 on the file handles.
 This way, you can determine the internal overhead in isolation.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--fake-io
\family default
 Additionally omit even 
\family typewriter
lseek64()
\family default
 operations, as well as creation of header data.
 Compare results with 
\family typewriter
--dry-run
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--fan-out=\SpecialChar \ldots{}

\family default
 In order to reduce parallelism on 
\family typewriter
pipe()
\family default
 channels, 
\family typewriter
blkreplay
\family default
 spawns internal distributor threads, forming a tree with limited fan-out
 degree.
 Too high numbers will increase pipe overhead, while too low numbers will
 increase thread and scheduling overhead.
 The optimum may depend on hardware (e.g.
 NUMA) and kernel version.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--no-dispatcher
\family default
 Omit distributor threads for the answer channels completely.
 When combined with high numbers of threads, this may increase the kernel
 overhead to almost unusable regions.
 Use this for kernel tuning.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--bottleneck=\SpecialChar \ldots{}

\family default
 Limit the number of flying IO requests.
 Has a similar effect than 
\family typewriter
--threads
\family default
, but is not the same.
 Can be used for determining internal kernel and even hardware overheads
 such as TLB thrashing (e.g.
 by keeping 
\family typewriter
--bottleneck
\family default
 constant and increasing 
\family typewriter
--threads
\family default
).
 Could be useful for kernel and hardware tuning.
\begin_inset Newline newline
\end_inset


\series bold
Hint:
\series default
 you can even set this to much higher numbers than 
\family typewriter
--threads
\family default
, in order to pre-fill the communication pipes with some ahead requests
 (minimizing the risk of waiting times).
 However, when the number is way too high, a 
\emph on
coprocess deadlock
\emph default
 could occur (see the literature about coprocesses).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
--speedup=\SpecialChar \ldots{}

\family default
 A very dangerous option when used with natural loads.
 Leads to heavy distortions of the relationship between timely and positionly
 behaviour; in essence 
\emph on
a natural load will not remain a natural load any more
\emph default
.
 For artificial loads, however, such as 
\family typewriter
*bursts*
\family default
, it is useful to reach top IOPS regions.
\end_layout

\begin_layout Standard
In any case, extremely high IOPS rates are likely to be reached only by
 running multiple 
\family typewriter
blkreplay
\family default
 instances in parallel.
\end_layout

\begin_layout Chapter
How to use 
\family typewriter
blktrace
\family default
 for Recording of Natural Loads
\begin_inset CommandInset label
LatexCommand label
name "chap:How-to-use"

\end_inset


\end_layout

\begin_layout Standard
The tool 
\family typewriter
blktrace
\family default
, as well as its friends like 
\family typewriter
blkparse
\family default
, are described elsewhere.
 Here are some additions from a practical viewpoint of IT operations:
\end_layout

\begin_layout Itemize
You will need a kernel with compile-time option 
\family typewriter
CONFIG_BLK_DEV_IO_TRACE
\family default
 enabled.
 Although many newer Linux distros have it already enabled by default, you
 may stumble over elder systems / distributions where it was missing (so
 you may need to install and/or recompile a different kernel).
\end_layout

\begin_layout Itemize
Please note that some 
\family typewriter
man
\family default
 pages about 
\family typewriter
blktrace
\family default
 contain unclear / misleading information about the physical unit of the
 
\family typewriter
-b
\family default
 / 
\family typewriter
--buffer-size
\family default
 option, which may lead to serious problems (including kernel hangups /
 crashes) if you (accidentally) try to allocate more kernel memory than
 physically available, and/or your production server is heavily loaded.
 Unless you are recording extremely high-performance loads, the default
 settings will succeed; so there will be no risk for IT operations.
\end_layout

\begin_layout Itemize
Tracing multiple devices in parallel is possible, but sometimes works differentl
y than (inconsistently) described in the 
\family typewriter
man
\family default
, possibly leading to kernel problems (depending on versions etc).
 Before recording blocktraces at production sites, we recommend to test
 it somewhere in advance.
\end_layout

\begin_layout Standard
Please note that you need enough filesystem space for recording, residing
 on a 
\emph on
different
\emph default
 partition than you want to record the IO traffic! Otherwise, your measurements
 might be distorted.
\end_layout

\begin_layout Standard
You should record at least 8 hours of operation; better 24h.
\end_layout

\begin_layout Standard
Once 
\family typewriter
blktrace
\family default
 has finished, some files 
\family typewriter
\emph on
yourname
\emph default
.blktrace.{0..7}
\family default
 (or other numbers, corresponding to the number of CPUs) will exist in the
 current working directory of your production machine.
 Please copy all of them to your workstation before proceeding (never risk
 more than necessary on production systems).
\end_layout

\begin_layout Standard
After copying, issue the following command:
\end_layout

\begin_layout Quote

\family typewriter
/path/to/blkreplay/scripts/conv_blktrace_to_load.sh 
\emph on
yourname
\end_layout

\begin_layout Standard
This will produce an output file 
\family typewriter
\emph on
yourname
\emph default
.load.gz
\family default
 in your working directory.
 You should seriously consider renaming it to 
\family typewriter
\emph on
some_better_name
\emph default
.load.gz
\family default
.
\end_layout

\begin_layout Standard
The result file will contain as much IO requests as the script could find
 in the 
\family typewriter
blktrace
\family default
 recording, but it will not contain any original timings (durations of the
 requests, aka latencies).
 The latency column is simply set to 
\begin_inset Formula $0.0$
\end_inset

 everywhere.
\end_layout

\begin_layout Standard
There is a reason for it.
 Extracting timing information 
\emph on
reliably
\emph default
 from a 
\family typewriter
blktrace
\family default
 recording is difficult.
 The problem is that each driver / subsystem in den kernel uses different
 trace action characters to indicate the start of an IO request (as well
 as for indicating various intermediate stages of request processing).
 Request completion is 
\emph on
another
\emph default
 event denoted by character '
\family typewriter
C
\family default
'.
 Depending on kernel version / driver, the completion events may be even
 missing at all (some devices on very old kernels, which nevertheless occur
 in practice).
 More seriously, it is a hard task to determine pairs of events, i.e.
 determining which completion event belongs to which submission event.
 There is no concept of a 
\begin_inset Quotes eld
\end_inset

request ID
\begin_inset Quotes erd
\end_inset

 which would allow this easily.
 You have to guess matching pairs from their sector numbers / length etc.
 Guessing may be wrong due to ambiguities.
 In general, a completion could sometimes even belong to multiple submission
 events.
 In addition, when you start and/or end your 
\family typewriter
blktrace
\family default
 recording at a heavily loaded 
\emph on
production
\emph default
 server (which is just the most interesting case), some pairs will very
 likely remain incomplete at the start and at the end.
\end_layout

\begin_layout Standard
Even under 
\begin_inset Quotes eld
\end_inset

normal
\begin_inset Quotes erd
\end_inset

 circumstances, we found many cases where non-matching events led to an
 incomplete 
\family typewriter
*.load.gz
\family default
 result.
 Therefore we decided to omit timings from our default conversion script,
 at least for now
\begin_inset Foot
status open

\begin_layout Plain Layout
Please submit any improvements to the author as patches / git pull requests.
\end_layout

\end_inset

.
 However, there is another script which tries to 
\emph on
guess
\emph default
 the timings as best as it can:
\end_layout

\begin_layout Quote

\family typewriter
/path/to/blkreplay/scripts/conv_blktrace_to_load_with_guessed_timing.sh 
\emph on
yourname
\end_layout

\begin_layout Standard
\noindent
The latter script will name its output 
\family typewriter
\emph on
yourname
\emph default
.guessed.load.gz,
\family default
 which is different from the former one.
 There is a reason for it.
 Please don't remove the 
\family typewriter
*guessed*
\family default
 infix when renaming that result file.
 We found cases where the difference was considerable.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Very often, some single guessed timings are obviously wrong.
 You can see it at the sonar diagram.
\end_layout

\begin_layout Standard
Please use the 
\family typewriter
*guessed*
\family default
 version only for 
\series bold
analysis
\series default
, such as producing 
\family typewriter
*.latency.flying
\family default
 or 
\family typewriter
*.smooth.latency.flying
\family default
 from 
\family typewriter
graph.sh --static --dynamic
\family default
 in order to determine the original IO parallelism and other interesting
 properties.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Only use the 
\family typewriter
*.load.gz
\family default
 output stemming from 
\family typewriter
conv_blktrace_to_load.sh
\family default
 as input for 
\family typewriter
blkreplay
\family default
.
 Never use 
\family typewriter
*.guessed.load.gz
\family default
.
 You may be missing some essential behaviour which could be crucial for
 detection of incidents!
\end_layout

\begin_layout Standard
When your load contains some interesting effects and could be of broader
 interest to a larger community, please contact the author of this paper
 by Email in order to publish your load at 
\family typewriter
www.blkreplay.org
\family default
.
 In particular, unusual and novel application areas, heavy ordinary workloads,
 and recordings of incidents, are of a broader interest.
\end_layout

\begin_layout Chapter
Experiences with some Setups and some Loads
\end_layout

\begin_layout Section
Overload Tests
\end_layout

\begin_layout Standard
Overload tests try to reveal the behaviour of a system under exceptionary
 conditions, similar to a worst-case scenario.
\end_layout

\begin_layout Standard
Overload tests can (and should) be done with natural loads (such as 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

http://www.blkreplay.org/loads/natural/1and1/natural-derived/
\end_layout

\end_inset

).
 In addition, it is one of the rare places where 
\emph on
artificial
\emph default
 loads can reach 
\emph on
some
\emph default
 justification.
\end_layout

\begin_layout Subsection
Overload with Artificial Bursts
\end_layout

\begin_layout Standard
The following examples are using the artificial loads from 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

http://www.blkreplay.org/loads/artificial/random.bursts/
\end_layout

\end_inset

, which are rather 
\begin_inset Quotes eld
\end_inset

anal
\begin_inset Quotes erd
\end_inset

 ones (see file 
\family typewriter
00README
\family default
 there).
 The load is linearly increasing over time, until the test candidate will
 
\begin_inset Quotes eld
\end_inset

collapse
\begin_inset Quotes erd
\end_inset

 under the overload.
\end_layout

\begin_layout Standard
We use a single 
\family typewriter
blkreplay
\family default
 instance, and compare between an old SATA RAID-6 and a newer RAID-6 with
 faster SAS disks (15k RPM).
 For now, we consistently use 
\family typewriter
--threads=128
\family default
.
 All results can be found in the directory 
\family typewriter
blkreplay.git/example-run/
\family default
 and subdirs.
\end_layout

\begin_layout Standard
Here is the sonar diagram for the SATA system.
 It 
\emph on
should
\emph default
 have run 300s, but it actually run much longer in realtime:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
At the beginning, the load is rather low, and the system can catch up.
 Soon, it reaches a kind of 
\begin_inset Quotes eld
\end_inset

maximum throughput
\begin_inset Quotes erd
\end_inset

 where the latencies go up to about 1 second.
 You can see on the realtime x axis that the replay is delayed as a whole.
 This can be better seen at the delay diagram:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g04.delay.setpoint.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
Here the x axis shows the 
\emph on
intended
\emph default
 starting time of each request, which is deliberately concentrated at whole
 seconds (with gaps inbetween), a property of of the 
\family typewriter
*bursts*
\family default
 loads.
 The 
\begin_inset Quotes eld
\end_inset

collapsing point
\begin_inset Quotes erd
\end_inset

 is about 60s, which means that it roughly can deliver 600 random IOPS.
 This can be even better seen at the overwiew 
\family typewriter
thrp
\family default
 comparing the actual with the demanded throughput, as well as the detailed
 
\family typewriter
thrp
\family default
 diagram of the actual throughput ordered by completion time of the requests:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g00.thrp.completed.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
Now we turn to better hardware: the SAS system uses smaller disks with 15k
 RPM, and thus needs more spindles to reach the same capacity.
 If we would run the same test for 300s (as in the SATA case), we would
 not notice any collapsing.
 Thus we double 
\family typewriter
replay_duration
\family default
 to 600s in order to be able to see the collapsing point.
 Please take this into account when looking at the following graphics:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sas/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sas.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
Notice that the system is even able to reproduce the bursting behaviour
 of the load, much better than the SATA system.
 Only at higher request rates, some 
\begin_inset Quotes eld
\end_inset

blurring clouds
\begin_inset Quotes erd
\end_inset

 appear on the x axis, until it finally collapses under too high load.
 It is amazing to see the 
\begin_inset Quotes eld
\end_inset

binary
\begin_inset Quotes erd
\end_inset

 behaviour at the collapsing point: as soon as the actual throughput rate
 cannot catch up with the request rate, the internal queues will suddenly
 grow up to some internal maximum, and thus lead to latencies around 0.1
 seconds, which is better than SATA by an order of magnitude.
 Of course, the same effect can be observed at the delay diagram and the
 detailed throughput diagram:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sas/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sas.g03.latency.points.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sas/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sas.g00.thrp.completed.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Subsection
Overload with (Derived) Natural Loads
\begin_inset CommandInset label
LatexCommand label
name "sub:Overload-with-(Derived)"

\end_inset


\end_layout

\begin_layout Standard
Overload tests with natural loads are probably the most interesting ones.
 They can tell you a lot upon both the test candidate, as well as upon your
 application.
\end_layout

\begin_layout Standard
In many cases, natural loads are varying very much over time.
 Frequently, the variance in the IOPS rate can span an order of magnitude,
 or even more.
 Typical reasons are cron jobs or nightly backups on an otherwise (almost)
 idle server.
\end_layout

\begin_layout Standard
In order to find the peaks in the load, you have to draw 
\family typewriter
blktrace
\family default
s from production servers for at least 24h.
 After conversion to 
\family typewriter
*.load.gz
\family default
, it is rather cumbersome to find the load peaks by hand in order to get
 suitable input data for 
\emph on
overload
\emph default
 tests.
 Thus, we have developed the script 
\family typewriter
create_derived_load.sh
\family default
 which will do the following for you:
\end_layout

\begin_layout Enumerate
It splits the input file(s) into snippets of equal length in realtime (by
 default, 5min length each).
\end_layout

\begin_layout Enumerate
It sorts the snippets according to IOPS in reverse order, i.e.
 the 
\begin_inset Quotes eld
\end_inset

heaviest
\begin_inset Quotes erd
\end_inset

 snippet will come first.
\end_layout

\begin_layout Enumerate
It distributes the snippets to a number of output files in a round-robin
 fashion (while adjusting the timestamps to the new order).
\end_layout

\begin_layout Standard
As a result, the output files will start with the heaviest IOPS load first,
 and later declining.
 This methodology has a number of advantages:
\end_layout

\begin_layout Itemize
The overload part is easily accessible at the beginning of each output file
 (interesting part first).
\end_layout

\begin_layout Itemize
Thanks to the distribution to many output files, higher replay parallelism
 is possible in an uncorrolated way.
\end_layout

\begin_layout Itemize
In case your test candidate should get overloaded: you can easily see its
 
\emph on
recovery behaviour
\emph default
 from overload, thanks to the declination of the load over time.
 Will it recover at all, and how fast?
\end_layout

\begin_layout Standard
The latter is an important property of any enterprise-grade storage system.
\end_layout

\begin_layout Standard
Here are some example comparisons between an old SATA system and a newer
 SAS system, both equipped with hardware RAID-6, and loaded with a replay
 parallelism of 4.
 You can find the complete results with additional graphics as a tarball
 at 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

http://www.blkreplay.org/examples/
\end_layout

\end_inset

.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sas/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sas.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
It is interesting to observe the behaviour of the SATA system in contrast
 to the high-performance SAS: at first, the real IOPS rate (orange) cannot
 catch up with the demanded rate (green).
 However, when the demanded rate decreases under the green line after some
 time, the orange line becomes higher than the green one until the backlog
 has been worked up.
 Finally, the system can (almost) run in sync with the lower demands, and
 the orange line is hiding the green one.
\end_layout

\begin_layout Standard
The same effects can also be seen in the corresponding delay diagrams, where
 you can even notice multiple top lines stemming from overlay of replays
 in parallel, and even watch their different fallback points:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sata.bbu-1.vg-test.g04.delay.setpoint.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sas/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sas.vg-test.g04.delay.setpoint.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Section
Influence of Replay Parameters
\end_layout

\begin_layout Standard
The following examples are run on the rather slow SATA system.
 By default, we use 
\family typewriter
threads=128
\family default
 and 4 
\family typewriter
blkreplay
\family default
 instances, operating in parallel on 4 LVM devices created from the same
 volume group.
 Therefore, the total IO parallelism is 
\begin_inset Formula $128*4=512$
\end_inset

.
 Exceptions from that are denoted.
\end_layout

\begin_layout Subsection
Influence of Request Ordering
\begin_inset CommandInset label
LatexCommand label
name "sub:Influence-of-Request"

\end_inset


\end_layout

\begin_layout Standard
We start with the simplest case: mode 
\family typewriter
--with-conflicts
\family default
 fires up all requests without guaranteeing any order (besides that requests
 are never started too early), which may lead to damaged IO.
 Here is the sonar diagram as well as the throughput diagram:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
It is clear that the SATA cannot catch up with the high demands here.
 But why? Is it the damaged IO?
\end_layout

\begin_layout Standard
A first hint is the rather 
\begin_inset Quotes eld
\end_inset

thick
\begin_inset Quotes erd
\end_inset

 red 
\begin_inset Quotes eld
\end_inset

cloud carpet
\begin_inset Quotes erd
\end_inset

 in the latency diagram near 1s, indicating some queueing behaviour.
 Interestingly, this carpet falls a little down at the end, when the replay
 parallelism decreases due to different termination of different 
\family typewriter
blkreplay
\family default
 instances.
\end_layout

\begin_layout Standard
Now we turn to 
\family typewriter
--with-drop
\family default
.
 Surprisingly, there is only a small difference, although damaged IO is
 avoided at the cost of some lost requests:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-drop/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-drop.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-drop/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-drop.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
This is a clear indication that damaged IO cannot be the reason for the
 delays, since it is avoided by 
\family typewriter
--with-drop
\family default
.
\end_layout

\begin_layout Standard
Now we turn to 
\family typewriter
--with-partial
\family default
.
 Most of the time, there is almost the same behaviour as before.
 Only the tail is different, because the actual running times are more diverging
 between the four 
\family typewriter
blkreplay
\family default
 instances started in parallel:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
The tail may look disturbing to some people.
 It has a rather 
\emph on
low
\emph default
 IOPS rate.
 In case of delays, 
\family typewriter
blkreplay
\family default
 should try to catch up as fast as possible, shouldn't it? What the heck
 is going on there? Is there a bug in 
\family typewriter
blkreplay
\family default
?
\end_layout

\begin_layout Standard
Examination of 
\family typewriter
*.lv-data3.replay.gz
\family default
 from 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

http://www.blkreplay.org/examples/
\end_layout

\end_inset

 tells us that the real last tail consists almost exclusively of pushed-back
 requests.
 In detail, there are lots of requests depending on each other 
\emph on
transitively
\emph default
, reaching over many generations.
 They have accumulated over a long time, because 
\family typewriter
--with-partial
\family default
 is 
\emph on
defined
\emph default
 to do so.
 The tail is nothing but catch-up of pushed-back requests which could not
 be run earlier, in order to obey the storage semantics.
 Many of them are mutually hindering each other.
 The ordinary non-conflicting requests form a different class than then
 conflicting ones, leading to a 2-class society with different chances to
 be processed (unfairness).
 Thus one of the both classes finishes earlier than the other.
 This is a disadvantage of 
\family typewriter
--with-partial
\family default
 you should know.
\end_layout

\begin_layout Standard
On the other hand, this can be turned into an advantage, because you can
 use 
\family typewriter
--with-partial
\family default
 for detection of such a behaviour.
 Think of the race between Achilles and the turtle.
 If you find a much slower turtle, your test candidate has probably some
 problems with that 
\emph on
specific
\emph default
 load, but not necessarily with other loads at the same IOPS level.
\end_layout

\begin_layout Standard
Hint: the 2-class society caused by 
\family typewriter
--with-partial
\family default
 is often visible in the delay diagrams.
 Check out yourself!
\end_layout

\begin_layout Standard
Let us return to the comparion of request ordering modes: more prominent
 differences can be found when we turn to 
\family typewriter
--with-ordering
\family default
.
 Now the system can catch up 
\emph on
in average
\emph default
 (but not in general at load peaks).
 The sonar diagram shows that the 
\begin_inset Quotes eld
\end_inset

thick clouds
\begin_inset Quotes erd
\end_inset

 at about 1s have fallen down and have become thinner in densitiy:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-ordering/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-ordering.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-ordering/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-ordering.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
This rises the question for reasons.
 Why should 
\family typewriter
--with-ordering
\family default
 be better than 
\family typewriter
--with-partial
\family default
 or even 
\family typewriter
--with-drop
\family default
? Full ordering will usually 
\emph on
reduce
\emph default
 the IO parallelism when compared to partial ordering.
 The full answer is in sections 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Influence-of-Number"

\end_inset

 and 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Influence-of-Bottlenecks"

\end_inset

, but for now you can take the small 
\begin_inset Quotes eld
\end_inset

gaps
\begin_inset Quotes erd
\end_inset

 on the x axis as a hint that there must be something related to small micro-sta
lls, which seem to 
\emph on
accelerate
\emph default
 overall throughput in some 
\series bold
counter-intuitive
\series default
 way.
 The micro-stalls are easily explainable by the submission delays caused
 by 
\family typewriter
--with-ordering
\family default
 , reducing the IO parallelism in case of conflicts.
 But why does that 
\emph on
accelerate
\emph default
 the overall throughput?
\end_layout

\begin_layout Standard
Before proceeding, we take a closer look at the micro-stall effect.
 It is better visible in 
\family typewriter
*.latency.flying
\family default
 showing the 
\emph on
actually achieved IO parallelism
\emph default
 during the replay.
 Here is the comparison between 
\family typewriter
--with-conflicts
\family default
, and 
\family typewriter
--with-drop
\family default
, and 
\family typewriter
--with-
\family default
partial:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-drop/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-drop.strong-1.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
The sum of both reads and writes (olive line) indicates that the RAID controller
 is filled with a maximum of 
\begin_inset Formula $128*4=512$
\end_inset

 requests (caused by 
\family typewriter
threads=128
\family default
 and a replay parallelism of 
\begin_inset Formula $4$
\end_inset

) most of the time, except at the tail.
\end_layout

\begin_layout Standard
And now, compare these with 
\family typewriter
--with-ordering
\family default
:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-ordering/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-ordering.strong-1.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
It is clear that by definition of 
\family typewriter
ordering
\family default
, some kind of stalls have to be introduced to guarantee some virtual 
\begin_inset Quotes eld
\end_inset

strict storage semantics
\begin_inset Quotes erd
\end_inset

.
 It is also clear that these micro-stalls will sometimes reduce the actual
 IO parallelism, which is clearly visible in the graphics.
 However, can these micro-stalls be made 
\emph on
responsible
\emph default
 for an 
\emph on
increase
\emph default
 of overall throughput?
\end_layout

\begin_layout Standard
In order to investigate that, we turn to the influence of the 
\family typewriter
threads=
\family default
 parameter.
\end_layout

\begin_layout Subsection
Influence of Number of Threads
\begin_inset CommandInset label
LatexCommand label
name "sub:Influence-of-Number"

\end_inset


\end_layout

\begin_layout Standard
The following throughput graphics are a direct comparison between 8, 16,
 32, 64, 128, 256 and 512 threads on the SATA system using 
\family typewriter
--with-conflicts
\family default
, which guarantees the best possible IO parallelism and avoids any non-linear
 influences from any conflict-handling strategies:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-008/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-008.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-016/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-016.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-064/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-064.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-256/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-256.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-512/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-512.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
It is clear that a too low number of threads, such as 8, can hinder overall
 throughput (even when in reality the total IO parallelism is 
\begin_inset Formula $8*4=32$
\end_inset

 by taking the number of 
\family typewriter
blkreplay
\family default
 instances into account).
 It is therefore no wonder that 16 through 64 threads perform better than
 8.
 But, why is there a counter-productive break-down when increasing the number
 of threads after that? The effect is rather strong: 512 threads will decrease
 throughput almost by a factor of three when compared to the 
\begin_inset Quotes eld
\end_inset

best
\begin_inset Quotes erd
\end_inset

 setting.
 This is no peanuts! What the hell is going on here?
\end_layout

\begin_layout Standard
The answer can only be buried in the internals of the hardware and/or its
 firmware and/or its driver.
 Typically, some systems have problems when filled with too many requests
 in parallel.
 Here, we see a typical 
\series bold
non-linear behaviour
\series default
 as explained in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Replay"

\end_inset

.
\end_layout

\begin_layout Standard
Consequences: the 
\family typewriter
threads=
\family default
 parameter can be very important.
 And its 
\begin_inset Quotes eld
\end_inset

optimum
\begin_inset Quotes erd
\end_inset

 can non-linearly depend from the request ordering parameters.
 Here, you see a proof for the warnings posed in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Replay"

\end_inset

.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 
\emph on
Never
\emph default
 adjust the 
\family typewriter
threads=
\family default
 parameter to the value delivering the 
\begin_inset Quotes eld
\end_inset

best
\begin_inset Quotes erd
\end_inset

 performance when comparing different systems / vendors.
 Doing so will be a 
\emph on
lie(!)
\emph default
 to your audience! Always use the 
\family typewriter
threads=
\family default
 setting which corresponds to your 
\series bold
real application
\series default
 based on 
\series bold
knowledge
\series default
 (by examining the submission IO parallelism at the original 
\family typewriter
blktrace
\family default
 recording site, e.g.
 by looking at the 
\family typewriter
*.flying
\family default
 graphics produced from the original 
\family typewriter
blktrace
\family default
 recording), and 
\series bold
not
\series default
 based on any assumptions which could be faked in any direction! Be responsible
 and tell your audience 
\emph on
why
\emph default
 you select a specific value, and what would happen if you selected a different
 one! Or, provide a series of different 
\family typewriter
threads=
\family default
 settings telling the 
\emph on
whole
\emph default
 story!
\end_layout

\begin_layout Subsection
Influence of BBU units at RAID controllers
\end_layout

\begin_layout Standard
The following comparison is an example for the performace degradation resulting
 from defective (or missing) BBU units:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-0/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-0.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
In essence, the missing BBU will slow down write requests.
 This can be easily seen when comparing the sonar diagrams (just look at
 the colors and where they appear):
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-0/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-0.vg-test.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
More interesting differences can be found in the tarball at 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

http://www.blkreplay.org/examples/
\end_layout

\end_inset

.
\end_layout

\begin_layout Subsection
Influence of Bottlenecks
\begin_inset CommandInset label
LatexCommand label
name "sub:Influence-of-Bottlenecks"

\end_inset


\end_layout

\begin_layout Standard
The expert option 
\family typewriter
--bottleneck=<number>
\family default
 can be used to limit the number of requests submitted to the worker threads,
 which is usually higher than the number of threads.
 In each individual pipeline connecting the main thread to each worker thread,
 up to 8 requests may be queued in advance.
\end_layout

\begin_layout Standard
In the following example using 
\family typewriter
--with-conflicts
\family default
 and 
\family typewriter
--threads=128
\family default
, we also limit the number of requests by 
\family typewriter
--bottleneck=128
\family default
, which leads to almost the same effect as the above discussion regarding
 
\family typewriter
--with-ordering
\family default
:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.bbu-1.vg-test.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
This result is unexpected.
 In theory, it should not make any difference whether there is 1 request
 in between the main thread and a worker, or many.
 However, in practice it makes a difference: the 
\emph on
communication latencies
\emph default
 over the pipelines are avoided when the pipeline remains always filled
 with some requests in advance.
\end_layout

\begin_layout Standard
When 
\family typewriter
--bottleneck
\family default
 has exactly the same value as 
\family typewriter
--threads
\family default
, a 
\begin_inset Quotes eld
\end_inset

logical request
\begin_inset Quotes erd
\end_inset

 can be either working in a worker thread, or being processed by the main
 thread, or underway over the pipes.
 It cannot be present at more than one of these places at the same time.
\end_layout

\begin_layout Standard
An example, just for demonstration of the principle: Assume that the IO
 system can catch up (at least at the beginning), and that the IO latencies
 are 
\emph on
constant
\emph default
.
 Assume that the communication latencies are exactly the same constant.
 What will be the effect? One half of the requests will always reside in
 the IO system, but the other half will reside in the pipe communication
 infrastructure, in average.
 Did you get the point?
\end_layout

\begin_layout Standard
Now, assume that both the IO latencies as well as the communication latencies
 are varying (e.g.
 due to kernel scheduling etc).
 Then the queue length might be staggering around, as observed in the following
 graphics (first is the default bottleneck setting 
\family typewriter
--bottleneck=$(( threads * 8 ))
\family default
, second is 
\family typewriter
--bottleneck=${threads}
\family default
):
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.bbu-1.vg-test.g08.latency.flying.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
As before, the SATA system shows its counter-productive behaviour when too
 many IO requests are fired up in parallel.
 Once the number of requests has reduced, it works better.
 But somewhen it staggers to the other extreme, flipping around.
 And to the opposite.
\end_layout

\begin_layout Standard
Interesting side note: similar behaviour can be observed in real-life production
 systems, because natural loads (as opposed to standard benchmark tests)
 often vary by orders of magnitude.
 When spuriously plagued by seemingly 
\begin_inset Quotes eld
\end_inset

unmotivated
\begin_inset Quotes erd
\end_inset

 incidents, look for places where non-linear and counter-productive behaviour
 may cause unexpected (and possibly self-amplifying) effects!
\end_layout

\begin_layout Subsection
Influence of 
\family typewriter
strong
\family default
 Mode
\begin_inset CommandInset label
LatexCommand label
name "sub:Influence-of-strong"

\end_inset


\end_layout

\begin_layout Standard
Here is a direct comparison between all three 
\family typewriter
--strong=
\family default
 modes, when combined with 
\family typewriter
--with-partial
\family default
.
 The effect is highly dependable from the load, and usually rather small
 (when compared to the drastic effects demonstrated in some of the previous
 sections).
 But don't take this as given! Better check for it.
 First, we compare at 
\family typewriter
--threads=128
\family default
:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
The 
\begin_inset Quotes eld
\end_inset

long tail
\begin_inset Quotes erd
\end_inset

 explained in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Influence-of-Request"

\end_inset

 is disturbing.
 The 2-class treatment inherent to 
\family typewriter
--with-partial
\family default
 leads to a high number of 
\emph on
transitively
\emph default
 pushed-back requests, racing against the ordinary requests.
 This race is in some sense 
\begin_inset Quotes eld
\end_inset

non-deterministic
\begin_inset Quotes erd
\end_inset

 and may depend non-linearly from the 
\family typewriter
--strong=
\family default
 mode.
 Therefore, we reduce the 2-class treatment by turning to 
\family typewriter
--threads=32
\family default
:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
Now the tail effect has almost vanished, because the better throughput leads
 to less queueing, which in turn decrease the chances for a conflict to
 actually occur during the time window 
\family typewriter
--ahead_limit=
\family default
 (default 
\begin_inset Formula $1$
\end_inset

 second).
 In consequence, any self-amplifying 
\begin_inset Quotes eld
\end_inset

traffic jam effects
\begin_inset Quotes erd
\end_inset

 are much lower.
 As expected, the actual throughput decreases with 
\family typewriter
strong
\family default
er mode, but the effect is really tiny, probably below measurement tolerances.
 In order to really see something, we take a closer look at the sonar diagrams:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-032.test-sata.bbu-1.vg-test.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-032.test-sata.bbu-1.vg-test.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-032.test-sata.bbu-1.vg-test.g01.latency.realtime.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
As expected, the number of 
\family typewriter
Write Pushbacks
\family default
 is increasing with 
\family typewriter
--strong=
\family default
 mode, and 
\family typewriter
Read Pushbacks
\family default
 come also into play.
 But the differences are minor.
 A litte more impressive are the differences at 
\family typewriter
*.latency.flying
\family default
, where the 
\begin_inset Quotes eld
\end_inset

staggering effect
\begin_inset Quotes erd
\end_inset

 is clearly increasing with growing 
\family typewriter
--strong=
\family default
 mode:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-032.test-sata.bbu-1.vg-test.g08.latency.flying.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-032.test-sata.bbu-1.vg-test.g08.latency.flying.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-032.test-sata.bbu-1.vg-test.g08.latency.flying.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
More details can be explored when looking at the purple 
\begin_inset Quotes eld
\end_inset

Write Pushes
\begin_inset Quotes erd
\end_inset

 / deep-blue 
\begin_inset Quotes eld
\end_inset

Read Pushes
\begin_inset Quotes erd
\end_inset

 displayed in further graphics at 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

http://www.blkreplay.org/examples/
\end_layout

\end_inset

.
\end_layout

\begin_layout Chapter
\start_of_appendix
Config File Parameters
\begin_inset CommandInset label
LatexCommand label
name "chap:Config-File-Parameters"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstset{ %
\end_layout

\begin_layout Plain Layout

  language=bash,
\end_layout

\begin_layout Plain Layout

  basicstyle=
\backslash
tt
\backslash
footnotesize,
\end_layout

\begin_layout Plain Layout

%  numbers=left,
\end_layout

\begin_layout Plain Layout

%  numberstyle=
\backslash
tiny
\backslash
color{gray},
\end_layout

\begin_layout Plain Layout

%  stepnumber=2,
\end_layout

\begin_layout Plain Layout

%  numbersep=5pt,
\end_layout

\begin_layout Plain Layout

%backgroundcolor=
\backslash
color{white},
\end_layout

\begin_layout Plain Layout

  showspaces=false,
\end_layout

\begin_layout Plain Layout

  showstringspaces=true,
\end_layout

\begin_layout Plain Layout

  showtabs=true,
\end_layout

\begin_layout Plain Layout

%frame=single,
\end_layout

\begin_layout Plain Layout

%rulecolor=
\backslash
color{black},
\end_layout

\begin_layout Plain Layout

  tabsize=8,
\end_layout

\begin_layout Plain Layout

  captionpos=t,
\end_layout

\begin_layout Plain Layout

  breaklines=true,
\end_layout

\begin_layout Plain Layout

  breakatwhitespace=false,
\end_layout

\begin_layout Plain Layout

  title=
\backslash
lstname,
\end_layout

\begin_layout Plain Layout

  keywordstyle=
\backslash
color{brown},
\end_layout

\begin_layout Plain Layout

  commentstyle=
\backslash
color{OliveGreen},
\end_layout

\begin_layout Plain Layout

}
\end_layout

\end_inset


\end_layout

\begin_layout Section
Basic Parameters
\end_layout

\begin_layout Subsection
File 
\family typewriter
user_modules.conf
\family default
:
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/user_modules.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-main.conf
\family default
:
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-main.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Section
Ordinary Module Parameters
\begin_inset CommandInset label
LatexCommand label
name "sec:Module-Parameters"

\end_inset


\end_layout

\begin_layout Standard
The following list is in the order of activation.
\end_layout

\begin_layout Subsection
File 
\family typewriter
default-recreate_lvm.conf
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-recreate_lvm.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-create_lv.conf
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-create_lv.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-iscsi_target_iet.conf
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-iscsi_target_iet.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-iscsi_initiator.conf
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-iscsi_initiator.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-scheduler.conf
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-scheduler.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-wipe.conf
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-wipe.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-bbu_megaraid.conf
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-wipe.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-graph.conf
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-graph.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Section
Pipe Module Parameters
\begin_inset CommandInset label
LatexCommand label
name "sec:Pipe-Module-Parameters"

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-pipe_repeat.conf
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-pipe_repeat.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-pipe_slip.conf
\begin_inset CommandInset label
LatexCommand label
name "sub:File-default-pipe_slip.conf"

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-pipe_slip.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-pipe_subst.conf
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-pipe_subst.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-pipe_spread.conf
\begin_inset CommandInset label
LatexCommand label
name "sub:File-default-pipe_spread.conf"

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-pipe_spread.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-pipe_resize.conf
\begin_inset CommandInset label
LatexCommand label
name "sub:File-default-pipe_resize.conf"

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-pipe_resize.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Subsection
File 
\family typewriter
default-pipe_cmd.conf
\end_layout

\begin_layout Standard
\noindent
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting[language=bash]{example-run/default-pipe_cmd.conf}
\end_layout

\end_inset


\end_layout

\begin_layout Chapter
File Format
\end_layout

\begin_layout Standard
The 
\family typewriter
*.load.gz
\family default
 and 
\family typewriter
*.replay.gz
\family default
 files are in essence compressed CSV files, using 
\family typewriter

\begin_inset Quotes eld
\end_inset

;
\begin_inset Quotes erd
\end_inset


\family default
 as delimiter.
 However, errors and comments can also occur in the file, violating the
 classical CSV format.
\end_layout

\begin_layout Standard
In order to extract classical CSV, use the following command:
\end_layout

\begin_layout Quotation

\family typewriter
zgrep 
\begin_inset Quotes eld
\end_inset

;
\begin_inset Quotes erd
\end_inset

 
\emph on
myname
\emph default
.replay.gz | less
\end_layout

\begin_layout Standard
The first line of the output contains the column names (and could be removed
 by inserting 
\family typewriter
| tail -n +2
\family default
 into the pipeline).
 In 
\emph on
theory
\emph default
, you could even use MS Excel to work on the above output.
 However, in practice the file will be so extremely large that interactive
 workstation tools will likely start swapping your system to death, or even
 bombing.
\end_layout

\begin_layout Standard
So the usual way to operate on the output is a 
\begin_inset Quotes eld
\end_inset

pipe and filters style
\begin_inset Quotes erd
\end_inset

, using stream-processing tools like 
\family typewriter
sed
\family default
, 
\family typewriter
awk
\family default
, 
\family typewriter
perl
\family default
, and so on.
\end_layout

\begin_layout Standard
Both 
\family typewriter
*.load.gz
\family default
 and 
\family typewriter
*.replay.gz
\family default
 use the same format.
 However, 
\family typewriter
*.load.gz
\family default
 
\series bold
\emph on
must
\series default
\emph default
 be sorted numerically according to the timestamp, while 
\family typewriter
*.replay.gz
\family default
 is usually unsorted (roughly showing the order of request completion in
 most cases, which is different from the submission timestamp ordering).
 In order to re-use 
\family typewriter
*.replay.gz
\family default
 again as input for another run of 
\family typewriter
blkreplay
\family default
, you must sort it with 
\family typewriter
sort -n
\family default
.
 In some cases, public 
\family typewriter
blktrace
\family default
 recordings at 
\family typewriter
www.blktrace.org/loads/
\family default
 will contain only constants 0.0 as delays and latencies.
\end_layout

\begin_layout Standard
Feel free to filter your results, or create your own artifical loads.
\end_layout

\begin_layout Standard
\noindent
Enjoy.
\end_layout

\begin_layout Chapter
Validation of the 
\family typewriter
blkreplay
\family default
 Tool
\end_layout

\begin_layout Standard
Some effort has been spent to check that 
\family typewriter
blkreplay
\family default
 really produces the intended kind of IO load on the target system, and
 that its measurement results are likely to be valid.
 However, there is no guarantee against hidden bugs, misconfigurations,
 or other types of accidents.
 See also the disclaimer of the GPL license.
\end_layout

\begin_layout Standard
The following tools try the 
\emph on
help
\emph default
 you by making 
\family typewriter
blkreplay
\family default
 as transparent to the public as possible, but cannot form an absolute guarantee
 against all possible sources of errors.
\end_layout

\begin_layout Standard
Please report any bugs to the author, or, even better, send patches or git
 pull requests.
\end_layout

\begin_layout Section
Running 
\family typewriter
blktrace
\family default
 during 
\family typewriter
blkreplay
\end_layout

\begin_layout Standard
The timeliness of IO requests is already measured and displayed by the 
\family typewriter
*.latency.*
\family default
 graphics.
 The following steps will thus concentrate on 
\emph on
completeness
\emph default
.
\end_layout

\begin_layout Standard
In order to check that all IO requests present in a 
\family typewriter
*.load.gz
\family default
 file are actually applied to the test candidate, you can (and should) re-record
 the IO load during an execution of 
\family typewriter
blkreplay
\family default
 via 
\family typewriter
blktrace
\family default
.
 The following steps are to be done manually:
\end_layout

\begin_layout Enumerate
Start 
\family typewriter
blktrace
\family default
 manually, as described in chapter 
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:How-to-use"

\end_inset

, 
\emph on
before
\emph default
 you start 
\family typewriter
tree-replay.sh
\family default
.
 Be sure to catch all involved devices on all involved hosts.
\end_layout

\begin_layout Enumerate
Either kill 
\family typewriter
blktrace
\family default
 with Ctrl-C 
\emph on
after
\emph default
 tree-replay.sh is done, or use the 
\family typewriter
-w
\family default
 option to specify a 
\emph on
strictly longer
\emph default
 (plus some safety margin) recording time.
\begin_inset Newline newline
\end_inset

Attention! some versions of 
\family typewriter
blktrace
\family default
 seem to loose some requests in some buffers, at least under some conditions.
 Usually the number of lost events is rather low, about 10.
 Keep that in mind if you later discover some purpoted discrepancies!
\end_layout

\begin_layout Enumerate
Copy the resulting 
\family typewriter
*.blktrace.[0-9]+
\family default
 file to your workstation.
\end_layout

\begin_layout Enumerate
Invoke 
\family typewriter
/path/to/blkreplay/scripts/check_replay_against_blktrace.sh 
\emph on
myresult
\emph default
.replay.gz 
\emph on
mytrace-prefix
\family default
\emph default
 (exactly two parameters).
\begin_inset Newline newline
\end_inset

Hint: the coincidence between 
\family typewriter
\emph on
myresult
\emph default
.replay.gz
\family default
 and 
\family typewriter
\emph on
myload
\emph default
.load.gz
\family default
 must be checked separately, using tools like 
\family typewriter
cut
\family default
 and 
\family typewriter
diff
\family default
.
\end_layout

\begin_layout Enumerate
Enjoy the result in form of a wide side-by-side 
\family typewriter
diff
\family default
 output.
\end_layout

\begin_layout Standard
Hint: no output means that no differences have been found.
 The timeliness as well as the order of requests is 
\emph on
not
\emph default
 checked.
 Any difference means that a request is missing somewhere, or has been altered
 in some way.
 Expect some minor differences (usully less than 1%, but sometimes more)
 from request splitting by some device drivers.
\end_layout

\begin_layout Standard
Keep in mind that sometimes 
\family typewriter
blktrace
\family default
 looses some events, leading to 
\begin_inset Quotes eld
\end_inset

false positives
\begin_inset Quotes erd
\end_inset

.
 You should take this as a validation of the validation tool ;)
\end_layout

\begin_layout Section
Verbosity Graphics
\end_layout

\begin_layout Standard
Experts only.
 It is easy to draw wrong conclusions from this.
\end_layout

\begin_layout Standard
Enable the following parameters: 
\family typewriter
verbose=10;
\family default
 
\family typewriter
enable_compress_ssh=1; graph_options="--static --verbose"
\family default
 in your respective 
\family typewriter
*.conf
\family default
 files.
\end_layout

\begin_layout Standard
The resulting 
\family typewriter
*.replay.gz
\family default
 file will now contain 
\emph on
huge masses
\emph default
 of lines starting with 
\family typewriter
INFO:
\family default
 and carrying lots of internal variables, similar to debug / tracing output.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 The sheer masses of data written to 
\family typewriter
stdout
\family default
 (as such) my lead to distortions of your measurements, e.g.
 artificial delays or even hangs.
 Always compare to 
\family typewriter
--verbose=0
\family default
 under otherwise identical conditions, to be sure! In order to check whether
 
\family typewriter
stdout
\family default
 could have become a bottleneck, please search for the lines with 
\family typewriter
flush_total=n.nnnnnnnnn
\family default
 (some timing with nanosecond resolution).
 In particular, the main thread (additionally indicated by 
\family typewriter
role='main'
\family default
) should have spent much less time in 
\family typewriter
fflush(stdout)
\family default
 than total running time.
 When the network is the bottleneck, 
\family typewriter
enable_compress_ssh=1
\family default
 may help; otherwise disabling could be better -- please check.
\end_layout

\begin_layout Standard
\noindent
In addition, the following types of internal visualizations will be produced:
\end_layout

\begin_layout Description
\noindent

\family typewriter
*.submit_level.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

Shows the actual filling level of the pipelines between the main thread
 and the workers.
 The x axis is in units of requests, and in 
\emph on
completion order
\emph default
 (different from the submission order).
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g50.submit_level.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Description
\noindent

\family typewriter
*.pushback_level.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

Shows the current number of requests on the pushback list.
 Only meaningful at 
\family typewriter
--with-partial
\family default
.
\end_layout

\begin_layout Description
\noindent

\family typewriter
*.submit_ahead.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

Shows the time difference between submission to the pipelines (which is
 usually ahead of the execution time, but not always) and the intended execution
 timestamp.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g52.submit_ahead.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Don't draw wrong conclusions from negative values here! They 
\emph on
can
\emph default
 be a sign of 
\begin_inset Quotes eld
\end_inset

too late
\begin_inset Quotes erd
\end_inset

, but the 
\emph on
reason
\emph default
 is almost always in the test candidate, just when ordinary delays are produced.
\end_layout

\begin_layout Description
\noindent

\family typewriter
*.submit_lag.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The time difference between submission to the pipelines, and reception by
 the worker thread (aka communication latency).
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g53.submit_lag.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Description
\noindent

\family typewriter
*.submit_lag_cumul.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The same, but cumulized over time.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g53.submit_lag_cumul.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Notice than cumulation of time lags belonging to different threads is 
\begin_inset Quotes eld
\end_inset

unfair
\begin_inset Quotes erd
\end_inset

 by concept.
 Large values can regularly occur because of queueing in the submit pipelines.
 Complaints about high values make only sense in case of 
\family typewriter
$(( bottleneck <= threads )).
\end_layout

\begin_layout Description
\noindent

\family typewriter
*.answer_lag.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The time difference between submission to the answer pipeline caused by
 a worker thread, and reception by the main thread (aka communication latency).
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g54.answer_lag.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Description
\noindent

\family typewriter
*.answer_lag_cumul.png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

Dito, cumulated over time.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g54.answer_lag_cumul.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Similar warning as before: high values occur regularly in 
\family typewriter
--with-ordering
\family default
 mode.
 Only be startled in 
\emph on
some
\emph default
 other cases!
\end_layout

\begin_layout Description
\noindent

\family typewriter
*.input_wait[_cumul].png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The time differences before and after 
\family typewriter
fgets()
\family default
 (aka stalls in 
\family typewriter
stdin
\family default
).
 When these numbers get too high, this time you should be alerted quite
 rightly.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g55.input_wait.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g55.input_wait_cumul.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Description
\noindent

\family typewriter
*.answer_wait[_cumul].png
\family default
 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

The time difference of the main thread it is spending for answer wait.
 Not to be confused with communication latencies! High number will occur
 regularly.
 It is even a sign of good performance when the cumulation almost reaches
 the total processing time.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g56.answer_wait.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g56.answer_wait_cumul.png
	lyxscale 50
	width 100col%

\end_inset


\end_layout

\begin_layout Chapter
GNU Free Documentation License
\begin_inset CommandInset label
LatexCommand label
name "chap:GNU-FDL"

\end_inset


\end_layout

\begin_layout Standard
\noindent

\family typewriter
\size footnotesize
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting{fdl.txt}
\end_layout

\end_inset


\end_layout

\end_body
\end_document
