

@preamble{{\providecommand{\MaxMinAntSystem}{{$\cal MAX$--$\cal MIN$} {Ant} {System}} } # {\providecommand{\rpackage}[1]{{#1}} } # {\providecommand{\softwarepackage}[1]{{#1}} } # {\providecommand{\proglang}[1]{{#1}} }}

@string{and = { and }}

@string{lncs = {Lecture Notes in Computer Science}}

@string{add-berlin = { Berlin, Germany }}

@string{add-cham = { Cham, Switzerland }}

@string{add-heidelberg = { Heidelberg }}

@string{add-ny = { New York, NY }}

@string{aaaip-pub = {{AAAI} Press}}

@string{acm-pub = {ACM Press}}

@string{ieeep = {IEEE Press}}

@string{ieeep-ad = {Piscataway, NJ}}

@string{springer = {Springer}}

@string{iridia = {IRIDIA, Universit{\'e} Libre de Bruxelles, Belgium}}

@string{proc_of = {Proceedings of }}

@string{proc_of_the = proc_of # { the }}

@string{aaai = proc_of_the # {{AAAI} Conference on Artificial Intelligence}}

@string{cec = { Congress on Evolutionary Computation (CEC}}

@string{cec2006 = proc_of_the # {2006} # cec # { 2006)}}

@string{gecco = proc_of_the # {Genetic and Evolutionary Computation Conference, GECCO }}

@string{gecco2013 = gecco # {2013}}

@string{icml = { International Conference on Machine Learning, {ICML} }}

@string{icml2014 = proc_of_the # {31st} # icml # {2014}}

@string{acm-cs = {{ACM} Computing Surveys}}

@string{cor = {Computers \& Operations Research}}

@string{jair = {Journal of Artificial Intelligence Research}}

@string{joh = {Journal of Heuristics}}

@string{orp = {Operations Research Perspectives}}

@string{telo = {ACM Transactions on Evolutionary Learning and Optimization}}

@string{alba_e = { Alba, Enrique }}

@string{balaprakash = {  Prasanna Balaprakash }}

@string{bartz-beielstein = { Thomas Bartz-Beielstein }}

@string{battiti = { Roberto Battiti }}

@string{biedenkapp = { Biedenkapp, Andr{\'e} }}

@string{birattari = { Mauro Birattari }}

@string{blum = { Christian Blum }}

@string{branke = { J{\"u}rgen Branke }}

@string{chiarandini = { Marco Chiarandini }}

@string{desouza = { Marcelo {De Souza} }}

@string{dubois-lacoste = { J{\'e}r{\'e}mie Dubois-Lacoste }}

@string{fawcett = { Chris Fawcett }}

@string{fonseca = { Carlos M. Fonseca }}

@string{hamadi = { Youssef Hamadi }}

@string{hoos = { Holger H. Hoos }}

@string{hutter = { Frank Hutter }}

@string{lau_hc = { Hoong Chuin Lau }}

@string{leyton-brown = { Kevin Leyton-Brown }}

@string{lindauer_m = { Marius Thomas Lindauer }}

@string{lopez-ibanez = { Manuel L{\'o}pez-Ib{\'a}{\~n}ez }}

@string{mcgeoch_cc = { Catherine C. McGeoch }}

@string{montesdeoca = { Marco A. {Montes de Oca} }}

@string{paquete = { Lu{\'i}s Paquete }}

@string{pardalos = { Panos M. Pardalos }}

@string{perez_l = {  P{\'e}rez C{\'a}ceres, Leslie}}

@string{preuss_m = { Mike Preuss }}

@string{ritt = { Marcus Ritt}}

@string{schneider_m = { Marius Schneider }}

@string{schoenauer = { Marc Schoenauer }}

@string{stuetzle = { Thomas St{\"u}tzle }}

@string{yuan_z = { Zhi Yuan }}

@article{DesRitLopPer2021acviz,
  author = desouza # and # ritt # and # lopez-ibanez # and # perez_l,
  title = {{\softwarepackage{ACVIZ}}: A Tool for the Visual Analysis of
                  the Configuration of Algorithms with {\rpackage{irace}}},
  journal = orp,
  year = 2021,
  doi = {10.1016/j.orp.2021.100186},
  supplement = {https://zenodo.org/record/4714582},
  abstract = {This paper introduces acviz, a tool that helps to analyze the
                  automatic configuration of algorithms with irace. It provides
                  a visual representation of the configuration process,
                  allowing users to extract useful information, e.g. how the
                  configurations evolve over time. When test data is available,
                  acviz also shows the performance of each configuration on the
                  test instances. Using this visualization, users can analyze
                  and compare the quality of the resulting configurations and
                  observe the performance differences on training and test
                  instances.},
  volume = 8,
  pages = 100186
}

@article{FawHoos2015ablation,
  title = {Analysing Differences Between Algorithm Configurations
                  through Ablation},
  author = fawcett # and # hoos,
  journal = joh,
  pages = {431--458},
  volume = 22,
  number = 4,
  year = 2016
}

@article{HutHooLeyStu2009jair,
  author = hutter # and # hoos # and # leyton-brown # and # stuetzle,
  title = {{\softwarepackage{ParamILS}:} An Automatic Algorithm Configuration Framework},
  journal = jair,
  year = 2009,
  volume = 36,
  pages = {267--306},
  month = oct,
  doi = {10.1613/jair.2861}
}

@article{LopBraPaq2021telo,
  author = lopez-ibanez # and # branke # and # paquete,
  title = {Reproducibility in Evolutionary Computation},
  journal = telo,
  year = 2021,
  volume = 1,
  number = 4,
  pages = {1--21},
  doi = {10.1145/3466624},
  epub = {https://arxiv.org/abs/2102.03380},
  abstract = {Experimental studies are prevalent in Evolutionary
                  Computation (EC), and concerns about the reproducibility and
                  replicability of such studies have increased in recent times,
                  reflecting similar concerns in other scientific fields. In
                  this article, we suggest a classification of different types
                  of reproducibility that refines the badge system of the
                  Association of Computing Machinery (ACM) adopted by TELO. We
                  discuss, within the context of EC, the different types of
                  reproducibility as well as the concepts of artifact and
                  measurement, which are crucial for claiming
                  reproducibility. We identify cultural and technical obstacles
                  to reproducibility in the EC field. Finally, we provide
                  guidelines and suggest tools that may help to overcome some
                  of these reproducibility obstacles.},
  keywords = {Evolutionary Computation, Reproducibility, Empirical study,
                  Benchmarking}
}

@article{LopDubPerStuBir2016irace,
  author = lopez-ibanez # and # dubois-lacoste # and # perez_l # and # stuetzle # and # birattari,
  title = {The {\rpackage{irace}} Package: Iterated Racing for Automatic
                  Algorithm Configuration},
  journal = orp,
  year = 2016,
  supplement = {http://iridia.ulb.ac.be/supp/IridiaSupp2016-003/},
  doi = {10.1016/j.orp.2016.09.002},
  volume = 3,
  pages = {43--58}
}

@article{McG1992vrt,
  author = mcgeoch_cc,
  title = {Analyzing Algorithms by Simulation: Variance Reduction
                  Techniques and Simulation Speedups},
  abstract = {Although experimental studies have been widely applied to the
                  investigation of algorithm performance, very little attention
                  has been given to experimental method in this area. This is
                  unfortunate, since much can be done to improve the quality of
                  the data obtained; often, much improvement may be needed for
                  the data to be useful. This paper gives a tutorial discussion
                  of two aspects of good experimental technique: the use of
                  variance reduction techniques and simulation speedups in
                  algorithm studies.  In an illustrative study, application of
                  variance reduction techniques produces a decrease in variance
                  by a factor 1000 in one case, giving a dramatic improvement
                  in the precision of experimental results. Furthermore, the
                  complexity of the simulation program is improved from
                  $\Theta(m n/H_n)$ to $\Theta(m + n \log n)$ (where $m$ is
                  typically much larger than $n$), giving a much faster
                  simulation program and therefore more data per unit of
                  computation time. The general application of variance
                  reduction techniques is also discussed for a variety of
                  algorithm problem domains.},
  volume = 24,
  doi = {10.1145/130844.130853},
  number = 2,
  journal = acm-cs,
  year = 1992,
  keywords = {experimental analysis of algorithms, move-to-front rule,
                  self-organizing sequential search, statistical analysis of
                  algorithms, transpose rule, variance reduction techniques},
  pages = {195--212}
}

@article{SouRitLop2021cap,
  author = desouza # and # ritt # and # lopez-ibanez,
  title = {Capping Methods for the Automatic Configuration of
                  Optimization Algorithms},
  journal = cor,
  doi = {10.1016/j.cor.2021.105615},
  year = 2022,
  volume = 139,
  pages = 105615,
  supplement = {https://github.com/souzamarcelo/supp-cor-capopt},
  abstract = {Automatic configuration techniques are widely and
                  successfully used to find good parameter settings for
                  optimization algorithms. Configuration is costly, because it
                  is necessary to evaluate many configurations on different
                  instances. For decision problems, when the objective is to
                  minimize the running time of the algorithm, many
                  configurators implement capping methods to discard poor
                  configurations early. Such methods are not directly
                  applicable to optimization problems, when the objective is to
                  optimize the cost of the best solution found, given a
                  predefined running time limit. We propose new capping methods
                  for the automatic configuration of optimization
                  algorithms. They use the previous executions to determine a
                  performance envelope, which is used to evaluate new
                  executions and cap those that do not satisfy the envelope
                  conditions. We integrate the capping methods into the irace
                  configurator and evaluate them on different optimization
                  scenarios. Our results show that the proposed methods can
                  save from about 5\% to 78\% of the configuration effort,
                  while finding configurations of the same quality. Based on
                  the computational analysis, we identify two conservative and
                  two aggressive methods, that save an average of about 20\%
                  and 45\% of the configuration effort, respectively. We also
                  provide evidence that capping can help to better use the
                  available budget in scenarios with a configuration time
                  limit.}
}

@inproceedings{BieLinEggFraFawHoo2017,
  author = biedenkapp # and # lindauer_m # and # {Eggensperger,
                  Katharina} # and # hutter # and # fawcett # and # hoos,
  title = {Efficient Parameter Importance Analysis via Ablation with
                  Surrogates},
  crossref = {AAAI2017},
  doi = {10.1609/aaai.v31i1.10657}
}

@incollection{BirYuaBal2010:emaoa,
  author = birattari # and # yuan_z # and # balaprakash # and # stuetzle,
  title = {{F}-Race and Iterated {F}-Race: An Overview},
  pages = {311--336},
  crossref = {BarChiPaqPre2010emaoa},
  keywords = {F-race, iterated F-race, irace, tuning},
  doi = {10.1007/978-3-642-02538-9_13}
}

@incollection{FonPaqLop06:hypervolume,
  author = fonseca # and # paquete # and # lopez-ibanez,
  title = {An improved dimension-\hspace{0pt}sweep
                  algorithm for the hypervolume indicator},
  crossref = {CEC2006},
  pages = {1157--1163},
  doi = {10.1109/CEC.2006.1688440},
  pdf = {FonPaqLop06-hypervolume.pdf},
  abstract = {This paper presents a recursive, dimension-sweep
                  algorithm for computing the hypervolume indicator of
                  the quality of a set of $n$ non-dominated points in
                  $d>2$ dimensions. It improves upon the existing HSO
                  (Hypervolume by Slicing Objectives) algorithm by
                  pruning the recursion tree to avoid repeated
                  dominance checks and the recalculation of partial
                  hypervolumes. Additionally, it incorporates a recent
                  result for the three-dimensional special case.  The
                  proposed algorithm achieves $O(n^{d-2} \log n)$ time
                  and linear space complexity in the worst-case, but
                  experimental results show that the pruning
                  techniques used may reduce the time complexity
                  exponent even further.}
}

@incollection{HutHooLey2013lion,
  author = hutter # and # hoos # and # leyton-brown,
  title = {Identifying Key Algorithm Parameters and Instance Features
                  using Forward Selection},
  crossref = {LION2013},
  pages = {364--381},
  doi = {10.1007/978-3-642-44973-4_40},
  keywords = {parameter importance}
}

@inproceedings{HutHooLey2014icml,
  author = hutter # and # hoos # and # leyton-brown,
  title = {An Efficient Approach for Assessing Hyperparameter
                  Importance},
  crossref = {ICML2014},
  pages = {754--762},
  url = {https://proceedings.mlr.press/v32/hutter14.html},
  keywords = {fANOVA, parameter importance}
}

@techreport{IRIDIA-2004-001,
  author = birattari,
  title = {On the Estimation of the Expected Performance of a Metaheuristic on a Class of Instances. How Many Instances, How Many Runs?},
  institution = iridia,
  year = 2004,
  number = {TR/IRIDIA/2004-001}
}

@techreport{LopDubStu2011irace,
  author = lopez-ibanez # and # dubois-lacoste # and # stuetzle # and # birattari,
  title = {The {\rpackage{irace}} package, Iterated Race for Automatic
                  Algorithm Configuration},
  institution = iridia,
  year = 2011,
  number = {TR/IRIDIA/2011-004},
  url = {http://iridia.ulb.ac.be/IridiaTrSeries/link/IridiaTr2011-004.pdf},
  note = {Published in } # orp # {~\cite{LopDubPerStuBir2016irace}}
}

@incollection{PerLopHooStu2017:lion,
  author = perez_l # and # lopez-ibanez # and # hoos # and # stuetzle,
  title = {An Experimental Study of Adaptive Capping in {\rpackage{irace}}},
  crossref = {LION2017},
  pages = {235--250},
  pdf = {PerLopHooStu2017lion.pdf},
  doi = {10.1007/978-3-319-69404-7_17},
  supplement = {http://iridia.ulb.ac.be/supp/IridiaSupp2016-007/}
}

@incollection{SchHoo2012quanti,
  title = {Quantifying Homogeneity of Instance Sets for
                  Algorithm Configuration},
  crossref = {LION2012},
  keywords = {Quantifying Homogeneity; Empirical Analysis;
                  Parameter Optimization; Algorithm Configuration},
  author = schneider_m # and # hoos,
  pages = {190--204},
  doi = {10.1007/978-3-642-34413-8_14}
}

@incollection{YuaStuMonLauBir13,
  author = yuan_z # and # montesdeoca # and # stuetzle # and # lau_hc # and # birattari,
  title = {An Analysis of Post-selection in Automatic Configuration},
  pages = {1557--1564},
  crossref = {GECCO2013}
}

@book{AAAI2017,
  booktitle = aaai,
  editor = {Satinder P. Singh and Shaul Markovitch},
  title = {Proceedings of the Thirty-First {AAAI} Conference on
                  Artificial Intelligence, February 4-9, 2017, San Francisco,
                  California, {USA}},
  year = 2017,
  month = feb,
  publisher = aaaip-pub
}

@book{BarChiPaqPre2010emaoa,
  title = {Experimental Methods for the Analysis of
                  Optimization Algorithms},
  booktitle = {Experimental Methods for the Analysis of
                  Optimization Algorithms},
  publisher = springer,
  address = add-berlin,
  year = 2010,
  editor = bartz-beielstein # and # chiarandini # and # paquete # and # preuss_m
}

@proceedings{CEC2006,
  key = {IEEE CEC},
  title = {Proceedings of the 2006 Congress on Evolutionary
                  Computation (CEC 2006)},
  booktitle = cec2006,
  year = 2006,
  month = jul,
  publisher = ieeep,
  address = ieeep-ad
}

@book{GECCO2013,
  title = {Genetic and Evolutionary Computation Conference, GECCO 2013,
                  Proceedings, Amsterdam, The Netherlands, July 6-10, 2013},
  booktitle = gecco2013,
  editor = blum # and # alba_e,
  year = 2013,
  publisher = acm-pub,
  address = add-ny,
  isbn = {978-1-4503-1963-8}
}

@proceedings{ICML2014,
  editor = {Xing, Eric P. and Jebara, Tony},
  title = {Proceedings of the 31st International Conference on Machine
                  Learning, {ICML} 2014, Beijing, China, 21-26 June 2014},
  booktitle = icml2014,
  volume = 32,
  year = 2014,
  url = {http://jmlr.org/proceedings/papers/v32/}
}

@book{LION2012,
  title = {6th International Conference, LION 6, Paris, France,
                  January 16-20, 2012. Selected Papers},
  booktitle = {Learning and Intelligent Optimization, 6th
                  International Conference, LION 6},
  year = 2012,
  series = lncs,
  publisher = springer,
  address = add-heidelberg,
  editor = hamadi # and # schoenauer,
  volume = 7219
}

@book{LION2013,
  title = {7th International Conference, LION 7, Catania,
                  Italy, January 7-11, 2013. Selected Papers},
  booktitle = {Learning and Intelligent Optimization, 7th
                  International Conference, LION 7},
  year = 2013,
  series = lncs,
  editor = pardalos # and # {G. Nicosia},
  volume = 7997,
  publisher = springer,
  address = add-heidelberg
}

@book{LION2017,
  title = {11th International Conference, LION 11, Nizhny Novgorod,
                  Russia, June 19-21, 2017, Revised Selected Papers},
  booktitle = {Learning and Intelligent Optimization, 11th International
                  Conference, LION 11},
  year = 2017,
  editor = battiti # and # {Dmitri E. Kvasov} # and # {Yaroslav D. Sergeyev},
  volume = 10556,
  series = lncs,
  publisher = springer,
  address = add-cham
}
