

@preamble{{\providecommand{\MaxMinAntSystem}{{$\cal MAX$--$\cal MIN$} {Ant} {System}} } # {\providecommand{\rpackage}[1]{{#1}} } # {\providecommand{\softwarepackage}[1]{{#1}} } # {\providecommand{\proglang}[1]{{#1}} } # {\providecommand{\BIBdepartment}[1]{{#1}, } }}

@string{and = { and }}

@string{lncs = {Lecture Notes in Computer Science}}

@string{add-berlin-heidelberg = {Berlin\slash Heidelberg}}

@string{add-cham = { Cham, Switzerland}}

@string{add-heidelberg = { Heidelberg, Germany}}

@string{add-ny = { New York, NY}}

@string{aaaip-pub = {{AAAI} Press}}

@string{acm-pub = {ACM Press}}

@string{pmlr-pub = {{PMLR}}}

@string{springer = {Springer}}

@string{iridia = {IRIDIA, Universit{\'e} Libre de Bruxelles, Belgium}}

@string{proc_of = {Proceedings of }}

@string{proc_of_the = proc_of # { the }}

@string{aaai = proc_of_the # {{AAAI} Conference on Artificial Intelligence}}

@string{gecco = proc_of_the # {Genetic and Evolutionary Computation Conference, GECCO }}

@string{gecco2013 = gecco # {2013}}

@string{icml = { International Conference on Machine Learning, {ICML} }}

@string{icml2014 = proc_of_the # {31st} # icml # {2014}}

@string{lion_prefix = {Learning and Intelligent Optimization, }}

@string{lion2012 = lion_prefix # {6th International Conference, LION 6}}

@string{lion2013 = lion_prefix # {7th International Conference, LION 7}}

@string{lion2017 = lion_prefix # {11th International Conference, LION 11}}

@string{acm-cs = {{ACM} Computing Surveys}}

@string{cor = {Computers \& Operations Research}}

@string{ejor = {European Journal of Operational Research}}

@string{jair = {Journal of Artificial Intelligence Research}}

@string{joh = {Journal of Heuristics}}

@string{orp = {Operations Research Perspectives}}

@string{tec = {IEEE Transactions on Evolutionary Computation}}

@string{telo = {ACM Transactions on Evolutionary Learning and Optimization}}

@string{alba_e = { Alba, Enrique }}

@string{balaprakash = {  Prasanna Balaprakash }}

@string{bartz-beielstein = { Thomas Bartz-Beielstein }}

@string{battiti = { Roberto Battiti }}

@string{biedenkapp = { Biedenkapp, Andr{\'e} }}

@string{birattari = { Mauro Birattari }}

@string{blum = { Christian Blum }}

@string{branke = { J{\"u}rgen Branke }}

@string{chiarandini = { Marco Chiarandini }}

@string{desouza = { Marcelo {De Souza} }}

@string{doerr_c = { Carola Doerr }}

@string{dreo_j = { Johann Dreo }}

@string{dubois-lacoste = { J{\'e}r{\'e}mie Dubois-Lacoste }}

@string{eggensperger_k = { Katharina Eggensperger }}

@string{fawcett = { Chris Fawcett }}

@string{hamadi = { Youssef Hamadi }}

@string{hoos = { Holger H. Hoos }}

@string{hutter = { Frank Hutter }}

@string{lau_hc = { Hoong Chuin Lau }}

@string{leyton-brown = { Kevin Leyton-Brown }}

@string{lindauer_m = { Marius Thomas Lindauer }}

@string{lopez-ibanez = { Manuel L{\'o}pez-Ib{\'a}{\~n}ez }}

@string{mcgeoch_cc = { Catherine C. McGeoch }}

@string{montesdeoca = { Marco A. {Montes de Oca} }}

@string{paquete = { Lu{\'i}s Paquete }}

@string{pardalos = { Panos M. Pardalos }}

@string{perez_l = {  P{\'e}rez C{\'a}ceres, Leslie }}

@string{preuss_m = { Mike Preuss }}

@string{ritt = { Marcus Ritt}}

@string{schneider_m = { Marius Schneider }}

@string{schoenauer = { Marc Schoenauer }}

@string{stuetzle = { Thomas St{\"u}tzle }}

@string{vermetten_d = { Diederick Vermetten }}

@string{yuan_z = { Zhi Yuan }}

@article{DesRitLopPer2021acviz,
  author = desouza # and # ritt # and # lopez-ibanez # and # perez_l,
  title = {{\softwarepackage{ACVIZ}}: A Tool for the Visual Analysis of
                  the Configuration of Algorithms with {\rpackage{irace}}},
  journal = orp,
  year = 2021,
  doi = {10.1016/j.orp.2021.100186},
  supplement = {https://zenodo.org/record/4714582},
  abstract = {This paper introduces acviz, a tool that helps to analyze the
                  automatic configuration of algorithms with irace. It provides
                  a visual representation of the configuration process,
                  allowing users to extract useful information, e.g. how the
                  configurations evolve over time. When test data is available,
                  acviz also shows the performance of each configuration on the
                  test instances. Using this visualization, users can analyze
                  and compare the quality of the resulting configurations and
                  observe the performance differences on training and test
                  instances.},
  volume = 8,
  pages = 100186
}

@article{FawHoos2015ablation,
  title = {Analysing Differences Between Algorithm Configurations
                  through Ablation},
  author = fawcett # and # hoos,
  journal = joh,
  pages = {431--458},
  volume = 22,
  number = 4,
  year = 2016
}

@article{HutHooLeyStu2009jair,
  author = hutter # and # hoos # and # leyton-brown # and # stuetzle,
  title = {{\softwarepackage{ParamILS}:} An Automatic Algorithm
                  Configuration Framework},
  journal = jair,
  year = 2009,
  volume = 36,
  pages = {267--306},
  month = oct,
  doi = {10.1613/jair.2861}
}

@article{LopBraPaq2021telo,
  author = lopez-ibanez # and # branke # and # paquete,
  title = {Reproducibility in Evolutionary Computation},
  journal = telo,
  year = 2021,
  volume = 1,
  number = 4,
  pages = {1--21},
  doi = {10.1145/3466624},
  epub = {https://arxiv.org/abs/2102.03380},
  abstract = {Experimental studies are prevalent in Evolutionary
                  Computation (EC), and concerns about the reproducibility and
                  replicability of such studies have increased in recent times,
                  reflecting similar concerns in other scientific fields. In
                  this article, we suggest a classification of different types
                  of reproducibility that refines the badge system of the
                  Association of Computing Machinery (ACM) adopted by TELO. We
                  discuss, within the context of EC, the different types of
                  reproducibility as well as the concepts of artifact and
                  measurement, which are crucial for claiming
                  reproducibility. We identify cultural and technical obstacles
                  to reproducibility in the EC field. Finally, we provide
                  guidelines and suggest tools that may help to overcome some
                  of these reproducibility obstacles.},
  keywords = {Evolutionary Computation, Reproducibility, Empirical study,
                  Benchmarking}
}

@article{LopDubPerStuBir2016irace,
  author = lopez-ibanez # and # dubois-lacoste # and # perez_l # and # stuetzle # and # birattari,
  title = {The {\rpackage{irace}} Package: Iterated Racing for Automatic
                  Algorithm Configuration},
  journal = orp,
  year = 2016,
  supplement = {http://iridia.ulb.ac.be/supp/IridiaSupp2016-003/},
  doi = {10.1016/j.orp.2016.09.002},
  volume = 3,
  pages = {43--58}
}

@article{LopStu2013ejor,
  author = lopez-ibanez # and # stuetzle,
  title = {Automatically Improving the Anytime Behaviour of Optimisation
                  Algorithms},
  journal = ejor,
  year = 2014,
  volume = 235,
  number = 3,
  pages = {569--582},
  doi = {10.1016/j.ejor.2013.10.043},
  pdf = {LopStu2014ejor.pdf},
  supplement = {http://iridia.ulb.ac.be/supp/IridiaSupp2012-011/},
  abstract = {Optimisation algorithms with good anytime behaviour try to
                  return as high-quality solutions as possible independently of
                  the computation time allowed. Designing algorithms with good
                  anytime behaviour is a difficult task, because performance is
                  often evaluated subjectively, by plotting the trade-off curve
                  between computation time and solution quality. Yet, the
                  trade-off curve may be modelled also as a set of mutually
                  nondominated, bi-objective points. Using this model, we
                  propose to combine an automatic configuration tool and the
                  hypervolume measure, which assigns a single quality measure
                  to a nondominated set. This allows us to improve the anytime
                  behaviour of optimisation algorithms by means of
                  automatically finding algorithmic configurations that produce
                  the best nondominated sets. Moreover, the recently proposed
                  weighted hypervolume measure is used here to incorporate the
                  decision-maker's preferences into the automatic tuning
                  procedure. We report on the improvements reached when
                  applying the proposed method to two relevant scenarios: (i)
                  the design of parameter variation strategies for MAX-MIN Ant
                  System and (ii) the tuning of the anytime behaviour of SCIP,
                  an open-source mixed integer programming solver with more
                  than 200 parameters.}
}

@article{LopVerDreDoe2025,
  author = lopez-ibanez # and # vermetten_d # and # dreo_j # and # doerr_c,
  title = {Using the Empirical Attainment Function for Analyzing
                  Single-objective Black-box Optimization Algorithms},
  journal = tec,
  year = 2025,
  volume = 29,
  number = 5,
  pages = {1774--1782},
  annote = {Pre-print: \url{https://doi.org/10.48550/arXiv.2404.02031}},
  doi = {10.1109/TEVC.2024.3462758},
  abstract = {A widely accepted way to assess the performance of iterative
                  black-box optimizers is to analyze their empirical cumulative
                  distribution function (ECDF) of pre-defined quality targets
                  achieved not later than a given runtime. In this work, we
                  consider an alternative approach, based on the empirical
                  attainment function (EAF) and we show that the target-based
                  ECDF is an approximation of the EAF. We argue that the EAF
                  has several advantages over the target-based ECDF. In
                  particular, it does not require defining a priori quality
                  targets per function, captures performance differences more
                  precisely, and enables the use of additional summary
                  statistics that enrich the analysis. We also show that the
                  average area over the convergence curves is a
                  simpler-to-calculate, but equivalent, measure of anytime
                  performance. To facilitate the accessibility of the EAF, we
                  integrate a module to compute it into the IOHanalyzer
                  platform. Finally, we illustrate the use of the EAF via
                  synthetic examples and via the data available for the BBOB
                  suite.},
  keywords = {EAF-based ECDF}
}

@article{McG1992vrt,
  author = mcgeoch_cc,
  title = {Analyzing Algorithms by Simulation: Variance Reduction
                  Techniques and Simulation Speedups},
  abstract = {Although experimental studies have been widely applied to the
                  investigation of algorithm performance, very little attention
                  has been given to experimental method in this area. This is
                  unfortunate, since much can be done to improve the quality of
                  the data obtained; often, much improvement may be needed for
                  the data to be useful. This paper gives a tutorial discussion
                  of two aspects of good experimental technique: the use of
                  variance reduction techniques and simulation speedups in
                  algorithm studies.  In an illustrative study, application of
                  variance reduction techniques produces a decrease in variance
                  by a factor 1000 in one case, giving a dramatic improvement
                  in the precision of experimental results. Furthermore, the
                  complexity of the simulation program is improved from
                  $\Theta(m n/H_n)$ to $\Theta(m + n \log n)$ (where $m$ is
                  typically much larger than $n$), giving a much faster
                  simulation program and therefore more data per unit of
                  computation time. The general application of variance
                  reduction techniques is also discussed for a variety of
                  algorithm problem domains.},
  volume = 24,
  doi = {10.1145/130844.130853},
  number = 2,
  journal = acm-cs,
  year = 1992,
  keywords = {experimental analysis of algorithms, move-to-front rule,
                  self-organizing sequential search, statistical analysis of
                  algorithms, transpose rule, variance reduction techniques},
  pages = {195--212}
}

@article{SouRitLop2021cap,
  author = desouza # and # ritt # and # lopez-ibanez,
  title = {Capping Methods for the Automatic Configuration of
                  Optimization Algorithms},
  journal = cor,
  doi = {10.1016/j.cor.2021.105615},
  year = 2022,
  volume = 139,
  pages = 105615,
  supplement = {https://github.com/souzamarcelo/supp-cor-capopt},
  abstract = {Automatic configuration techniques are widely and
                  successfully used to find good parameter settings for
                  optimization algorithms. Configuration is costly, because it
                  is necessary to evaluate many configurations on different
                  instances. For decision problems, when the objective is to
                  minimize the running time of the algorithm, many
                  configurators implement capping methods to discard poor
                  configurations early. Such methods are not directly
                  applicable to optimization problems, when the objective is to
                  optimize the cost of the best solution found, given a
                  predefined running time limit. We propose new capping methods
                  for the automatic configuration of optimization
                  algorithms. They use the previous executions to determine a
                  performance envelope, which is used to evaluate new
                  executions and cap those that do not satisfy the envelope
                  conditions. We integrate the capping methods into the irace
                  configurator and evaluate them on different optimization
                  scenarios. Our results show that the proposed methods can
                  save from about 5\% to 78\% of the configuration effort,
                  while finding configurations of the same quality. Based on
                  the computational analysis, we identify two conservative and
                  two aggressive methods, that save an average of about 20\%
                  and 45\% of the configuration effort, respectively. We also
                  provide evidence that capping can help to better use the
                  available budget in scenarios with a configuration time
                  limit.}
}

@incollection{BieLinEggFraFawHoo2017,
  author = biedenkapp # and # lindauer_m # and # eggensperger_k # and # hutter # and # fawcett # and # hoos,
  title = {Efficient Parameter Importance Analysis via Ablation with
                  Surrogates},
  crossref = {AAAI2017},
  doi = {10.1609/aaai.v31i1.10657}
}

@incollection{BirYuaBal2010:emaoa,
  author = birattari # and # yuan_z # and # balaprakash # and # stuetzle,
  title = {{F}-Race and Iterated {F}-Race: An Overview},
  pages = {311--336},
  crossref = {BarChiPaqPre2010emaoa},
  keywords = {F-race, iterated F-race, irace, tuning},
  doi = {10.1007/978-3-642-02538-9_13}
}

@incollection{HutHooLey2013lion,
  author = hutter # and # hoos # and # leyton-brown,
  title = {Identifying Key Algorithm Parameters and Instance Features
                  using Forward Selection},
  pages = {364--381},
  crossref = {LION2013},
  doi = {10.1007/978-3-642-44973-4_40},
  abstract = {Most state-of-the-art algorithms for large-scale optimization
                  problems expose free parameters, giving rise to combinatorial
                  spaces of possible configurations. Typically, these spaces
                  are hard for humans to understand. In this work, we study a
                  model-based approach for identifying a small set of both
                  algorithm parameters and instance features that suffices for
                  predicting empirical algorithm performance well. Our
                  empirical analyses on a wide variety of hard combinatorial
                  problem benchmarks spanning SAT, MIP, and TSP show that--for
                  parameter configurations sampled uniformly at random--very
                  good performance predictions can typically be obtained based
                  on just two key parameters, and that similarly, few instance
                  features and algorithm parameters suffice to predict the most
                  salient algorithm performance characteristics in the combined
                  configuration/feature space. We also use these models to
                  identify settings of these key parameters that are predicted
                  to achieve the best overall performance, both on average
                  across instances and in an instance-specific way. This serves
                  as a further way of evaluating model quality and also
                  provides a tool for further understanding the parameter
                  space. We provide software for carrying out this analysis on
                  arbitrary problem domains and hope that it will help
                  algorithm developers gain insights into the key parameters of
                  their algorithms, the key features of their instances, and
                  their interactions.},
  keywords = {parameter importance}
}

@inproceedings{HutHooLey2014icml,
  author = hutter # and # hoos # and # leyton-brown,
  title = {An Efficient Approach for Assessing Hyperparameter
                  Importance},
  crossref = {ICML2014},
  pages = {754--762},
  url = {https://proceedings.mlr.press/v32/hutter14.html},
  keywords = {fANOVA, parameter importance}
}

@techreport{IRIDIA-2004-001,
  author = birattari,
  title = {On the Estimation of the Expected Performance of a Metaheuristic on a Class of Instances. How Many Instances, How Many Runs?},
  institution = iridia,
  year = 2004,
  number = {TR/IRIDIA/2004-001}
}

@techreport{LopDubStu2011irace,
  author = lopez-ibanez # and # dubois-lacoste # and # stuetzle # and # birattari,
  title = {The {\rpackage{irace}} package, Iterated Race for Automatic
                  Algorithm Configuration},
  institution = iridia,
  year = 2011,
  number = {TR/IRIDIA/2011-004},
  url = {http://iridia.ulb.ac.be/IridiaTrSeries/link/IridiaTr2011-004.pdf},
  note = {Published in } # orp # {~\cite{LopDubPerStuBir2016irace}}
}

@incollection{PerLopHooStu2017:lion,
  author = perez_l # and # lopez-ibanez # and # hoos # and # stuetzle,
  title = {An Experimental Study of Adaptive Capping in {\rpackage{irace}}},
  crossref = {LION2017},
  pages = {235--250},
  pdf = {PerLopHooStu2017lion.pdf},
  doi = {10.1007/978-3-319-69404-7_17},
  supplement = {http://iridia.ulb.ac.be/supp/IridiaSupp2016-007/}
}

@incollection{SchHoo2012quanti,
  author = schneider_m # and # hoos,
  title = {Quantifying Homogeneity of Instance Sets for Algorithm
                  Configuration},
  pages = {190--204},
  crossref = {LION2012},
  keywords = {Quantifying Homogeneity; Empirical Analysis; Parameter
                  Optimization; Algorithm Configuration},
  doi = {10.1007/978-3-642-34413-8_14}
}

@incollection{YuaStuMonLauBir13,
  author = yuan_z # and # montesdeoca # and # stuetzle # and # lau_hc # and # birattari,
  title = {An Analysis of Post-selection in Automatic Configuration},
  pages = {1557--1564},
  crossref = {GECCO2013}
}

@book{AAAI2017,
  booktitle = aaai,
  editor = {Satinder P. Singh and Shaul Markovitch},
  title = {Proceedings of the Thirty-First {AAAI} Conference on
                  Artificial Intelligence, February 4-9, 2017, San Francisco,
                  California, {USA}},
  year = 2017,
  month = feb,
  publisher = aaaip-pub
}

@book{BarChiPaqPre2010emaoa,
  title = {Experimental Methods for the Analysis of
                  Optimization Algorithms},
  booktitle = {Experimental Methods for the Analysis of
                  Optimization Algorithms},
  publisher = springer,
  address = add-berlin-heidelberg,
  year = 2010,
  editor = bartz-beielstein # and # chiarandini # and # paquete # and # preuss_m
}

@book{GECCO2013,
  title = {Genetic and Evolutionary Computation Conference, GECCO 2013,
                  Proceedings, Amsterdam, The Netherlands, July 6-10, 2013},
  booktitle = gecco2013,
  editor = blum # and # alba_e,
  year = 2013,
  publisher = acm-pub,
  address = add-ny,
  isbn = {978-1-4503-1963-8}
}

@proceedings{ICML2014,
  title = {Proceedings of the 31st International Conference on Machine
                  Learning, {ICML} 2014, Beijing, China, 21-26 June 2014},
  year = 2014,
  booktitle = icml2014,
  editor = {Xing, Eric P. and Jebara, Tony},
  volume = 32,
  publisher = pmlr-pub,
  key = {ICML}
}

@book{LION2012,
  editor = hamadi # and # schoenauer,
  title = {6th International Conference, LION 6, Paris, France, January
                  16-20, 2012. Selected Papers},
  year = 2012,
  publisher = springer,
  booktitle = lion2012,
  volume = 7219,
  series = lncs,
  address = add-heidelberg
}

@book{LION2013,
  editor = pardalos # and # {G. Nicosia},
  title = {7th International Conference, LION 7, Catania, Italy, January
                  7-11, 2013. Selected Papers},
  year = 2013,
  publisher = springer,
  booktitle = lion2013,
  volume = 7997,
  series = lncs,
  address = add-heidelberg
}

@book{LION2017,
  editor = battiti # and # {Dmitri E. Kvasov} # and # {Yaroslav D. Sergeyev},
  title = {11th International Conference, LION 11, Nizhny Novgorod,
                  Russia, June 19-21, 2017, Revised Selected Papers},
  year = 2017,
  publisher = springer,
  booktitle = lion2017,
  volume = 10556,
  series = lncs,
  address = add-cham
}
