// Copyright 2022-2023 Bloomberg Finance L.P
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <clangscandeps.h>

#include <compilerdefaults.h>
#include <deps.h>
#include <env.h>
#include <fileutils.h>
#include <shellutils.h>
#include <subprocess.h>

#include <buildboxcommon_digestgenerator.h>
#include <buildboxcommon_fileutils.h>
#include <buildboxcommon_logging.h>
#include <buildboxcommon_permissions.h>
#include <buildboxcommon_stringutils.h>
#include <buildboxcommon_systemutils.h>
#include <buildboxcommon_temporarydirectory.h>
#include <buildboxcommonmetrics_countingmetricutil.h>

#include <cerrno>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <sstream>
#include <sys/types.h>
#include <sys/wait.h>
#include <system_error>
#include <unistd.h>

#ifdef RECC_CLANG_SCAN_DEPS
#include <nlohmann/json.hpp>
using json = nlohmann::json;
#endif

static const std::string COUNTER_NAME_CLANG_SCAN_DEPS_INVOCATION_SUCCESS =
    "recc.clang_scan_deps_invocation_success";
static const std::string COUNTER_NAME_CLANG_SCAN_DEPS_INVOCATION_FAILURE =
    "recc.clang_scan_deps_invocation_failure";
static const std::string COUNTER_NAME_CLANG_SCAN_DEPS_TARGET_SUCCESS =
    "recc.clang_scan_deps_target_success";
static const std::string COUNTER_NAME_CLANG_SCAN_DEPS_TARGET_FAILURE =
    "recc.clang_scan_deps_target_failure";

namespace recc {

#ifdef RECC_CLANG_SCAN_DEPS

namespace {

const std::string RECC_DEPENDENCIES = "recc-scan-deps.d";
const std::string RECC_DEPENDENCIES_LOCK = RECC_DEPENDENCIES + ".lock";
const std::string RECC_DEPENDENCIES_TIMESTAMP = ".cache_timestamp";

/**
 * Split the clang-scan-deps output into one file per target.
 *
 * The clang-scan-deps output consists of `make`-compatible rules describing
 * the dependencies of each target. For projects with many targets, this can
 * get fairly large. To avoid requiring each `recc` invocation in a project
 * build to parse all rules, this function splits the dependencies into one
 * file per rule, using the hash of the target as filename.
 *
 * Example rule as produced by clang-scan-deps:
 *     target.o: \
 *       /src/foo/target.cpp \
 *       /usr/include/header.h
 */
void splitScanDepsRules(const std::string &rules, const std::string &depsdir)
{
    std::set<std::string> targets;

    size_t rulestart = 0;
    while (rulestart < rules.size()) {
        size_t pos = rulestart;
        size_t rulesize = 0;

        // Scan for an unescaped newline to find the end of the rule
        while (true) {
            pos = rules.find('\n', pos);
            if (pos == std::string::npos) {
                // End of file
                rulesize = rules.size() - rulestart;
                break;
            }
            else if (pos > 0 && rules[pos - 1] == '\\') {
                // Escaped newline, rule continues
                pos++;
            }
            else {
                // End of rule
                rulesize = pos + 1 - rulestart;
                break;
            }
        }

        const std::string rule = rules.substr(rulestart, rulesize);
        size_t colon = rule.find(':');
        if (colon != std::string::npos) {
            const std::string target =
                buildboxcommon::StringUtils::trim(rule.substr(0, colon));
            const auto targetDigest =
                buildboxcommon::DigestGenerator::hash(target);
            const std::string path = depsdir + "/" + targetDigest.hash();

            if (targets.find(target) == targets.end()) {
                targets.insert(target);
                buildboxcommon::FileUtils::writeFileAtomically(
                    path, rule, PERMISSION_RWUSR_RALL);
            }
            else {
                // Duplicate target name, remove dependency file to trigger
                // fallback to dependencies command.
                if (unlink(path.c_str()) < 0 && errno != ENOENT) {
                    BUILDBOXCOMMON_THROW_SYSTEM_EXCEPTION(
                        std::system_error, errno, std::system_category,
                        "Failed to remove file \"" << path << "\"");
                }
            }
        }
        else if (!buildboxcommon::StringUtils::trim(rule).empty()) {
            BUILDBOXCOMMON_THROW_EXCEPTION(
                std::runtime_error,
                "Failed to parse clang-scan-deps rule: " << rule);
        }
        // Start of next rule
        rulestart += rulesize;
    }
}

/**
 * Determine predefined macros and system include directories of the
 * compiler.
 *
 * If the compiler is not part of the same toolchain as `clang-scan-deps`,
 * there may be differences in predefined macros and system include
 * directories.
 *
 * Given a set of arguments, extract the compiler arguments that can
 * affect the predefined macros for the compiler & append to
 * relevantCommandLine
 */
template <typename Container>
void extractCompileCommandsOptionsForCompiler(
    std::vector<std::string> *relevantCommandLine, const Container &arguments)
{
    relevantCommandLine->push_back(arguments.front());
    for (auto it = arguments.begin(); it != arguments.end(); ++it) {
        const auto &argument = *it;
        // NOLINTBEGIN (cppcoreguidelines-avoid-magic-numbers)
        if (argument.substr(0, 5) == "-std=" ||
            argument.substr(0, 6) == "--std=" ||
            argument.substr(0, 2) == "-O" || argument.substr(0, 2) == "-f" ||
            argument.substr(0, 2) == "-m" || argument.substr(0, 2) == "-x" ||
            argument == "-undef" || argument == "-nostdinc") {
            // These flags may affect predefined macros or include directories
            if (argument == "-x") {
                // If the argument is "-x", then we need to push the next
                // argument
                relevantCommandLine->push_back(argument);
                if (++it != arguments.end()) {
                    relevantCommandLine->push_back(*it);
                }
            }
            else {
                relevantCommandLine->push_back(argument);
            }
        }
        // NOLINTEND (cppcoreguidelines-avoid-magic-numbers)
    }
}

/**
 * This function generates a header file for predefined macros
 * and a list of extra command-line arguments for `clang-scan-deps` to use
 * the predefined macros and system include directories from the compiler
 * instead of the defaults from the `clang-scan-deps` toolchain.
 */
std::vector<std::string>
getExtraArgsForScanDeps(const std::string &basedir, std::string &compilerKey,
                        std::vector<std::string> &compilerPrintCommand)
{
    const std::string emptyHeader = basedir + "/recc-empty.h";
    std::ofstream emptyHeaderStream(emptyHeader);
    emptyHeaderStream.close();

    compilerPrintCommand.push_back("-E");
    compilerPrintCommand.push_back("-dM");
    compilerPrintCommand.push_back("-Wp,-v");
    compilerPrintCommand.push_back(emptyHeader);

    BUILDBOX_LOG_DEBUG(
        "Getting predefined macros and system include directories for "
        "compiler: "
        << compilerKey << " using command: "
        << buildboxcommon::StringUtils::join(compilerPrintCommand, " "));

    const auto subprocessResult =
        Subprocess::execute(compilerPrintCommand, true, true, RECC_DEPS_ENV);

    if (subprocessResult.d_exitCode != 0) {
        std::string errorMsg = "Failed to execute: ";
        for (const auto &token : compilerPrintCommand) {
            errorMsg += (token + " ");
        }
        BUILDBOX_LOG_WARNING(errorMsg);
        BUILDBOX_LOG_WARNING("Exit status: " << subprocessResult.d_exitCode);
        BUILDBOX_LOG_WARNING("stdout: " << subprocessResult.d_stdOut);
        BUILDBOX_LOG_WARNING("stderr: " << subprocessResult.d_stdErr);
        throw subprocess_failed_error(subprocessResult.d_exitCode);
    }

    std::vector<std::string> extraArgs;
    extraArgs.push_back("-undef");
    extraArgs.push_back("-nostdinc");

    const auto compilerKeyDigest =
        buildboxcommon::DigestGenerator::hash(compilerKey);

    const std::string predefinedHeader =
        basedir + "/" + compilerKeyDigest.hash() + "-recc-scan-deps.h";
    buildboxcommon::FileUtils::writeFileAtomically(
        predefinedHeader, subprocessResult.d_stdOut, PERMISSION_RWUSR_RALL);

    extraArgs.push_back("-include");
    extraArgs.push_back(predefinedHeader);

    // Get the system include directories
    std::istringstream preprocessorStream(subprocessResult.d_stdErr);
    std::string line;
    bool inSearchList = false;
    while (std::getline(preprocessorStream, line)) {
        if (line == "#include <...> search starts here:") {
            inSearchList = true;
        }
        else if (line == "End of search list.") {
            break;
        }
        else if (inSearchList) {
            buildboxcommon::StringUtils::ltrim(&line);
            extraArgs.push_back("-idirafter");
            extraArgs.push_back(line);
        }
    }

    return extraArgs;
}

/**
 * The function runs the scan deps executable located at `scanDepsPath`
 * The compilation database is passed as a json object. The json object is
 * written to a temporary file and passed as the --compilation-database command
 * to the command and the output is returned
 */
const std::string getScanDepsOutput(const std::string &scanDepsPath,
                                    const json &compilationDatabase,
                                    const bool logError = false)
{
    buildboxcommon::TemporaryFile compilationDatabaseFile;
    std::ofstream compilationDatabaseStream(compilationDatabaseFile.strname());
    compilationDatabaseStream << compilationDatabase;
    compilationDatabaseStream.close();

    std::vector<std::string> scanDepsCommand;
    scanDepsCommand.push_back(scanDepsPath);
    scanDepsCommand.push_back("--compilation-database=" +
                              compilationDatabaseFile.strname());

    std::ostringstream depCommand;
    for (auto &token : scanDepsCommand) {
        depCommand << token << " ";
    }
    BUILDBOX_LOG_DEBUG(
        "Getting dependencies using the command: " << depCommand.str());

    const auto subprocessResult =
        Subprocess::execute(scanDepsCommand, true, true, RECC_DEPS_ENV);

    if (subprocessResult.d_exitCode != 0) {
        std::string errorMsg = "Failed to execute: ";
        for (const auto &token : scanDepsCommand) {
            errorMsg += (token + " ");
        }
        if (logError) {
            BUILDBOX_LOG_ERROR(errorMsg);
            BUILDBOX_LOG_ERROR("Exit status: " << subprocessResult.d_exitCode);
        }
        else {
            BUILDBOX_LOG_WARNING(errorMsg);
            BUILDBOX_LOG_WARNING(
                "Exit status: " << subprocessResult.d_exitCode);
        }
        BUILDBOX_LOG_DEBUG("stdout: " << subprocessResult.d_stdOut);
        BUILDBOX_LOG_DEBUG("stderr: " << subprocessResult.d_stdErr);
        throw subprocess_failed_error(subprocessResult.d_exitCode);
    }

    return subprocessResult.d_stdOut;
}

/**
 * Return `true` if the argument is a GCC compiler option that doesn't affect
 * dependency scanning and might not be supported by clang-scan-deps.
 */
bool ignoreCompilerArgumentForScanDeps(const std::string &argument)
{
    /* Source: Optimization options from GCC 14 `common.opt` */

    static const std::set<std::string> ignoreOptions{
        "-fno-aggressive-loop-optimizations",
        "-falign-functions",
        "-fno-align-functions",
        "-falign-jumps",
        "-fno-align-jumps",
        "-falign-labels",
        "-fno-align-labels",
        "-falign-loops",
        "-fno-align-loops",
        "-fallocation-dce",
        "-fno-allocation-dce",
        "-fallow-store-data-races",
        "-fno-allow-store-data-races",
        "-fassociative-math",
        "-fno-associative-math",
        "-fasynchronous-unwind-tables",
        "-fno-asynchronous-unwind-tables",
        "-fauto-inc-dec",
        "-fno-auto-inc-dec",
        "-fbit-tests",
        "-fno-bit-tests",
        "-fbranch-count-reg",
        "-fno-branch-count-reg",
        "-fbranch-probabilities",
        "-fno-branch-probabilities",
        "-fcaller-saves",
        "-fno-caller-saves",
        "-fcode-hoisting",
        "-fno-code-hoisting",
        "-fcombine-stack-adjustments",
        "-fno-combine-stack-adjustments",
        "-fcompare-elim",
        "-fno-compare-elim",
        "-fconcepts",
        "-fno-concepts",
        "-fconserve-stack",
        "-fno-conserve-stack",
        "-fcprop-registers",
        "-fno-cprop-registers",
        "-fcrossjumping",
        "-fno-crossjumping",
        "-fcse-follow-jumps",
        "-fno-cse-follow-jumps",
        "-fcx-fortran-rules",
        "-fno-cx-fortran-rules",
        "-fcx-limited-range",
        "-fno-cx-limited-range",
        "-fdce",
        "-fno-dce",
        "-fdefer-pop",
        "-fno-defer-pop",
        "-fdelayed-branch",
        "-fno-delayed-branch",
        "-fdelete-dead-exceptions",
        "-fno-delete-dead-exceptions",
        "-fdelete-null-pointer-checks",
        "-fno-delete-null-pointer-checks",
        "-fdevirtualize",
        "-fno-devirtualize",
        "-fdevirtualize-speculatively",
        "-fno-devirtualize-speculatively",
        "-fdse",
        "-fno-dse",
        "-fearly-inlining",
        "-fno-early-inlining",
        "-fexceptions",
        "-fno-exceptions",
        "-fexpensive-optimizations",
        "-fno-expensive-optimizations",
        "-ffast-math",
        "-fno-fast-math",
        "-ffinite-loops",
        "-fno-finite-loops",
        "-ffinite-math-only",
        "-fno-finite-math-only",
        "-ffloat-store",
        "-fno-float-store",
        "-ffold-mem-offsets",
        "-fno-fold-mem-offsets",
        "-fforward-propagate",
        "-fno-forward-propagate",
        "-ffp-int-builtin-inexact",
        "-fno-fp-int-builtin-inexact",
        "-ffunction-cse",
        "-fno-function-cse",
        "-fgcse",
        "-fno-gcse",
        "-fgcse-after-reload",
        "-fno-gcse-after-reload",
        "-fgcse-las",
        "-fno-gcse-las",
        "-fgcse-lm",
        "-fno-gcse-lm",
        "-fgcse-sm",
        "-fno-gcse-sm",
        "-fgraphite",
        "-fno-graphite",
        "-fgraphite-identity",
        "-fno-graphite-identity",
        "-fguess-branch-probability",
        "-fno-guess-branch-probability",
        "-fhardcfr-check-exceptions",
        "-fno-hardcfr-check-exceptions",
        "-fhardcfr-check-returning-calls",
        "-fno-hardcfr-check-returning-calls",
        "-fhardcfr-skip-leaf",
        "-fno-hardcfr-skip-leaf",
        "-fharden-compares",
        "-fno-harden-compares",
        "-fharden-conditional-branches",
        "-fno-harden-conditional-branches",
        "-fharden-control-flow-redundancy",
        "-fno-harden-control-flow-redundancy",
        "-fhoist-adjacent-loads",
        "-fno-hoist-adjacent-loads",
        "-fif-conversion",
        "-fno-if-conversion",
        "-fif-conversion2",
        "-fno-if-conversion2",
        "-findirect-inlining",
        "-fno-indirect-inlining",
        "-finline",
        "-fno-inline",
        "-finline-atomics",
        "-fno-inline-atomics",
        "-finline-functions",
        "-fno-inline-functions",
        "-finline-functions-called-once",
        "-fno-inline-functions-called-once",
        "-finline-small-functions",
        "-fno-inline-small-functions",
        "-finline-stringops",
        "-fno-inline-stringops",
        "-fipa-bit-cp",
        "-fno-ipa-bit-cp",
        "-fipa-cp",
        "-fno-ipa-cp",
        "-fipa-cp-clone",
        "-fno-ipa-cp-clone",
        "-fipa-icf",
        "-fno-ipa-icf",
        "-fipa-icf-functions",
        "-fno-ipa-icf-functions",
        "-fipa-icf-variables",
        "-fno-ipa-icf-variables",
        "-fipa-modref",
        "-fno-ipa-modref",
        "-fipa-profile",
        "-fno-ipa-profile",
        "-fipa-pta",
        "-fno-ipa-pta",
        "-fipa-pure-const",
        "-fno-ipa-pure-const",
        "-fipa-ra",
        "-fno-ipa-ra",
        "-fipa-reference",
        "-fno-ipa-reference",
        "-fipa-reference-addressable",
        "-fno-ipa-reference-addressable",
        "-fipa-sra",
        "-fno-ipa-sra",
        "-fipa-stack-alignment",
        "-fno-ipa-stack-alignment",
        "-fipa-strict-aliasing",
        "-fno-ipa-strict-aliasing",
        "-fipa-vrp",
        "-fno-ipa-vrp",
        "-fira-hoist-pressure",
        "-fno-ira-hoist-pressure",
        "-fira-loop-pressure",
        "-fno-ira-loop-pressure",
        "-fira-share-save-slots",
        "-fno-ira-share-save-slots",
        "-fira-share-spill-slots",
        "-fno-ira-share-spill-slots",
        "-fisolate-erroneous-paths-attribute",
        "-fno-isolate-erroneous-paths-attribute",
        "-fisolate-erroneous-paths-dereference",
        "-fno-isolate-erroneous-paths-dereference",
        "-fivopts",
        "-fno-ivopts",
        "-fjump-tables",
        "-fno-jump-tables",
        "-fkeep-gc-roots-live",
        "-fno-keep-gc-roots-live",
        "-flifetime-dse",
        "-fno-lifetime-dse",
        "-flimit-function-alignment",
        "-fno-limit-function-alignment",
        "-flive-patching",
        "-fno-live-patching",
        "-flive-range-shrinkage",
        "-fno-live-range-shrinkage",
        "-floop-interchange",
        "-fno-loop-interchange",
        "-floop-nest-optimize",
        "-fno-loop-nest-optimize",
        "-floop-parallelize-all",
        "-fno-loop-parallelize-all",
        "-floop-unroll-and-jam",
        "-fno-loop-unroll-and-jam",
        "-flra-remat",
        "-fno-lra-remat",
        "-fmath-errno",
        "-fno-math-errno",
        "-fmodulo-sched",
        "-fno-modulo-sched",
        "-fmodulo-sched-allow-regmoves",
        "-fno-modulo-sched-allow-regmoves",
        "-fmove-loop-invariants",
        "-fno-move-loop-invariants",
        "-fmove-loop-stores",
        "-fno-move-loop-stores",
        "-fno-inline-stringops",
        "-fno-no-inline-stringops",
        "-fnon-call-exceptions",
        "-fno-non-call-exceptions",
        "-fomit-frame-pointer",
        "-fno-omit-frame-pointer",
        "-foptimize-sibling-calls",
        "-fno-optimize-sibling-calls",
        "-foptimize-strlen",
        "-fno-optimize-strlen",
        "-fopt-info",
        "-fno-opt-info",
        "-fpack-struct",
        "-fno-pack-struct",
        "-fpartial-inlining",
        "-fno-partial-inlining",
        "-fpeel-loops",
        "-fno-peel-loops",
        "-fpeephole",
        "-fno-peephole",
        "-fpeephole2",
        "-fno-peephole2",
        "-fplt",
        "-fno-plt",
        "-fpredictive-commoning",
        "-fno-predictive-commoning",
        "-fprefetch-loop-arrays",
        "-fno-prefetch-loop-arrays",
        "-fprofile-partial-training",
        "-fno-profile-partial-training",
        "-fprofile-reorder-functions",
        "-fno-profile-reorder-functions",
        "-freciprocal-math",
        "-fno-reciprocal-math",
        "-free",
        "-fno-ree",
        "-freg-struct-return",
        "-fno-reg-struct-return",
        "-frename-registers",
        "-fno-rename-registers",
        "-freorder-blocks",
        "-fno-reorder-blocks",
        "-freorder-blocks-and-partition",
        "-fno-reorder-blocks-and-partition",
        "-freorder-functions",
        "-fno-reorder-functions",
        "-frerun-cse-after-loop",
        "-fno-rerun-cse-after-loop",
        "-freschedule-modulo-scheduled-loops",
        "-fno-reschedule-modulo-scheduled-loops",
        "-frounding-math",
        "-fno-rounding-math",
        "-fsave-optimization-record",
        "-fno-save-optimization-record",
        "-fsched2-use-superblocks",
        "-fno-sched2-use-superblocks",
        "-fsched-critical-path-heuristic",
        "-fno-sched-critical-path-heuristic",
        "-fsched-dep-count-heuristic",
        "-fno-sched-dep-count-heuristic",
        "-fsched-group-heuristic",
        "-fno-sched-group-heuristic",
        "-fsched-interblock",
        "-fno-sched-interblock",
        "-fsched-last-insn-heuristic",
        "-fno-sched-last-insn-heuristic",
        "-fsched-pressure",
        "-fno-sched-pressure",
        "-fsched-rank-heuristic",
        "-fno-sched-rank-heuristic",
        "-fsched-spec",
        "-fno-sched-spec",
        "-fsched-spec-insn-heuristic",
        "-fno-sched-spec-insn-heuristic",
        "-fsched-spec-load",
        "-fno-sched-spec-load",
        "-fsched-spec-load-dangerous",
        "-fno-sched-spec-load-dangerous",
        "-fsched-stalled-insns",
        "-fno-sched-stalled-insns",
        "-fsched-stalled-insns-dep",
        "-fno-sched-stalled-insns-dep",
        "-fschedule-fusion",
        "-fno-schedule-fusion",
        "-fschedule-insns",
        "-fno-schedule-insns",
        "-fschedule-insns2",
        "-fno-schedule-insns2",
        "-fsection-anchors",
        "-fno-section-anchors",
        "-fselective-scheduling",
        "-fno-selective-scheduling",
        "-fselective-scheduling2",
        "-fno-selective-scheduling2",
        "-fsel-sched-pipelining",
        "-fno-sel-sched-pipelining",
        "-fsel-sched-pipelining-outer-loops",
        "-fno-sel-sched-pipelining-outer-loops",
        "-fsel-sched-reschedule-pipelined",
        "-fno-sel-sched-reschedule-pipelined",
        "-fsemantic-interposition",
        "-fno-semantic-interposition",
        "-fshrink-wrap",
        "-fno-shrink-wrap",
        "-fshrink-wrap-separate",
        "-fno-shrink-wrap-separate",
        "-fsignaling-nans",
        "-fno-signaling-nans",
        "-fsigned-zeros",
        "-fno-signed-zeros",
        "-fsingle-precision-constant",
        "-fno-single-precision-constant",
        "-fsplit-ivs-in-unroller",
        "-fno-split-ivs-in-unroller",
        "-fsplit-loops",
        "-fno-split-loops",
        "-fsplit-paths",
        "-fno-split-paths",
        "-fsplit-wide-types",
        "-fno-split-wide-types",
        "-fsplit-wide-types-early",
        "-fno-split-wide-types-early",
        "-fssa-backprop",
        "-fno-ssa-backprop",
        "-fssa-phiopt",
        "-fno-ssa-phiopt",
        "-fstack-clash-protection",
        "-fno-stack-clash-protection",
        "-fstack-protector",
        "-fno-stack-protector",
        "-fstack-protector-all",
        "-fno-stack-protector-all",
        "-fstack-protector-explicit",
        "-fno-stack-protector-explicit",
        "-fstack-protector-strong",
        "-fno-stack-protector-strong",
        "-fstdarg-opt",
        "-fno-stdarg-opt",
        "-fstore-merging",
        "-fno-store-merging",
        "-fstrict-aliasing",
        "-fno-strict-aliasing",
        "-fstrict-volatile-bitfields",
        "-fno-strict-volatile-bitfields",
        "-fthread-jumps",
        "-fno-thread-jumps",
        "-ftoplevel-reorder",
        "-fno-toplevel-reorder",
        "-ftracer",
        "-fno-tracer",
        "-ftrapping-math",
        "-fno-trapping-math",
        "-ftrapv",
        "-fno-trapv",
        "-ftree-bit-ccp",
        "-fno-tree-bit-ccp",
        "-ftree-builtin-call-dce",
        "-fno-tree-builtin-call-dce",
        "-ftree-ccp",
        "-fno-tree-ccp",
        "-ftree-ch",
        "-fno-tree-ch",
        "-ftree-coalesce-vars",
        "-fno-tree-coalesce-vars",
        "-ftree-copy-prop",
        "-fno-tree-copy-prop",
        "-ftree-cselim",
        "-fno-tree-cselim",
        "-ftree-dce",
        "-fno-tree-dce",
        "-ftree-dominator-opts",
        "-fno-tree-dominator-opts",
        "-ftree-dse",
        "-fno-tree-dse",
        "-ftree-forwprop",
        "-fno-tree-forwprop",
        "-ftree-fre",
        "-fno-tree-fre",
        "-ftree-loop-distribute-patterns",
        "-fno-tree-loop-distribute-patterns",
        "-ftree-loop-distribution",
        "-fno-tree-loop-distribution",
        "-ftree-loop-if-convert",
        "-fno-tree-loop-if-convert",
        "-ftree-loop-im",
        "-fno-tree-loop-im",
        "-ftree-loop-ivcanon",
        "-fno-tree-loop-ivcanon",
        "-ftree-loop-optimize",
        "-fno-tree-loop-optimize",
        "-ftree-loop-vectorize",
        "-fno-tree-loop-vectorize",
        "-ftree-lrs",
        "-fno-tree-lrs",
        "-ftree-partial-pre",
        "-fno-tree-partial-pre",
        "-ftree-phiprop",
        "-fno-tree-phiprop",
        "-ftree-pre",
        "-fno-tree-pre",
        "-ftree-pta",
        "-fno-tree-pta",
        "-ftree-reassoc",
        "-fno-tree-reassoc",
        "-ftree-scev-cprop",
        "-fno-tree-scev-cprop",
        "-ftree-sink",
        "-fno-tree-sink",
        "-ftree-slp-vectorize",
        "-fno-tree-slp-vectorize",
        "-ftree-slsr",
        "-fno-tree-slsr",
        "-ftree-sra",
        "-fno-tree-sra",
        "-ftree-switch-conversion",
        "-fno-tree-switch-conversion",
        "-ftree-tail-merge",
        "-fno-tree-tail-merge",
        "-ftree-ter",
        "-fno-tree-ter",
        "-ftree-vectorize",
        "-fno-tree-vectorize",
        "-ftree-vrp",
        "-fno-tree-vrp",
        "-funconstrained-commons",
        "-fno-unconstrained-commons",
        "-funreachable-traps",
        "-fno-unreachable-traps",
        "-funroll-all-loops",
        "-fno-unroll-all-loops",
        "-funroll-loops",
        "-fno-unroll-loops",
        "-funsafe-math-optimizations",
        "-fno-unsafe-math-optimizations",
        "-funswitch-loops",
        "-fno-unswitch-loops",
        "-funwind-tables",
        "-fno-unwind-tables",
        "-fvar-tracking-assignments",
        "-fno-var-tracking-assignments",
        "-fvariable-expansion-in-unroller",
        "-fno-variable-expansion-in-unroller",
        "-fversion-loops-for-strides",
        "-fno-version-loops-for-strides",
        "-fvpt",
        "-fno-vpt",
        "-fweb",
        "-fno-web",
        "-fwrapv",
        "-fno-wrapv",
        "-fwrapv-pointer",
        "-fno-wrapv-pointer",
        "-pedantic-errors",
    };

    return ignoreOptions.count(argument) > 0 ||
           argument.starts_with("-finstrument-functions");
}

/**
 * This function validates the dependencies cache by comparing mtimes
 * to ensure that it is newer than the compilation database.
 */
bool depsCacheIsValid(const std::string &topbuilddir,
                      const std::string &depsdir)
{
    const std::string timestampFile =
        depsdir + "/" + RECC_DEPENDENCIES_TIMESTAMP;
    if (!buildboxcommon::FileUtils::isRegularFile(timestampFile.c_str())) {
        return false;
    }

    const std::string compilationDatabasePath =
        topbuilddir + "/" + RECC_COMPILATION_DATABASE;
    const std::chrono::system_clock::time_point compilationDatabaseMTime =
        buildboxcommon::FileUtils::getFileMtime(
            compilationDatabasePath.c_str());
    const std::chrono::system_clock::time_point depsMTime =
        buildboxcommon::FileUtils::getFileMtime(timestampFile.c_str());
    if (depsMTime < compilationDatabaseMTime) {
        BUILDBOX_LOG_DEBUG("Dependencies cache is invalid: "
                           << timestampFile
                           << " is older than compilation database at "
                           << compilationDatabasePath);
        return false;
    }

    return true;
}

/**
 * Invoke clang-scan-deps to determine dependencies.
 */
void populateDependenciesDirectory(const std::string &topbuilddir,
                                   const std::string &scanDepsPath,
                                   const std::string &depsdir)
{
    // Use a temporary directory for generated header files to ensure that
    // they aren't picked up by `*.h` glob patterns in build systems.
    buildboxcommon::TemporaryDirectory tempDir("recc");

    std::ifstream compilationDatabaseStream(topbuilddir + "/" +
                                            RECC_COMPILATION_DATABASE);
    json compilationDatabase = json::parse(compilationDatabaseStream);
    compilationDatabaseStream.close();

    json modifiedCompilationDatabase = json::array();
    std::map<std::string, std::vector<std::string>> extraArgsCache;

    for (auto &commandObject : compilationDatabase) {
        if (!commandObject.contains("file")) {
            throw std::runtime_error(
                "Command object in compilation database without file");
        }

        const std::string file = commandObject["file"];

        if (!Deps::is_source_file(file) ||
            !buildboxcommon::FileUtils::isRegularFile(file.c_str())) {
            /* Only C/C++ source files are supported by clang-scan-deps. Also
             * ignore files that don't exist as the compilation database may
             * contain files that are generated as part of the build process.
             */
            continue;
        }

        std::vector<std::string> arguments;

        if (commandObject.contains("command")) {
            arguments = ShellUtils::splitCommand(commandObject["command"]);
            commandObject.erase("command");
        }
        else if (commandObject.contains("arguments")) {
            arguments =
                commandObject["arguments"].get<std::vector<std::string>>();
        }
        else {
            BUILDBOXCOMMON_THROW_EXCEPTION(
                std::runtime_error, "Command object in compilation database "
                                    "without arguments or command");
        }

        if (arguments.empty()) {
            BUILDBOXCOMMON_THROW_EXCEPTION(
                std::runtime_error, "Command object in compilation database "
                                    "with empty argument list");
        }

        // Remove compiler arguments that aren't supported by clang-scan-deps
        // (and don't affect dependency scanning)
        arguments.erase(std::remove_if(arguments.begin(), arguments.end(),
                                       ignoreCompilerArgumentForScanDeps),
                        arguments.end());

        std::vector<std::string> compilerPrintCommand;
        // Get all the command line options that might affect the predefined
        // macros
        extractCompileCommandsOptionsForCompiler(&compilerPrintCommand,
                                                 arguments);

        std::string compilerKey =
            buildboxcommon::StringUtils::join(compilerPrintCommand);

        const auto cachedExtraArgs = extraArgsCache.find(compilerKey);
        std::vector<std::string> extraArgs;
        if (cachedExtraArgs != extraArgsCache.end()) {
            extraArgs = cachedExtraArgs->second;
        }
        else {
            extraArgs = getExtraArgsForScanDeps(tempDir.name(), compilerKey,
                                                compilerPrintCommand);
            (extraArgsCache)[compilerKey] = extraArgs;
        }
        arguments.insert(arguments.begin() + 1, extraArgs.begin(),
                         extraArgs.end());
        commandObject["arguments"] = arguments;

        modifiedCompilationDatabase.push_back(commandObject);
    }

    const std::string dependencies =
        getScanDepsOutput(scanDepsPath, modifiedCompilationDatabase, true);

    const std::string tempDepsDirName = depsdir + ".tmp";

    buildboxcommon::FileUtils::createDirectory(tempDepsDirName.c_str());

    splitScanDepsRules(dependencies, tempDepsDirName);

    const std::string timestampFile =
        tempDepsDirName + "/" + RECC_DEPENDENCIES_TIMESTAMP;
    buildboxcommon::FileUtils::writeFileAtomically(timestampFile, "",
                                                   PERMISSION_RWUSR_RALL);

    // Rename complete dependencies directory to its final name
    if (rename(tempDepsDirName.c_str(), depsdir.c_str()) < 0) {
        BUILDBOXCOMMON_THROW_SYSTEM_EXCEPTION(
            std::system_error, errno, std::system_category,
            "Failed to rename dependencies directory");
    }
}

std::string getTopBuildDir()
{
    // Directory with compilation database
    std::string topbuilddir =
        buildboxcommon::SystemUtils::getCurrentWorkingDirectory();

    while (!buildboxcommon::FileUtils::isRegularFile(
        (topbuilddir + "/" + RECC_COMPILATION_DATABASE).c_str())) {
        // Compilation database doesn't exist in current working directory.
        // Check the ancestors of the working directory
        // (e.g. for `cmake` subdirectory builds with `make`).
        std::size_t slash = topbuilddir.find_last_of("/");
        if (slash == std::string::npos || slash == 0) {
            // Compilation database not found.
            // Incompatible build system or disabled.
            return "";
        }
        topbuilddir = topbuilddir.substr(0, slash);
    }

    return topbuilddir;
}

std::string
getDependenciesDirectory(const CounterMetricCallback &recordCounterMetric)
{
    std::string topbuilddir = getTopBuildDir();
    if (topbuilddir.empty()) {
        return "";
    }

    // Allow the environment or config file to specify the command or path
    const std::string scanDepsPath =
        buildboxcommon::SystemUtils::getPathToCommand(CLANG_SCAN_DEPS);
    if (scanDepsPath.empty()) {
        // clang-scan-deps not available
        return "";
    }

    BUILDBOX_LOG_INFO("Using clang-scan-deps to get dependencies of "
                      << topbuilddir + "/" + RECC_COMPILATION_DATABASE);

    buildboxcommon::FileDescriptor topbuilddirfd(
        open(topbuilddir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC));
    if (topbuilddirfd.get() < 0) {
        BUILDBOXCOMMON_THROW_SYSTEM_EXCEPTION(
            std::system_error, errno, std::system_category,
            "Error opening top build directory \"" << topbuilddir << "\"");
    }

    const std::string depsdir = topbuilddir + "/" + RECC_DEPENDENCIES;

    if (buildboxcommon::FileUtils::isDirectory(depsdir.c_str())) {
        if (depsCacheIsValid(topbuilddir, depsdir)) {
            // Dependencies directory already written by another recc process
            return depsdir;
        }
    }

    buildboxcommon::LockFile lockfile(topbuilddirfd.get(),
                                      RECC_DEPENDENCIES_LOCK);

    // We have an exclusive lock, check whether dependencies file has been
    // written by another recc process in the meantime
    if (buildboxcommon::FileUtils::isDirectory(depsdir.c_str())) {
        if (depsCacheIsValid(topbuilddir, depsdir)) {
            // Dependencies directory already written by another recc process
            return depsdir;
        }
        else {
            buildboxcommon::FileUtils::deleteDirectory(depsdir.c_str());
        }
    }

    // This is the first recc process, invoke clang-scan-deps.
    try {
        populateDependenciesDirectory(topbuilddir, scanDepsPath, depsdir);
    }
    catch (const std::runtime_error &) {
        recordCounterMetric(COUNTER_NAME_CLANG_SCAN_DEPS_INVOCATION_FAILURE,
                            1);

        // If clang-scan-deps fails, create empty dependencies directory with
        // timestamp such that other recc processes won't try the same again.
        buildboxcommon::FileUtils::createDirectory(depsdir.c_str());
        const std::string timestampFile =
            depsdir + "/" + RECC_DEPENDENCIES_TIMESTAMP;
        buildboxcommon::FileUtils::writeFileAtomically(timestampFile, "",
                                                       PERMISSION_RWUSR_RALL);

        throw;
    }

    recordCounterMetric(COUNTER_NAME_CLANG_SCAN_DEPS_INVOCATION_SUCCESS, 1);

    return depsdir;
}

void updateResultFromDeps(std::set<std::string> *result,
                          std::set<std::string> &rawDependencies)
{
    for (const auto &dep : rawDependencies) {
        // Filter out the generated file for predefined macros
        if (dep.find("recc-scan-deps.h") != std::string::npos) {
            continue;
        }
        result->insert(dep);
    }
}

bool updateAndCacheSingleTranslationUnitDependencies(
    std::set<std::string> *result, const ParsedCommand &parsedCommand,
    const std::string &target,
    const CounterMetricCallback &recordCounterMetric,
    const std::string &depsdir, const std::string &depsPathDir)
{
    const std::string scanDepsPath =
        buildboxcommon::SystemUtils::getPathToCommand(CLANG_SCAN_DEPS);
    if (scanDepsPath.empty()) {
        // clang-scan-deps not available
        return false;
    }

    buildboxcommon::TemporaryDirectory tempDir("recc");
    std::list<std::string> arguments(parsedCommand.d_originalCommand.begin(),
                                     parsedCommand.d_originalCommand.end());

    // Remove compiler arguments that aren't supported by clang-scan-deps
    // (and don't affect dependency scanning)
    arguments.erase(std::remove_if(arguments.begin(), arguments.end(),
                                   ignoreCompilerArgumentForScanDeps),
                    arguments.end());

    std::vector<std::string> compilerPrintCommand;
    // Get all the command line options that might affect the predefined macros
    extractCompileCommandsOptionsForCompiler(&compilerPrintCommand, arguments);
    std::string compilerKey =
        buildboxcommon::StringUtils::join(compilerPrintCommand);

    std::vector<std::string> extraArgs = getExtraArgsForScanDeps(
        tempDir.name(), compilerKey, compilerPrintCommand);

    // Insert after the first element
    arguments.insert(++(arguments.begin()), extraArgs.begin(),
                     extraArgs.end());

    if (parsedCommand.d_inputFiles.size() != 1) {
        BUILDBOX_LOG_WARNING("Running clang-scan-deps is not supported for "
                             "more that one input files. Input Files: "
                             << buildboxcommon::StringUtils::join(
                                    parsedCommand.d_inputFiles, ","));
        return false;
    }

    json singleTranslationUnit;

    singleTranslationUnit["arguments"] = arguments;
    singleTranslationUnit["directory"] =
        FileUtils::getCurrentWorkingDirectory();
    singleTranslationUnit["file"] = parsedCommand.d_inputFiles[0];
    singleTranslationUnit["output"] = target;

    json compilationDatabase = json::array();
    compilationDatabase.push_back(singleTranslationUnit);

    BUILDBOX_LOG_DEBUG(
        "Generated Compilation Database for clang-scan-deps run: "
        << std::setw(4) << compilationDatabase);

    std::string lockdir = getTopBuildDir();
    if (lockdir.empty()) {
        // If the top build directory cannot be determined, use the current
        // working directory to prevent concurrent clang-scan-deps invocations
        // at least for a single directory.
        lockdir = ".";
    }
    buildboxcommon::FileDescriptor lockdirfd(
        open(lockdir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC));
    if (lockdirfd.get() < 0) {
        BUILDBOXCOMMON_THROW_SYSTEM_EXCEPTION(
            std::system_error, errno, std::system_category,
            "Error opening directory \"" << lockdir
                                         << "\" to create lock file");
    }

    // Use N scan-slot locks to limit the number of concurrent
    // clang-scan-deps invocations (controlled by RECC_SCAN_DEPS_PARALLELISM).
    // clang-scan-deps already attempts to make use of all available threads,
    // so unbounded concurrency of multiple clang-scan-deps invocation is
    // not expected to significantly improve overall performance even where
    // RAM is not a bottleneck.
    const size_t slot =
        std::hash<std::string>{}(target) % RECC_SCAN_DEPS_PARALLELISM;
    const std::string scanLockName =
        RECC_DEPENDENCIES_LOCK + "." + std::to_string(slot);

    std::string scanDepsOutput;
    {
        // Acquire one of N slot locks to limit scan concurrency.
        buildboxcommon::LockFile scanLock(lockdirfd.get(), scanLockName);
        scanDepsOutput = getScanDepsOutput(scanDepsPath, compilationDatabase);
    }
    if (!depsdir.empty()) {
        // Only update the cache if the depsdir is not an empty string
        // If the project contains no RECC_COMPILATION_DATABASE then depsdir
        // will be empty
        try {
            buildboxcommon::FileUtils::writeFileAtomically(
                depsPathDir, scanDepsOutput, PERMISSION_RWUSR_RALL);
        }
        catch (const std::system_error &e) {
            BUILDBOX_LOG_WARNING("Failed to cache clang-scan-deps output to \""
                                 << depsPathDir << "\": " << e.what());
        }
    }
    else {
        BUILDBOX_LOG_WARNING(
            "Not caching individual clang-scan-deps invocation result. A "
            "potential cause would be a missing compilation database");
    }

    std::set<std::string> rawDependencies =
        Deps::dependencies_from_make_rules(scanDepsOutput, false);
    updateResultFromDeps(result, rawDependencies);
    if (!result->empty()) {
        recordCounterMetric(COUNTER_NAME_CLANG_SCAN_DEPS_TARGET_SUCCESS, 1);
        return true;
    }

    return false;
}

} // unnamed namespace

bool ClangScanDeps::dependenciesForTarget(
    const ParsedCommand &parsedCommand, const std::string &target,
    std::set<std::string> *result,
    const CounterMetricCallback &recordCounterMetric)
{
    if (!parsedCommand.is_clang() && !parsedCommand.is_gcc()) {
        return false;
    }
    if (RECC_COMPILATION_DATABASE.empty()) {
        // Not enabled in configuration
        return false;
    }
    try {
        const std::string depsdir =
            getDependenciesDirectory(recordCounterMetric);

        const auto targetDigest =
            buildboxcommon::DigestGenerator::hash(target);
        const std::string path = depsdir + "/" + targetDigest.hash();
        if (!depsdir.empty()) {
            buildboxcommon::FileDescriptor fd(
                open(path.c_str(), O_RDONLY | O_CLOEXEC));
            if (fd.get() >= 0) {
                const std::string rules =
                    buildboxcommon::FileUtils::getFileContents(fd.get());

                auto rawDependencies =
                    Deps::dependencies_from_make_rules(rules, false);

                const auto lastCachedTimestamp =
                    buildboxcommon::FileUtils::getFileMtime(fd.get());

                for (const auto &dep : rawDependencies) {
                    // Filter out the generated file for predefined macros
                    if (dep.find("recc-scan-deps.h") != std::string::npos) {
                        continue;
                    }

                    if (!buildboxcommon::FileUtils::isRegularFile(
                            dep.c_str())) {
                        BUILDBOX_LOG_INFO(
                            "\"" << dep
                                 << "\" was removed after the invocation of "
                                    "clang-scan-deps. Running clang-scan-deps "
                                    "on individual translation unit");
                        if (updateAndCacheSingleTranslationUnitDependencies(
                                result, parsedCommand, target,
                                recordCounterMetric, depsdir, path)) {
                            return true;
                        }
                        BUILDBOX_LOG_WARNING(
                            "\""
                            << dep
                            << "\" was removed after the invocation of "
                               "clang-scan-deps. Running clang-scan-deps "
                               "on individual translation unit failed. "
                               "Falling back to the dependencies command");
                        recordCounterMetric(
                            COUNTER_NAME_CLANG_SCAN_DEPS_TARGET_FAILURE, 1);
                        return false;
                    }

                    if (buildboxcommon::FileUtils::getFileMtime(dep.c_str()) >
                        lastCachedTimestamp) {
                        BUILDBOX_LOG_INFO(
                            "\"" << dep
                                 << "\" was modified after the invocation of "
                                    "clang-scan-deps. Running clang-scan-deps "
                                    "on individual translation unit");
                        if (updateAndCacheSingleTranslationUnitDependencies(
                                result, parsedCommand, target,
                                recordCounterMetric, depsdir, path)) {
                            return true;
                        }
                        BUILDBOX_LOG_WARNING(
                            "\"" << dep
                                 << "\" was modified after the invocation of "
                                    "clang-scan-deps. Running clang-scan-deps "
                                    "on individual translation unit failed");
                        recordCounterMetric(
                            COUNTER_NAME_CLANG_SCAN_DEPS_TARGET_FAILURE, 1);
                        return false;
                    }
                }
                updateResultFromDeps(result, rawDependencies);
                if (!result->empty()) {
                    BUILDBOX_LOG_INFO("Using cached clang-scan-deps "
                                      "dependencies for \""
                                      << target << "\"");
                    recordCounterMetric(
                        COUNTER_NAME_CLANG_SCAN_DEPS_TARGET_SUCCESS, 1);
                    return true;
                }
            }
        }
        // This is expected for generated files, or when
        // RECC_COMPILATION_DATABASE is missing from the build directory
        BUILDBOX_LOG_INFO("No cached clang-scan-deps dependencies for\""
                          << target
                          << "\" found. Running clang-scan-deps on "
                             "individual translation unit");
        if (updateAndCacheSingleTranslationUnitDependencies(
                result, parsedCommand, target, recordCounterMetric, depsdir,
                path)) {
            return true;
        }
        BUILDBOX_LOG_WARNING("Running clang-scan-deps on individual "
                             "translation unit failed. Falling back to "
                             "dependencies command");
        recordCounterMetric(COUNTER_NAME_CLANG_SCAN_DEPS_TARGET_FAILURE, 1);
    }
    catch (const std::runtime_error &e) {
        BUILDBOX_LOG_WARNING("clang-scan-deps failed: " << e.what());
        BUILDBOX_LOG_INFO("Falling back to dependencies command");
        recordCounterMetric(COUNTER_NAME_CLANG_SCAN_DEPS_TARGET_FAILURE, 1);
    }

    return false;
}

#else

bool ClangScanDeps::dependenciesForTarget(const ParsedCommand &,
                                          const std::string &,
                                          std::set<std::string> *,
                                          const CounterMetricCallback &)
{
    return false;
}

#endif

} // namespace recc
