// Copyright 2018-2021 Bloomberg Finance L.P
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <buildboxcommon_fileutils.h>
#include <buildboxcommon_grpctestserver.h>
#include <buildboxcommon_logging.h>
#include <buildboxcommon_remoteexecutionclient.h>
#include <buildboxcommon_temporarydirectory.h>
#include <buildboxcommon_timeutils.h>
#include <buildboxcommontest_utils.h>

#include <build/bazel/remote/execution/v2/remote_execution_mock.grpc.pb.h>
#include <chrono>
#include <filesystem>
#include <gmock/gmock.h>
#include <google/longrunning/operations_mock.grpc.pb.h>
#include <google/protobuf/util/message_differencer.h>
#include <gtest/gtest.h>

#include <set>
#include <thread>
#include <unistd.h>

using namespace buildboxcommon;
using namespace testing;

namespace {
const auto digestFunctionInitializer = []() {
    buildboxcommon::DigestGenerator::init();
    return 0;
}();
} // namespace

/*
 * This fixture sets up all of the dependencies for execute_action
 *
 * Any dependencies can be overridden by setting them in the respective tests
 */
class RemoteExecutionClientTestFixture : public ::testing::Test {
  protected:
    GrpcTestServer testServer;
    std::shared_ptr<Execution::StubInterface> executionStub;
    std::shared_ptr<MockActionCacheStub> actionCacheStub;
    std::shared_ptr<google::longrunning::Operations::StubInterface>
        operationsStub;
    std::shared_ptr<ByteStream::StubInterface> bytestreamStub;
    std::shared_ptr<ContentAddressableStorage::StubInterface> casStub;

    std::shared_ptr<buildboxcommon::GrpcClient> grpcClient;
    std::shared_ptr<RemoteExecutionClient> reClient;
    std::shared_ptr<CASClient> casClient;

    const std::string instance_name = "";

    Digest actionDigest;
    ExecuteRequest expectedExecuteRequest;
    ExecuteResponse executeResponse;
    WaitExecutionRequest expectedWaitExecutionRequest;

    Digest stdErrDigest;

    google::longrunning::Operation incompleteOperation;
    google::longrunning::Operation operation;

    RemoteExecutionClientTestFixture()
        : executionStub(Execution::NewStub(testServer.channel())),
          actionCacheStub(std::make_shared<MockActionCacheStub>()),
          operationsStub(Operations::NewStub(testServer.channel())),
          bytestreamStub(ByteStream::NewStub(testServer.channel())),
          casStub(ContentAddressableStorage::NewStub(testServer.channel())),
          grpcClient(std::make_shared<buildboxcommon::GrpcClient>()),
          reClient(
              std::make_shared<RemoteExecutionClient>(grpcClient, grpcClient)),
          casClient(std::make_shared<CASClient>(grpcClient))
    {
        grpcClient->setInstanceName(instance_name);
        reClient->init(executionStub, actionCacheStub, operationsStub);
        casClient->init(bytestreamStub, casStub, nullptr, nullptr);

        // Construct the Digest we're passing in, and the ExecuteRequest we
        // expect the RemoteExecutionClient to send as a result.
        actionDigest.set_hash("Action digest hash here");
        *expectedExecuteRequest.mutable_action_digest() = actionDigest;

        // Begin constructing a fake ExecuteResponse to return to the client.
        const auto actionResultProto = executeResponse.mutable_result();
        actionResultProto->set_stdout_raw("Raw stdout.");
        std::string stdErr("Stderr, which will be sent as a digest.");
        stdErrDigest = DigestGenerator::hash(stdErr);
        *actionResultProto->mutable_stderr_digest() = stdErrDigest;
        actionResultProto->set_exit_code(123);

        // Add an output file to the response.
        OutputFile outputFile;
        outputFile.set_path("some/path/with/slashes.txt");
        outputFile.mutable_digest()->set_hash("File hash goes here");
        outputFile.mutable_digest()->set_size_bytes(1);
        *actionResultProto->add_output_files() = outputFile;

        // Add a minimal output tree to the response.
        Tree tree;
        auto treeRootFile = tree.mutable_root()->add_files();
        treeRootFile->mutable_digest()->set_hash("File hash goes here");
        treeRootFile->mutable_digest()->set_size_bytes(1);
        treeRootFile->set_name("out.txt");
        const auto treeDigest =
            DigestGenerator::hash(tree.SerializeAsString());
        *actionResultProto->add_output_directories()->mutable_tree_digest() =
            treeDigest;
        actionResultProto->mutable_output_directories(0)->set_path(
            "output/directory");

        incompleteOperation.set_name("operations/{1}");

        // Return a completed Operation when the client sends the Execute
        // request.
        operation.set_name("operations/{1}");
        operation.set_done(true);
        operation.mutable_response()->PackFrom(executeResponse);

        *expectedWaitExecutionRequest.mutable_name() = operation.name();
    }
};

class RemoteExecutionClientRetryTestFixture
    : public RemoteExecutionClientTestFixture {
  protected:
    RemoteExecutionClientRetryTestFixture()
        : RemoteExecutionClientTestFixture()
    {
        grpcClient->setRetryLimit(2);
    }
};

class RemoteExecutionClientPolicyTestFixture
    : public RemoteExecutionClientTestFixture {
  protected:
    RemoteExecutionClientPolicyTestFixture()
        : RemoteExecutionClientTestFixture()
    {
        ExecutionPolicy execPolicy;
        execPolicy.set_priority(1);
        expectedExecuteRequest.mutable_execution_policy()->CopyFrom(
            execPolicy);
        expectedExecuteRequest.set_skip_cache_lookup(true);
    }
};

MATCHER_P(MessageEq, expected, "")
{
    return google::protobuf::util::MessageDifferencer::Equals(expected, arg);
}

TEST_F(RemoteExecutionClientTestFixture, ExecuteRequestProxy)
{
    std::thread serverHandler([this]() {
        GrpcTestServerContext ctx(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx.read(expectedExecuteRequest);
        ctx.writeAndFinish(operation);
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::function<bool()> stop_lambda = [&] { return false; };
    try {
        buildboxcommontest::MockWriter writer =
            buildboxcommontest::MockWriter();
        grpc::Status status = reClient->proxyExecuteRequest(
            expectedExecuteRequest, stop_lambda, &writer);

        EXPECT_TRUE(writer.write_calls == 1);
        EXPECT_TRUE(status.ok());
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture,
       ExecuteRequestProxyOperationNamePrefixing)
{
    std::thread serverHandler([this]() {
        GrpcTestServerContext ctx(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx.read(expectedExecuteRequest);
        ctx.writeAndFinish(operation);
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::function<bool()> stop_lambda = [&] { return false; };
    try {
        buildboxcommontest::MockWriter writer =
            buildboxcommontest::MockWriter();
        grpc::Status status = reClient->proxyExecuteRequest(
            expectedExecuteRequest, stop_lambda, &writer, "local-instance#");

        EXPECT_TRUE(writer.write_calls == 1);
        EXPECT_TRUE(writer.operations[0].name() ==
                    "local-instance#" + operation.name());
        EXPECT_TRUE(status.ok());
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, WaitExecutionRequestProxy)
{
    std::thread serverHandler([this]() {
        GrpcTestServerContext ctx(
            &testServer,
            "/build.bazel.remote.execution.v2.Execution/WaitExecution");
        ctx.read(expectedWaitExecutionRequest);
        ctx.writeAndFinish(operation);
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::function<bool()> stop_lambda = [&] { return false; };
    try {
        buildboxcommontest::MockWriter writer =
            buildboxcommontest::MockWriter();
        grpc::Status status = reClient->proxyWaitExecutionRequest(
            expectedWaitExecutionRequest, stop_lambda, &writer);

        EXPECT_TRUE(writer.write_calls == 1);
        EXPECT_TRUE(status.ok());
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture,
       WaitExecutionRequestProxyOperationNamePrefixing)
{
    std::thread serverHandler([this]() {
        GrpcTestServerContext ctx(
            &testServer,
            "/build.bazel.remote.execution.v2.Execution/WaitExecution");
        ctx.read(expectedWaitExecutionRequest);
        ctx.writeAndFinish(operation);
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::function<bool()> stop_lambda = [&] { return false; };
    try {
        buildboxcommontest::MockWriter writer =
            buildboxcommontest::MockWriter();
        grpc::Status status = reClient->proxyWaitExecutionRequest(
            expectedWaitExecutionRequest, stop_lambda, &writer,
            "local-instance#");

        EXPECT_TRUE(writer.write_calls == 1);
        EXPECT_TRUE(writer.operations[0].name() ==
                    "local-instance#" + operation.name());
        EXPECT_TRUE(status.ok());
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, ExecuteActionTest)
{
    std::thread serverHandler([this]() {
        GrpcTestServerContext ctx(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx.read(expectedExecuteRequest);
        ctx.writeAndFinish(operation);
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::atomic_bool stop_requested(false);
    try {
        const auto actionResult =
            reClient->executeAction(actionDigest, stop_requested);

        EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(
            executeResponse.result(), actionResult));
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientPolicyTestFixture, ExecuteActionWithPolicyTest)
{
    std::thread serverHandler([this]() {
        GrpcTestServerContext ctx(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx.read(expectedExecuteRequest);
        ctx.writeAndFinish(operation);
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::atomic_bool stop_requested(false);
    try {
        ExecutionPolicy execPolicy;
        execPolicy.set_priority(1);
        const auto actionResult = reClient->executeAction(
            actionDigest, stop_requested, true, &execPolicy);

        EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(
            executeResponse.result(), actionResult));
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, AsyncExecuteActionFails)
{
    grpcClient->setRequestTimeout(std::chrono::seconds(1));

    std::thread serverHandler([this]() {
        GrpcTestServerContext ctx(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx.read(expectedExecuteRequest);
        ctx.finish(grpc::Status(grpc::INVALID_ARGUMENT, "failed in test"));
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::atomic_bool stop_requested(false);
    try {
        ASSERT_THROW(
            reClient->asyncExecuteAction(actionDigest, stop_requested),
            GrpcError);
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, AsyncExecuteAction)
{
    grpcClient->setRequestTimeout(std::chrono::seconds(1));

    WaitExecutionRequest expectedWaitExecutionRequest;
    expectedWaitExecutionRequest.set_name(operation.name());

    std::thread serverHandler([this, &expectedWaitExecutionRequest]() {
        GrpcTestServerContext ctx(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx.read(expectedExecuteRequest);

        google::longrunning::Operation incompleteOperation;
        incompleteOperation.set_name("operations/{1}");
        ctx.write(incompleteOperation);

        // Handle expected WaitExecution request
        GrpcTestServerContext ctx2(
            &testServer,
            "/build.bazel.remote.execution.v2.Execution/WaitExecution");
        ctx2.read(expectedWaitExecutionRequest);
        ctx2.writeAndFinish(operation);
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::atomic_bool stop_requested(false);
    try {
        auto receivedOperation =
            reClient->asyncExecuteAction(actionDigest, stop_requested);
        EXPECT_EQ(receivedOperation.name(), "operations/{1}");
        EXPECT_FALSE(receivedOperation.done());

        receivedOperation = reClient->waitExecution(receivedOperation.name());
        EXPECT_TRUE(receivedOperation.done());

        ExecuteResponse receivedExecuteResponse;
        EXPECT_TRUE(
            receivedOperation.response().UnpackTo(&receivedExecuteResponse));

        EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(
            executeResponse, receivedExecuteResponse));
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, ExecuteActionTimeoutTest1)
{
    grpcClient->setRequestTimeout(std::chrono::seconds(1));

    std::thread serverHandler([this]() {
        GrpcTestServerContext ctx(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx.read(expectedExecuteRequest);

        google::longrunning::Operation incompleteOperation;
        incompleteOperation.set_name("fake-operation");
        ctx.write(incompleteOperation);

        // Further operation updates are allowed to take longer than the
        // request timeout.
        std::this_thread::sleep_for(std::chrono::seconds(2));
        ctx.writeAndFinish(operation);
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::atomic_bool stop_requested(false);
    try {
        const auto actionResult =
            reClient->executeAction(actionDigest, stop_requested);

        EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(
            executeResponse.result(), actionResult));
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, ExecuteActionTimeoutTest2)
{
    grpcClient->setRequestTimeout(std::chrono::seconds(1));

    std::thread serverHandler([this]() {
        GrpcTestServerContext ctx(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx.read(expectedExecuteRequest);

        // In this test the server takes too much time without sending an
        // initial Operation message.
        std::this_thread::sleep_for(std::chrono::seconds(2));
    });

    std::atomic_bool stop_requested(false);
    EXPECT_THROW(
        {
            const auto actionResult =
                reClient->executeAction(actionDigest, stop_requested);
        },
        GrpcError);

    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, ExecuteActionSubmissionFailTest)
{
    std::thread serverHandler([this]() {
        GrpcTestServerContext ctx(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx.read(expectedExecuteRequest);
        ctx.finish(grpc::Status(grpc::INVALID_ARGUMENT, "failed in test"));
    });

    std::atomic_bool stop_requested(false);
    try {
        // Verify that an exception is thrown when Execute returns an error
        EXPECT_THROW(reClient->executeAction(actionDigest, stop_requested),
                     GrpcError);
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, ExecuteActionGetOperationFailTest)
{
    std::thread serverHandler([this]() {
        GrpcTestServerContext ctx(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx.read(expectedExecuteRequest);

        google::longrunning::Operation incompleteOperation;
        incompleteOperation.set_name("operations/{1}");
        ctx.write(incompleteOperation);

        ctx.finish(grpc::Status(grpc::INVALID_ARGUMENT, "failed in test"));
    });

    std::atomic_bool stop_requested(false);
    try {
        // Verify that an exception is thrown when Execute initially succeeds
        // but an error is returned when streaming operation updates
        EXPECT_THROW(reClient->executeAction(actionDigest, stop_requested),
                     GrpcError);
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientRetryTestFixture, RpcRetryTest)
{
    ASSERT_EQ(grpcClient->retryLimit(), 2);

    std::thread serverHandler([this]() {
        // Fail the first attempt
        GrpcTestServerContext ctx1(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx1.read(expectedExecuteRequest);
        ctx1.finish(grpc::Status(grpc::UNAVAILABLE, "failed"));

        // Return a successful response on the second attempt
        GrpcTestServerContext ctx2(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx2.read(expectedExecuteRequest);
        ctx2.writeAndFinish(operation);
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::atomic_bool stop_requested(false);
    try {
        const auto actionResult =
            reClient->executeAction(actionDigest, stop_requested);

        EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(
            executeResponse.result(), actionResult));
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientRetryTestFixture, RpcRetryWithWaitExecute)
{
    ASSERT_EQ(grpcClient->retryLimit(), 2);

    std::thread serverHandler([this]() {
        // Succeed in creating an operation, but fail with UNAVAILABLE after
        GrpcTestServerContext ctx1(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx1.read(expectedExecuteRequest);
        ctx1.write(incompleteOperation);
        ctx1.finish(grpc::Status(grpc::UNAVAILABLE, "failed"));

        // Return operation on 2nd attempt
        GrpcTestServerContext ctx2(
            &testServer,
            "/build.bazel.remote.execution.v2.Execution/WaitExecution");
        ctx2.read(expectedWaitExecutionRequest);
        ctx2.writeAndFinish(operation);
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::atomic_bool stop_requested(false);
    try {
        const auto actionResult =
            reClient->executeAction(actionDigest, stop_requested);

        EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(
            executeResponse.result(), actionResult));
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientRetryTestFixture, RpcRetryWithProxyWaitExecute)
{
    ASSERT_EQ(grpcClient->retryLimit(), 2);

    std::thread serverHandler([this]() {
        // Succeed in creating an operation, but fail with UNAVAILABLE after
        GrpcTestServerContext ctx1(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx1.read(expectedExecuteRequest);
        ctx1.write(incompleteOperation);
        ctx1.finish(grpc::Status(grpc::UNAVAILABLE, "failed"));

        // Return operation on 2nd attempt
        GrpcTestServerContext ctx2(
            &testServer,
            "/build.bazel.remote.execution.v2.Execution/WaitExecution");

        ctx2.read(expectedWaitExecutionRequest);
        ctx2.writeAndFinish(operation);
    });

    // Ask the client to execute the action, and make sure the result is
    // correct.
    std::function<bool()> stop_lambda = [&] { return false; };
    try {
        buildboxcommontest::MockWriter writer =
            buildboxcommontest::MockWriter();
        grpc::Status status = reClient->proxyExecuteRequest(
            expectedExecuteRequest, stop_lambda, &writer, "bar#");

        EXPECT_TRUE(status.ok());
        // One during first response, one from retry
        EXPECT_EQ(writer.write_calls, 2);
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, CancelOperation)
{
    std::atomic_bool stop_requested(false);

    // Return an incomplete Operation when the client sends the
    // Execute request.
    operation.set_done(false);
    operation.set_name("fake-operation");

    CancelOperationRequest expectedCancelRequest;
    expectedCancelRequest.set_name(operation.name());

    std::thread serverHandler([this, &stop_requested,
                               &expectedCancelRequest]() {
        // Accept Execute request but never finish it
        GrpcTestServerContext ctx1(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx1.read(expectedExecuteRequest);
        ctx1.write(operation);

        // Request cancellation
        stop_requested = true;

        // Handle expected CancelOperation request
        GrpcTestServerContext ctx2(
            &testServer, "/google.longrunning.Operations/CancelOperation");
        ctx2.read(expectedCancelRequest);
        ctx2.writeAndFinish(google::protobuf::Empty());
    });

    try {
        ASSERT_THROW(reClient->executeAction(actionDigest, stop_requested),
                     buildboxcommon::GrpcError);
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, GetOperation)
{
    std::atomic_bool stop_requested(false);

    // Return an incomplete Operation when the client sends the
    // Execute request.
    operation.set_done(false);
    operation.set_name("fake-operation");

    GetOperationRequest expectedGetRequest;
    expectedGetRequest.set_name(operation.name());

    std::thread serverHandler([this, &expectedGetRequest]() {
        // Accept Execute request but never finish it
        GrpcTestServerContext ctx1(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx1.read(expectedExecuteRequest);
        ctx1.write(operation);

        // Handle expected GetOperation request
        GrpcTestServerContext ctx2(
            &testServer, "/google.longrunning.Operations/GetOperation");
        ctx2.read(expectedGetRequest);
        ctx2.writeAndFinish(operation);
    });

    try {
        reClient->asyncExecuteAction(actionDigest, stop_requested);

        const google::longrunning::Operation op =
            reClient->getOperation("fake-operation");
        ASSERT_EQ(op.name(), "fake-operation");
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, ListOperations)
{
    std::atomic_bool stop_requested(false);

    // Return an incomplete Operation when the client sends the
    // Execute request.
    operation.set_done(false);
    operation.set_name("fake-operation");

    ListOperationsRequest expectedListRequest;
    expectedListRequest.set_name("query");
    expectedListRequest.set_filter("stage!=completed");
    expectedListRequest.set_page_size(10);
    expectedListRequest.set_page_token("");

    ListOperationsResponse expectedResponse;
    Operation *ptr = expectedResponse.add_operations();
    *ptr = operation;

    std::thread serverHandler([this, &expectedListRequest,
                               &expectedResponse]() {
        // Accept Execute request but never finish it
        GrpcTestServerContext ctx1(
            &testServer, "/build.bazel.remote.execution.v2.Execution/Execute");
        ctx1.read(expectedExecuteRequest);
        ctx1.write(operation);

        // Handle expected GetOperation request
        GrpcTestServerContext ctx2(
            &testServer, "/google.longrunning.Operations/ListOperations");
        ctx2.read(expectedListRequest);
        ctx2.writeAndFinish(expectedResponse);
    });

    try {
        reClient->asyncExecuteAction(actionDigest, stop_requested);

        const google::longrunning::ListOperationsResponse actualResponse =
            reClient->listOperations("query", "stage!=completed", 10, "");
        ASSERT_EQ(actualResponse.operations(0).name(), "fake-operation");
        ASSERT_EQ(actualResponse.operations_size(), 1);
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();
}

TEST_F(RemoteExecutionClientTestFixture, ActionCacheTestMiss)
{
    EXPECT_CALL(*actionCacheStub, GetActionResult(_, _, _))
        .WillOnce(Return(grpc::Status(grpc::NOT_FOUND, "not found")));
    // `NOT_FOUND` => return false and do not write the ActionResult parameter.

    ActionResult actionResult;
    actionResult.set_exit_code(123);

    ActionResult actionResultOut = actionResult;
    std::set<std::string> outputs;

    const bool in_cache = reClient->fetchFromActionCache(actionDigest, outputs,
                                                         &actionResultOut);

    EXPECT_FALSE(in_cache);
    EXPECT_EQ(actionResultOut.exit_code(), actionResult.exit_code());
}

TEST_F(RemoteExecutionClientTestFixture, ActionCacheTestHit)
{
    EXPECT_CALL(*actionCacheStub, GetActionResult(_, _, _))
        .WillOnce(Return(grpc::Status::OK));
    // Return true and write the ActionResult parameter with the fetched one.

    ActionResult actionResult;
    actionResult.set_exit_code(123);

    ActionResult actionResultOut = actionResult;
    std::set<std::string> outputs;

    const bool in_cache = reClient->fetchFromActionCache(actionDigest, outputs,
                                                         &actionResultOut);

    EXPECT_TRUE(in_cache);
    EXPECT_EQ(actionResultOut.exit_code(), 0);
}

TEST_F(RemoteExecutionClientTestFixture, ActionCacheTestServerError)
{
    EXPECT_CALL(*actionCacheStub, GetActionResult(_, _, _))
        .WillOnce(Return(grpc::Status(grpc::StatusCode::PERMISSION_DENIED,
                                      "permission denied")));
    // All other server errors other than `NOT_FOUND` are thrown.

    ActionResult actionResultOut;
    std::set<std::string> outputs;
    ASSERT_THROW(reClient->fetchFromActionCache(actionDigest, outputs,
                                                &actionResultOut),
                 std::runtime_error);
}

TEST_F(RemoteExecutionClientTestFixture, ActionCacheTestMissNoActionResult)
{
    EXPECT_CALL(*actionCacheStub, GetActionResult(_, _, _))
        .WillOnce(Return(grpc::Status(grpc::NOT_FOUND, "not found")));
    // `NOT_FOUND` => return false and do not write the ActionResult parameter.

    std::set<std::string> outputs;
    const bool in_cache =
        reClient->fetchFromActionCache(actionDigest, outputs, nullptr);

    EXPECT_FALSE(in_cache);
}

TEST_F(RemoteExecutionClientTestFixture, ActionCacheHitNoActionResult)
{
    EXPECT_CALL(*actionCacheStub, GetActionResult(_, _, _))
        .WillOnce(Return(grpc::Status::OK));

    std::set<std::string> outputs;

    const bool in_cache =
        reClient->fetchFromActionCache(actionDigest, outputs, nullptr);

    EXPECT_TRUE(in_cache);
}

TEST_F(RemoteExecutionClientTestFixture, ActionCacheUpdate)
{
    ActionResult actionResult;
    actionResult.set_exit_code(123);

    UpdateActionResultRequest expectedUpdateRequest;
    expectedUpdateRequest.mutable_action_digest()->CopyFrom(actionDigest);
    expectedUpdateRequest.mutable_action_result()->CopyFrom(actionResult);

    EXPECT_CALL(*actionCacheStub,
                UpdateActionResult(_, MessageEq(expectedUpdateRequest), _))
        .WillOnce(
            DoAll(SetArgPointee<2>(actionResult), Return(grpc::Status::OK)));

    reClient->updateActionCache(actionDigest, actionResult);
}

static std::string getSymlinkTarget(int dirfd, const char *path)
{
    std::string target(PATH_MAX, '\0');
    ssize_t target_len =
        readlinkat(dirfd, path, &target[0], target.size() - 1);
    EXPECT_GT(target_len, 0);
    target.resize(target_len);
    return target;
}

TEST_F(RemoteExecutionClientTestFixture, DownloadOutputs)
{
    ActionResult actionResult;

    const mode_t mode = 0700;
    const std::chrono::system_clock::time_point mtime{};

    const std::string data1 = "hello, world";
    const std::string data2 = "two";
    const std::string data3 = "three";
    const Digest digest1 = DigestGenerator::hash(data1);
    const Digest digest2 = DigestGenerator::hash(data2);
    const Digest digest3 = DigestGenerator::hash(data3);

    // Check output files at the top level and in a subdirectory
    OutputFile outputFile1;
    outputFile1.set_path("hello.txt");
    outputFile1.mutable_digest()->CopyFrom(digest1);
    outputFile1.mutable_node_properties()->mutable_unix_mode()->set_value(
        mode);
    *outputFile1.mutable_node_properties()->mutable_mtime() =
        TimeUtils::make_timestamp(mtime);

    *actionResult.add_output_files() = outputFile1;
    OutputFile outputFile2;
    outputFile2.set_path("subdir/nested/two.txt");
    outputFile2.mutable_digest()->CopyFrom(digest2);
    *actionResult.add_output_files() = outputFile2;

    OutputSymlink outputSymlink;
    outputSymlink.set_path("subdir/symlink");
    outputSymlink.set_target("target one");
    *actionResult.add_output_symlinks() = outputSymlink;

    OutputDirectory outputDirectory;
    Tree tree;

    auto treeSubdir = tree.add_children();
    treeSubdir->mutable_node_properties()->mutable_unix_mode()->set_value(
        mode);
    *treeSubdir->mutable_node_properties()->mutable_mtime() =
        TimeUtils::make_timestamp(mtime);
    auto treeSubdirFile = treeSubdir->add_files();
    // Check handling of duplicate file digest
    treeSubdirFile->mutable_digest()->CopyFrom(digest2);
    treeSubdirFile->set_name("two.txt");
    auto treeSubdirNode = tree.mutable_root()->add_directories();
    treeSubdirNode->mutable_digest()->CopyFrom(
        DigestGenerator::hash(treeSubdir->SerializeAsString()));
    treeSubdirNode->set_name("treesubdir");
    auto dupTreeSubdirNode = tree.mutable_root()->add_directories();
    dupTreeSubdirNode->mutable_digest()->CopyFrom(
        DigestGenerator::hash(treeSubdir->SerializeAsString()));
    dupTreeSubdirNode->set_name("duptreesubdir");

    auto treeRootFile1 = tree.mutable_root()->add_files();
    treeRootFile1->mutable_digest()->CopyFrom(digest3);
    treeRootFile1->set_name("three.txt");

    auto treeRootSymlink = tree.mutable_root()->add_symlinks();
    treeRootSymlink->set_name("symlink");
    treeRootSymlink->set_target("target two");

    const auto treeData = tree.SerializeAsString();
    const auto treeDigest = DigestGenerator::hash(treeData);
    outputDirectory.mutable_tree_digest()->CopyFrom(treeDigest);
    outputDirectory.set_path("dir");
    *actionResult.add_output_directories() = outputDirectory;

    std::thread serverHandler([&]() {
        // The client needs to fetch the tree first
        BatchReadBlobsRequest expectedTreeReadRequest;
        *expectedTreeReadRequest.add_digests() = treeDigest;
        BatchReadBlobsResponse treeReadResponse;
        auto treeResp = treeReadResponse.add_responses();
        treeResp->mutable_digest()->CopyFrom(treeDigest);
        treeResp->set_data(treeData);

        GrpcTestServerContext ctx1(&testServer,
                                   "/build.bazel.remote.execution.v2."
                                   "ContentAddressableStorage/BatchReadBlobs");
        ctx1.read(expectedTreeReadRequest,
                  google::protobuf::util::MessageDifferencer::
                      RepeatedFieldComparison::AS_SET);
        ctx1.writeAndFinish(treeReadResponse);

        // And then the client can fetch all file blobs in a single batch
        BatchReadBlobsRequest expectedReadRequest;
        *expectedReadRequest.add_digests() = digest1;
        *expectedReadRequest.add_digests() = digest2;
        *expectedReadRequest.add_digests() = digest3;
        BatchReadBlobsResponse readResponse;
        auto resp1 = readResponse.add_responses();
        resp1->mutable_digest()->CopyFrom(digest1);
        resp1->set_data(data1);
        auto resp2 = readResponse.add_responses();
        resp2->mutable_digest()->CopyFrom(digest2);
        resp2->set_data(data2);
        auto resp3 = readResponse.add_responses();
        resp3->mutable_digest()->CopyFrom(digest3);
        resp3->set_data(data3);

        GrpcTestServerContext ctx2(&testServer,
                                   "/build.bazel.remote.execution.v2."
                                   "ContentAddressableStorage/BatchReadBlobs");
        ctx2.read(expectedReadRequest,
                  google::protobuf::util::MessageDifferencer::
                      RepeatedFieldComparison::AS_SET);
        ctx2.writeAndFinish(readResponse);
    });

    TemporaryDirectory outputDir;
    FileDescriptor dirfd(open(outputDir.name(), O_RDONLY | O_DIRECTORY));
    ASSERT_GE(dirfd.get(), 0);

    // Create one of the output files with different content to
    // validate existing files are replaced.
    FileUtils::writeFileAtomically(outputDir.strname() + "/hello.txt",
                                   "old content");
    std::filesystem::create_directory(outputDir.strname() + "/dir");
    std::filesystem::create_symlink(outputDir.strname() + "/hello.txt",
                                    outputDir.strname() + "/dir/symlink");

    try {
        reClient->downloadOutputs(casClient.get(), actionResult, dirfd.get(),
                                  nullptr);
    }
    catch (...) {
        serverHandler.join();
        throw;
    }
    serverHandler.join();

    // Verify output files
    const FileDescriptor outputFileFd1(
        openat(dirfd.get(), "hello.txt", O_RDONLY));
    const FileDescriptor treeSubdirFd(
        openat(dirfd.get(), "dir/treesubdir", O_DIRECTORY | O_RDONLY));
    EXPECT_EQ(FileUtils::getFileContents(dirfd.get(), "hello.txt"), data1);
    EXPECT_EQ(FileUtils::getUnixMode(outputFileFd1.get()) & 0777, mode);
    EXPECT_EQ(FileUtils::getFileMtime(outputFileFd1.get()), mtime);
    EXPECT_EQ(FileUtils::getUnixMode(treeSubdirFd.get()) & 0777, mode);
    EXPECT_EQ(FileUtils::getFileMtime(treeSubdirFd.get()), mtime);
    EXPECT_EQ(FileUtils::getFileContents(dirfd.get(), "subdir/nested/two.txt"),
              data2);
    EXPECT_EQ(
        FileUtils::getFileContents(dirfd.get(), "dir/treesubdir/two.txt"),
        data2);
    EXPECT_EQ(
        FileUtils::getFileContents(dirfd.get(), "dir/duptreesubdir/two.txt"),
        data2);
    EXPECT_EQ(FileUtils::getFileContents(dirfd.get(), "dir/three.txt"), data3);
    EXPECT_EQ(getSymlinkTarget(dirfd.get(), "subdir/symlink"), "target one");
    EXPECT_EQ(getSymlinkTarget(dirfd.get(), "dir/symlink"), "target two");
}

TEST(RemoteExecutionClient, checkThrows)
{
    std::shared_ptr<RemoteExecutionClient> reClient =
        std::make_shared<RemoteExecutionClient>(nullptr, nullptr);
    reClient->init();

    EXPECT_THROW(
        {
            try {
                Digest d;
                std::set<std::string> outputs;
                ActionResult r;
                reClient->fetchFromActionCache(d, outputs, &r);
            }
            catch (const std::runtime_error &e) {
                EXPECT_STREQ("ActionCache Stub not Configured", e.what());
                throw;
            }
        },
        std::runtime_error);

    EXPECT_THROW(
        {
            try {
                Digest d;
                ActionResult r;
                reClient->updateActionCache(d, r);
            }
            catch (const std::runtime_error &e) {
                EXPECT_STREQ("ActionCache Stub not Configured", e.what());
                throw;
            }
        },
        std::runtime_error);

    EXPECT_THROW(
        {
            try {
                Digest d;
                std::atomic_bool b;
                reClient->executeAction(d, b, false);
            }
            catch (const std::runtime_error &e) {
                EXPECT_STREQ("Execution Client not configured", e.what());
                throw;
            }
        },
        std::runtime_error);

    EXPECT_THROW(
        {
            try {
                ExecuteRequest r;
                std::function<bool()> stop_lambda = [&] { return false; };
                reClient->proxyExecuteRequest(r, stop_lambda, nullptr);
            }
            catch (const std::runtime_error &e) {
                EXPECT_STREQ("Execution Stubs not Configured", e.what());
                throw;
            }
        },
        std::runtime_error);

    EXPECT_THROW(
        {
            try {
                WaitExecutionRequest r;
                std::function<bool()> stop_lambda = [&] { return false; };
                reClient->proxyWaitExecutionRequest(r, stop_lambda, nullptr);
            }
            catch (const std::runtime_error &e) {
                EXPECT_STREQ("Execution Stubs not Configured", e.what());
                throw;
            }
        },
        std::runtime_error);

    EXPECT_THROW(
        {
            try {
                GetOperationRequest request;
                Operation response;
                reClient->proxyGetOperationRequest(request, &response);
            }
            catch (const std::runtime_error &e) {
                EXPECT_STREQ("Execution Stubs not Configured", e.what());
                throw;
            }
        },
        std::runtime_error);

    EXPECT_THROW(
        {
            try {
                CancelOperationRequest request;
                google::protobuf::Empty response;
                reClient->proxyCancelOperationRequest(request, &response);
            }
            catch (const std::runtime_error &e) {
                EXPECT_STREQ("Execution Stubs not Configured", e.what());
                throw;
            }
        },
        std::runtime_error);

    EXPECT_THROW(
        {
            try {
                std::string name;
                std::string filter;
                int page_size = 10;
                std::string page_token;
                reClient->listOperations(name, filter, page_size, page_token);
            }
            catch (const std::runtime_error &e) {
                EXPECT_STREQ("Execution Stubs not Configured", e.what());
                throw;
            }
        },
        std::runtime_error);

    EXPECT_THROW(
        {
            try {
                ListOperationsRequest request;
                ListOperationsResponse response;
                reClient->proxyListOperationsRequest(request, &response);
            }
            catch (const std::runtime_error &e) {
                EXPECT_STREQ("Execution Stubs not Configured", e.what());
                throw;
            }
        },
        std::runtime_error);
}
