# This file is part of Buildbot.  Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members


import os
import signal
import socket

from twisted.application import internet
from twisted.internet import defer
from twisted.internet import task
from twisted.internet import threads
from twisted.python import failure
from twisted.python import log

import buildbot
from buildbot import config
from buildbot import monkeypatches
from buildbot.buildbot_net_usage_data import sendBuildbotNetUsageData
from buildbot.changes.manager import ChangeManager
from buildbot.config.master import FileLoader
from buildbot.config.master import MasterConfig
from buildbot.data import connector as dataconnector
from buildbot.data import graphql
from buildbot.db import connector as dbconnector
from buildbot.db import exceptions
from buildbot.machine.manager import MachineManager
from buildbot.mq import connector as mqconnector
from buildbot.process import cache
from buildbot.process import debug
from buildbot.process import metrics
from buildbot.process.botmaster import BotMaster
from buildbot.process.users.manager import UserManagerManager
from buildbot.schedulers.manager import SchedulerManager
from buildbot.secrets.manager import SecretManager
from buildbot.util import check_functional_environment
from buildbot.util import service
from buildbot.util.eventual import eventually
from buildbot.wamp import connector as wampconnector
from buildbot.worker import manager as workermanager
from buildbot.worker.protocols.manager.msgpack import MsgManager
from buildbot.worker.protocols.manager.pb import PBManager
from buildbot.www import service as wwwservice


class LogRotation:

    def __init__(self):
        self.rotateLength = 1 * 1000 * 1000
        self.maxRotatedFiles = 10


class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService):

    # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
    # unclaimed; this should be at least 2 to avoid false positives
    UNCLAIMED_BUILD_FACTOR = 6

    def __init__(self, basedir, configFileName=None, umask=None, reactor=None, config_loader=None):
        super().__init__()

        if reactor is None:
            from twisted.internet import reactor
        self.reactor = reactor

        self.setName("buildmaster")

        self.umask = umask

        self.basedir = basedir
        if basedir is not None:  # None is used in tests
            assert os.path.isdir(self.basedir)

        if config_loader is not None and configFileName is not None:
            raise config.ConfigErrors([
                "Can't specify both `config_loader` and `configFilename`.",
            ])
        if config_loader is None:
            if configFileName is None:
                configFileName = 'master.cfg'
            config_loader = FileLoader(self.basedir, configFileName)
        self.config_loader = config_loader
        self.configFileName = configFileName

        # flag so we don't try to do fancy things before the master is ready
        self._master_initialized = False
        self.initLock = defer.DeferredLock()

        # set up child services
        self._services_d = self.create_child_services()

        # db configured values
        self.configured_db_url = None

        # configuration / reconfiguration handling
        self.config = MasterConfig()
        self.config_version = 0  # increased by one on each reconfig
        self.reconfig_active = False
        self.reconfig_requested = False
        self.reconfig_notifier = None

        # this stores parameters used in the tac file, and is accessed by the
        # WebStatus to duplicate those values.
        self.log_rotation = LogRotation()

        # local cache for this master's object ID
        self._object_id = None

        self._got_sigterm = False

        # Check environment is sensible
        check_functional_environment(self.config)

        # figure out local hostname
        try:
            self.hostname = os.uname()[1]  # only on unix
        except AttributeError:
            self.hostname = socket.getfqdn()

        # public attributes
        self.name = (f"{self.hostname}:{os.path.abspath(self.basedir or '.')}")
        if isinstance(self.name, bytes):
            self.name = self.name.decode('ascii', 'replace')
        self.masterid = None

    @defer.inlineCallbacks
    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        yield self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        yield self.caches.setServiceParent(self)

        self.pbmanager = PBManager()
        yield self.pbmanager.setServiceParent(self)

        self.msgmanager = MsgManager()
        yield self.msgmanager.setServiceParent(self)

        self.workers = workermanager.WorkerManager(self)
        yield self.workers.setServiceParent(self)

        self.change_svc = ChangeManager()
        yield self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster()
        yield self.botmaster.setServiceParent(self)

        self.machine_manager = MachineManager()
        yield self.machine_manager.setServiceParent(self)

        self.scheduler_manager = SchedulerManager()
        yield self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        yield self.user_manager.setServiceParent(self)

        self.db = dbconnector.DBConnector(self.basedir)
        yield self.db.setServiceParent(self)

        self.wamp = wampconnector.WampConnector()
        yield self.wamp.setServiceParent(self)

        self.mq = mqconnector.MQConnector()
        yield self.mq.setServiceParent(self)

        self.data = dataconnector.DataConnector()
        yield self.data.setServiceParent(self)

        self.graphql = graphql.GraphQLConnector()
        yield self.graphql.setServiceParent(self)

        self.www = wwwservice.WWWService()
        yield self.www.setServiceParent(self)

        self.debug = debug.DebugServices()
        yield self.debug.setServiceParent(self)

        self.secrets_manager = SecretManager()
        yield self.secrets_manager.setServiceParent(self)
        self.secrets_manager.reconfig_priority = 2000

        self.service_manager = service.BuildbotServiceManager()
        yield self.service_manager.setServiceParent(self)
        self.service_manager.reconfig_priority = 1000

        self.masterHouskeepingTimer = 0

        @defer.inlineCallbacks
        def heartbeat():
            if self.masterid is not None:
                yield self.data.updates.masterActive(name=self.name,
                                                     masterid=self.masterid)
            yield self.data.updates.expireMasters()
        self.masterHeartbeatService = internet.TimerService(60, heartbeat)
        self.masterHeartbeatService.clock = self.reactor
        # we do setServiceParent only when the master is configured
        # master should advertise itself only at that time

    # setup and reconfig handling

    _already_started = False

    @defer.inlineCallbacks
    def startService(self):
        assert not self._already_started, "can only start the master once"
        self._already_started = True

        # ensure child services have been set up. Normally we would do this in serServiceParent,
        # but buildmaster is used in contexts we can't control.
        if self._services_d is not None:
            yield self._services_d
            self._services_d = None

        log.msg(f"Starting BuildMaster -- buildbot.version: {buildbot.version}")

        # Set umask
        if self.umask is not None:
            os.umask(self.umask)

        # first, apply all monkeypatches
        monkeypatches.patch_all()

        # we want to wait until the reactor is running, so we can call
        # reactor.stop() for fatal errors
        d = defer.Deferred()
        self.reactor.callWhenRunning(d.callback, None)
        yield d

        startup_succeed = False
        try:
            yield self.initLock.acquire()
            # load the configuration file, treating errors as fatal
            try:
                # run the master.cfg in thread, so that it can use blocking
                # code
                self.config = yield threads.deferToThreadPool(
                    self.reactor, self.reactor.getThreadPool(),
                    self.config_loader.loadConfig)

            except config.ConfigErrors as e:
                log.msg("Configuration Errors:")
                for msg in e.errors:
                    log.msg("  " + msg)
                log.msg("Halting master.")
                self.reactor.stop()
                return
            except Exception:
                log.err(failure.Failure(), 'while starting BuildMaster')
                self.reactor.stop()
                return

            # set up services that need access to the config before everything
            # else gets told to reconfig
            try:
                yield self.db.setup()
            except exceptions.DatabaseNotReadyError:
                # (message was already logged)
                self.reactor.stop()
                return

            yield self.mq.setup()

            # the buildbot scripts send the SIGHUP signal to reconfig master
            if hasattr(signal, "SIGHUP"):
                def sighup(*args):
                    eventually(self.reconfig)
                signal.signal(signal.SIGHUP, sighup)

            # the buildbot scripts send the SIGUSR1 signal to stop master
            if hasattr(signal, "SIGUSR1"):
                def sigusr1(*args):
                    eventually(self.botmaster.cleanShutdown)
                signal.signal(signal.SIGUSR1, sigusr1)

            # get the masterid so other services can use it in
            # startup/reconfig.  This goes directly to the DB since the data
            # API isn't initialized yet, and anyway, this method is aware of
            # the DB API since it just called its setup function
            self.masterid = yield self.db.masters.findMasterId(
                name=self.name)

            # mark this master as stopped, in case it crashed before
            yield self.data.updates.masterStopped(name=self.name,
                                                  masterid=self.masterid)

            # call the parent method
            yield super().startService()

            # We make sure the housekeeping is done before configuring in order to cleanup
            # any remaining claimed schedulers or change sources from zombie
            # masters
            yield self.data.updates.expireMasters(forceHouseKeeping=True)

            # give all services a chance to load the new configuration, rather
            # than the base configuration
            yield self.reconfigServiceWithBuildbotConfig(self.config)

            # Mark the master as active now that mq is running
            yield self.data.updates.masterActive(name=self.name,
                                                 masterid=self.masterid)

            # Start the heartbeat timer
            yield self.masterHeartbeatService.setServiceParent(self)

            # send the statistics to buildbot.net, without waiting
            self.sendBuildbotNetUsageData()
            startup_succeed = True
        except Exception:
            f = failure.Failure()
            log.err(f, 'while starting BuildMaster')
            self.reactor.stop()

        finally:
            @defer.inlineCallbacks
            def call_after_signal(sig_num, stack):
                if not self._got_sigterm:
                    self._got_sigterm = True
                    yield self.disownServiceParent()
                    self.reactor.stop()
                else:
                    log.msg('Ignoring SIGTERM, master is already shutting down.')

            signal.signal(signal.SIGTERM, call_after_signal)
            if startup_succeed:
                log.msg("BuildMaster is running")
            else:
                log.msg("BuildMaster startup failed")

            yield self.initLock.release()
            self._master_initialized = True

    def sendBuildbotNetUsageData(self):
        if "TRIAL_PYTHONPATH" in os.environ and self.config.buildbotNetUsageData is not None:
            raise RuntimeError(
                "Should not enable buildbotNetUsageData in trial tests!")
        sendBuildbotNetUsageData(self)

    @defer.inlineCallbacks
    def stopService(self):
        try:
            yield self.initLock.acquire()

            if self.running:
                yield self.botmaster.cleanShutdown(quickMode=True, stopReactor=False)

            # Mark master as stopped only after all builds are shut down. Note that masterStopped
            # would forcibly mark all related build requests, builds, steps, logs, etc. as
            # complete, so this may make state inconsistent if done while the builds are still
            # running.
            if self.masterid is not None:
                yield self.data.updates.masterStopped(
                    name=self.name, masterid=self.masterid)

            if self.running:
                yield super().stopService()

            log.msg("BuildMaster is stopped")
            self._master_initialized = False
        finally:
            yield self.initLock.release()

    @defer.inlineCallbacks
    def reconfig(self):
        # this method wraps doConfig, ensuring it is only ever called once at
        # a time, and alerting the user if the reconfig takes too long
        if self.reconfig_active:
            log.msg("reconfig already active; will reconfig again after")
            self.reconfig_requested = True
            return

        self.reconfig_active = self.reactor.seconds()
        metrics.MetricCountEvent.log("loaded_config", 1)

        # notify every 10 seconds that the reconfig is still going on, the duration of reconfigs is
        # longer on larger installations and may take a while.
        self.reconfig_notifier = task.LoopingCall(
            lambda: log.msg("reconfig is ongoing for "
                            f"{self.reactor.seconds() - self.reconfig_active:.3f} s"))
        self.reconfig_notifier.start(10, now=False)

        timer = metrics.Timer("BuildMaster.reconfig")
        timer.start()

        try:
            yield self.doReconfig()
        except Exception as e:
            log.err(e, 'while reconfiguring')
        finally:
            timer.stop()
            self.reconfig_notifier.stop()
            self.reconfig_notifier = None
            self.reconfig_active = False
            if self.reconfig_requested:
                self.reconfig_requested = False
                self.reconfig()

    @defer.inlineCallbacks
    def doReconfig(self):
        log.msg("beginning configuration update")
        time_started = self.reactor.seconds()
        changes_made = False
        failed = False
        try:
            yield self.initLock.acquire()
            # Run the master.cfg in thread, so that it can use blocking code
            new_config = yield threads.deferToThreadPool(
                self.reactor, self.reactor.getThreadPool(),
                self.config_loader.loadConfig)
            changes_made = True
            self.config_version += 1
            self.config = new_config

            yield self.reconfigServiceWithBuildbotConfig(new_config)

        except config.ConfigErrors as e:
            for msg in e.errors:
                log.msg(msg)
            failed = True

        except Exception:
            log.err(failure.Failure(), 'during reconfig:')
            failed = True

        finally:
            yield self.initLock.release()

        if failed:
            if changes_made:
                msg = "WARNING: configuration update partially applied; master may malfunction"
            else:
                msg = "configuration update aborted without making any changes"
        else:
            msg = "configuration update complete"

        log.msg(f"{msg} (took {(self.reactor.seconds() - time_started):.3f} seconds)")

    def reconfigServiceWithBuildbotConfig(self, new_config):
        if self.configured_db_url is None:
            self.configured_db_url = new_config.db['db_url']
        elif self.configured_db_url != new_config.db['db_url']:
            config.error(
                "Cannot change c['db']['db_url'] after the master has started",
            )

        if self.config.mq['type'] != new_config.mq['type']:
            raise config.ConfigErrors([
                "Cannot change c['mq']['type'] after the master has started",
            ])

        return super().reconfigServiceWithBuildbotConfig(new_config)

    # informational methods
    def allSchedulers(self):
        return list(self.scheduler_manager)

    # state maintenance (private)
    def getObjectId(self):
        """
        Return the object id for this master, for associating state with the
        master.

        @returns: ID, via Deferred
        """
        # try to get the cached value
        if self._object_id is not None:
            return defer.succeed(self._object_id)

        # failing that, get it from the DB; multiple calls to this function
        # at the same time will not hurt

        d = self.db.state.getObjectId(self.name,
                                      "buildbot.master.BuildMaster")

        @d.addCallback
        def keep(id):
            self._object_id = id
            return id
        return d

    def _getState(self, name, default=None):
        "private wrapper around C{self.db.state.getState}"
        d = self.getObjectId()

        @d.addCallback
        def get(objectid):
            return self.db.state.getState(objectid, name, default)
        return d

    def _setState(self, name, value):
        "private wrapper around C{self.db.state.setState}"
        d = self.getObjectId()

        @d.addCallback
        def set(objectid):
            return self.db.state.setState(objectid, name, value)
        return d
