import sys
import os
import time
import apt_pkg

from twisted.internet.defer import Deferred, succeed, inlineCallbacks
from twisted.internet import reactor


from landscape.lib.fs import create_file, touch_file
from landscape.lib.fetch import fetch_async, FetchError
from landscape.lib import bpickle
from landscape.package.store import (
    PackageStore, UnknownHashIDRequest, FakePackageStore)
from landscape.package.reporter import (
    PackageReporter, HASH_ID_REQUEST_TIMEOUT, main, find_reporter_command,
    PackageReporterConfiguration, FakeGlobalReporter, FakeReporter)
from landscape.package import reporter
from landscape.package.facade import AptFacade
from landscape.package.tests.helpers import (
    AptFacadeHelper, SimpleRepositoryHelper,
    HASH1, HASH2, HASH3, PKGNAME1)
from landscape.tests.helpers import (
    LandscapeTest, BrokerServiceHelper, EnvironSaverHelper)
from landscape.tests.mocker import ANY

SAMPLE_LSB_RELEASE = "DISTRIB_CODENAME=codename\n"


class PackageReporterConfigurationTest(LandscapeTest):

    def test_force_apt_update_option(self):
        """
        The L{PackageReporterConfiguration} supports a '--force-apt-update'
        command line option.
        """
        config = PackageReporterConfiguration()
        config.default_config_filenames = (self.makeFile(""), )
        self.assertFalse(config.force_apt_update)
        config.load(["--force-apt-update"])
        self.assertTrue(config.force_apt_update)


class PackageReporterAptTest(LandscapeTest):

    helpers = [AptFacadeHelper, SimpleRepositoryHelper, BrokerServiceHelper]

    Facade = AptFacade

    def setUp(self):
        super(PackageReporterAptTest, self).setUp()
        self.store = PackageStore(self.makeFile())
        self.config = PackageReporterConfiguration()
        self.reporter = PackageReporter(
            self.store, self.facade, self.remote, self.config)
        self.reporter.get_session_id()
        # Assume update-notifier-common stamp file is not present by
        # default.
        self.reporter.update_notifier_stamp = "/Not/Existing"
        self.config.data_path = self.makeDir()
        os.mkdir(self.config.package_directory)
        self.check_stamp_file = self.config.detect_package_changes_stamp

    def _clear_repository(self):
        """Remove all packages from self.repository."""
        create_file(self.repository_dir + "/Packages", "")

    def set_pkg1_upgradable(self):
        """Make it so that package "name1" is considered to be upgradable.

        Return the hash of the package that upgrades "name1".
        """
        self._add_package_to_deb_dir(
            self.repository_dir, "name1", version="version2")
        self.facade.reload_channels()
        name1_upgrade = sorted(self.facade.get_packages_by_name("name1"))[1]
        return self.facade.get_package_hash(name1_upgrade)

    def set_pkg1_installed(self):
        """Make it so that package "name1" is considered installed."""
        self._install_deb_file(os.path.join(self.repository_dir, PKGNAME1))

    def _make_fake_apt_update(self, out="output", err="error", code=0):
        """Create a fake apt-update executable"""
        self.reporter.apt_update_filename = self.makeFile(
            "#!/bin/sh\n"
            "echo -n '%s'\n"
            "echo -n '%s' >&2\n"
            "exit %d" % (out, err, code))
        os.chmod(self.reporter.apt_update_filename, 0755)

    def test_set_package_ids_with_all_known(self):
        self.store.add_hash_id_request(["hash1", "hash2"])
        request2 = self.store.add_hash_id_request(["hash3", "hash4"])
        self.store.add_hash_id_request(["hash5", "hash6"])

        self.store.add_task("reporter",
                            {"type": "package-ids", "ids": [123, 456],
                             "request-id": request2.id})

        def got_result(result):
            self.assertEqual(self.store.get_hash_id("hash1"), None)
            self.assertEqual(self.store.get_hash_id("hash2"), None)
            self.assertEqual(self.store.get_hash_id("hash3"), 123)
            self.assertEqual(self.store.get_hash_id("hash4"), 456)
            self.assertEqual(self.store.get_hash_id("hash5"), None)
            self.assertEqual(self.store.get_hash_id("hash6"), None)

        deferred = self.reporter.handle_tasks()
        return deferred.addCallback(got_result)

    def test_set_package_ids_with_unknown_request_id(self):

        self.store.add_task("reporter",
                            {"type": "package-ids", "ids": [123, 456],
                             "request-id": 123})

        # Nothing bad should happen.
        return self.reporter.handle_tasks()

    def test_set_package_ids_with_unknown_hashes(self):
        message_store = self.broker_service.message_store

        message_store.set_accepted_types(["add-packages"])

        request1 = self.store.add_hash_id_request(["foo", HASH1, "bar"])

        self.store.add_task("reporter",
                            {"type": "package-ids",
                             "ids": [123, None, 456],
                             "request-id": request1.id})

        def got_result(result):
            message = message_store.get_pending_messages()[0]

            # The server will answer the "add-packages" message with a
            # "package-ids" message, so we must keep track of the hashes
            # for packages sent.
            request2 = self.store.get_hash_id_request(message["request-id"])
            self.assertEqual(request2.hashes, [HASH1])

            # Keeping track of the message id for the message with the
            # package data allows us to tell if we should consider our
            # request as lost for some reason, and thus re-request it.
            message_id = request2.message_id
            self.assertEqual(type(message_id), int)

            self.assertTrue(message_store.is_pending(message_id))

            self.assertMessages(
                message_store.get_pending_messages(),
                [{"packages": [{"description": u"Description1",
                                "installed-size": 28672,
                                "name": u"name1",
                                "relations":
                                    [(131074, u"providesname1"),
                                     (196610, u"name1 = version1-release1"),
                                     (262148,
                                      u"prerequirename1 = prerequireversion1"),
                                     (262148,
                                      u"requirename1 = requireversion1"),
                                     (393224, u"name1 < version1-release1"),
                                     (458768,
                                      u"conflictsname1 = conflictsversion1")],
                                "section": u"Group1",
                                "size": 1038,
                                "summary": u"Summary1",
                                "type": 65537,
                                "version": u"version1-release1"}],
                  "request-id": request2.id,
                  "type": "add-packages"}])

        deferred = self.reporter.handle_tasks()
        return deferred.addCallback(got_result)

    def test_set_package_ids_with_unknown_hashes_and_size_none(self):
        message_store = self.broker_service.message_store

        message_store.set_accepted_types(["add-packages"])

        request1 = self.store.add_hash_id_request(["foo", HASH1, "bar"])

        self.store.add_task("reporter",
                            {"type": "package-ids",
                             "ids": [123, None, 456],
                             "request-id": request1.id})

        def got_result(result):
            message = message_store.get_pending_messages()[0]
            request2 = self.store.get_hash_id_request(message["request-id"])
            self.assertMessages(
                message_store.get_pending_messages(),
                [{"packages": [{"description": u"Description1",
                                "installed-size": None,
                                "name": u"name1",
                                "relations": [],
                                "section": u"Group1",
                                "size": None,
                                "summary": u"Summary1",
                                "type": 65537,
                                "version": u"version1-release1"}],
                  "request-id": request2.id,
                  "type": "add-packages"}])

        class FakePackage(object):
            type = 65537
            name = u"name1"
            version = u"version1-release1"
            section = u"Group1"
            summary = u"Summary1"
            description = u"Description1"
            size = None
            installed_size = None
            relations = []

        mock_facade = self.mocker.patch(self.Facade)
        mock_facade.get_package_skeleton(ANY)
        self.mocker.result(FakePackage())
        self.mocker.replay()
        deferred = self.reporter.handle_tasks()
        return deferred.addCallback(got_result)

    def test_set_package_ids_with_unknown_hashes_and_failed_send_msg(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["add-packages"])

        class Boom(Exception):
            pass

        deferred = Deferred()
        deferred.errback(Boom())

        remote_mock = self.mocker.patch(self.reporter._broker)
        remote_mock.send_message(ANY, ANY, True)
        self.mocker.result(deferred)
        self.mocker.replay()

        request_id = self.store.add_hash_id_request(["foo", HASH1, "bar"]).id

        self.store.add_task("reporter", {"type": "package-ids",
                                         "ids": [123, None, 456],
                                         "request-id": request_id})

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(), [])
            self.assertEqual(
                [request.id for request in self.store.iter_hash_id_requests()],
                [request_id])

        result = self.reporter.handle_tasks()
        self.assertFailure(result, Boom)
        return result.addCallback(got_result)

    def test_set_package_ids_removes_request_id_when_done(self):
        request = self.store.add_hash_id_request(["hash1"])
        self.store.add_task("reporter", {"type": "package-ids", "ids": [123],
                                         "request-id": request.id})

        def got_result(result):
            self.assertRaises(UnknownHashIDRequest,
                              self.store.get_hash_id_request, request.id)

        deferred = self.reporter.handle_tasks()
        return deferred.addCallback(got_result)

    def test_fetch_hash_id_db(self):

        # Assume package_hash_id_url is set
        self.config.data_path = self.makeDir()
        self.config.package_hash_id_url = "http://fake.url/path/"
        os.makedirs(os.path.join(self.config.data_path, "package", "hash-id"))
        hash_id_db_filename = os.path.join(self.config.data_path, "package",
                                           "hash-id", "uuid_codename_arch")

        # Fake uuid, codename and arch
        message_store = self.broker_service.message_store
        message_store.set_server_uuid("uuid")
        self.reporter.lsb_release_filename = self.makeFile(SAMPLE_LSB_RELEASE)
        self.facade.set_arch("arch")

        # Let's say fetch_async is successful
        hash_id_db_url = self.config.package_hash_id_url + "uuid_codename_arch"
        fetch_async_mock = self.mocker.replace("landscape.lib."
                                               "fetch.fetch_async")
        fetch_async_mock(hash_id_db_url, cainfo=None)
        fetch_async_result = Deferred()
        fetch_async_result.callback("hash-ids")
        self.mocker.result(fetch_async_result)

        # The download should be properly logged
        logging_mock = self.mocker.replace("logging.info")
        logging_mock("Downloaded hash=>id database from %s" % hash_id_db_url)
        self.mocker.result(None)

        # We don't have our hash=>id database yet
        self.assertFalse(os.path.exists(hash_id_db_filename))

        # Now go!
        self.mocker.replay()
        result = self.reporter.fetch_hash_id_db()

        # Check the database
        def callback(ignored):
            self.assertTrue(os.path.exists(hash_id_db_filename))
            self.assertEqual(open(hash_id_db_filename).read(), "hash-ids")
        result.addCallback(callback)

        return result

    def test_fetch_hash_id_db_does_not_download_twice(self):

        # Let's say that the hash=>id database is already there
        self.config.package_hash_id_url = "http://fake.url/path/"
        self.config.data_path = self.makeDir()
        os.makedirs(os.path.join(self.config.data_path, "package", "hash-id"))
        hash_id_db_filename = os.path.join(self.config.data_path, "package",
                                           "hash-id", "uuid_codename_arch")
        open(hash_id_db_filename, "w").write("test")

        # Fake uuid, codename and arch
        message_store = self.broker_service.message_store
        message_store.set_server_uuid("uuid")
        self.reporter.lsb_release_filename = self.makeFile(SAMPLE_LSB_RELEASE)
        self.facade.set_arch("arch")

        # Intercept any call to fetch_async
        fetch_async_mock = self.mocker.replace("landscape.lib."
                                               "fetch.fetch_async")
        fetch_async_mock(ANY)

        # Go!
        self.mocker.replay()
        result = self.reporter.fetch_hash_id_db()

        def callback(ignored):
            # Check that fetch_async hasn't been called
            self.assertRaises(AssertionError, self.mocker.verify)
            fetch_async(None)

            # The hash=>id database is still there
            self.assertEqual(open(hash_id_db_filename).read(), "test")

        result.addCallback(callback)

        return result

    def test_fetch_hash_id_db_undetermined_server_uuid(self):
        """
        If the server-uuid can't be determined for some reason, no download
        should be attempted and the failure should be properly logged.
        """
        message_store = self.broker_service.message_store
        message_store.set_server_uuid(None)

        logging_mock = self.mocker.replace("logging.warning")
        logging_mock("Couldn't determine which hash=>id database to use: "
                     "server UUID not available")
        self.mocker.result(None)
        self.mocker.replay()

        result = self.reporter.fetch_hash_id_db()
        return result

    def test_fetch_hash_id_db_undetermined_codename(self):

        # Fake uuid
        message_store = self.broker_service.message_store
        message_store.set_server_uuid("uuid")

        # Undetermined codename
        self.reporter.lsb_release_filename = self.makeFile("Foo=bar")

        # The failure should be properly logged
        logging_mock = self.mocker.replace("logging.warning")
        logging_mock("Couldn't determine which hash=>id database to use: "
                     "missing code-name key in %s" %
                     self.reporter.lsb_release_filename)
        self.mocker.result(None)

        # Go!
        self.mocker.replay()
        result = self.reporter.fetch_hash_id_db()

        return result

    def test_fetch_hash_id_db_undetermined_arch(self):

        # Fake uuid and codename
        message_store = self.broker_service.message_store
        message_store.set_server_uuid("uuid")
        self.reporter.lsb_release_filename = self.makeFile(SAMPLE_LSB_RELEASE)

        # Undetermined arch
        self.facade.set_arch("")

        # The failure should be properly logged
        logging_mock = self.mocker.replace("logging.warning")
        logging_mock("Couldn't determine which hash=>id database to use: "
                     "unknown dpkg architecture")
        self.mocker.result(None)

        # Go!
        self.mocker.replay()
        result = self.reporter.fetch_hash_id_db()

        return result

    def test_fetch_hash_id_db_with_default_url(self):

        # Let's say package_hash_id_url is not set but url is
        self.config.data_path = self.makeDir()
        self.config.package_hash_id_url = None
        self.config.url = "http://fake.url/path/message-system/"
        os.makedirs(os.path.join(self.config.data_path, "package", "hash-id"))
        hash_id_db_filename = os.path.join(self.config.data_path, "package",
                                           "hash-id", "uuid_codename_arch")

        # Fake uuid, codename and arch
        message_store = self.broker_service.message_store
        message_store.set_server_uuid("uuid")
        self.reporter.lsb_release_filename = self.makeFile(SAMPLE_LSB_RELEASE)
        self.facade.set_arch("arch")

        # Check fetch_async is called with the default url
        hash_id_db_url = "http://fake.url/path/hash-id-databases/" \
                         "uuid_codename_arch"
        fetch_async_mock = self.mocker.replace("landscape.lib."
                                               "fetch.fetch_async")
        fetch_async_mock(hash_id_db_url, cainfo=None)
        fetch_async_result = Deferred()
        fetch_async_result.callback("hash-ids")
        self.mocker.result(fetch_async_result)

        # Now go!
        self.mocker.replay()
        result = self.reporter.fetch_hash_id_db()

        # Check the database
        def callback(ignored):
            self.assertTrue(os.path.exists(hash_id_db_filename))
            self.assertEqual(open(hash_id_db_filename).read(), "hash-ids")
        result.addCallback(callback)
        return result

    def test_fetch_hash_id_db_with_download_error(self):

        # Assume package_hash_id_url is set
        self.config.data_path = self.makeDir()
        self.config.package_hash_id_url = "http://fake.url/path/"

        # Fake uuid, codename and arch
        message_store = self.broker_service.message_store
        message_store.set_server_uuid("uuid")
        self.reporter.lsb_release_filename = self.makeFile(SAMPLE_LSB_RELEASE)
        self.facade.set_arch("arch")

        # Let's say fetch_async fails
        hash_id_db_url = self.config.package_hash_id_url + "uuid_codename_arch"
        fetch_async_mock = self.mocker.replace("landscape.lib."
                                               "fetch.fetch_async")
        fetch_async_mock(hash_id_db_url, cainfo=None)
        fetch_async_result = Deferred()
        fetch_async_result.errback(FetchError("fetch error"))
        self.mocker.result(fetch_async_result)

        # The failure should be properly logged
        logging_mock = self.mocker.replace("logging.warning")
        logging_mock("Couldn't download hash=>id database: fetch error")
        self.mocker.result(None)

        # Now go!
        self.mocker.replay()
        result = self.reporter.fetch_hash_id_db()

        # We shouldn't have any hash=>id database
        def callback(ignored):
            hash_id_db_filename = os.path.join(
                self.config.data_path, "package", "hash-id",
                "uuid_codename_arch")
            self.assertEqual(os.path.exists(hash_id_db_filename), False)
        result.addCallback(callback)

        return result

    def test_fetch_hash_id_db_with_undetermined_url(self):

        # We don't know where to fetch the hash=>id database from
        self.config.url = None
        self.config.package_hash_id_url = None

        # Fake uuid, codename and arch
        message_store = self.broker_service.message_store
        message_store.set_server_uuid("uuid")
        self.reporter.lsb_release_filename = self.makeFile(SAMPLE_LSB_RELEASE)
        self.facade.set_arch("arch")

        # The failure should be properly logged
        logging_mock = self.mocker.replace("logging.warning")
        logging_mock("Can't determine the hash=>id database url")
        self.mocker.result(None)

        # Let's go
        self.mocker.replay()

        result = self.reporter.fetch_hash_id_db()

        # We shouldn't have any hash=>id database
        def callback(ignored):
            hash_id_db_filename = os.path.join(
                self.config.data_path, "package", "hash-id",
                "uuid_codename_arch")
            self.assertEqual(os.path.exists(hash_id_db_filename), False)
        result.addCallback(callback)

        return result

    def test_fetch_hash_id_db_with_custom_certificate(self):
        """
        The L{PackageReporter.fetch_hash_id_db} method takes into account the
        possible custom SSL certificate specified in the client configuration.
        """

        self.config.url = "http://fake.url/path/message-system/"
        self.config.ssl_public_key = "/some/key"

        # Fake uuid, codename and arch
        message_store = self.broker_service.message_store
        message_store.set_server_uuid("uuid")
        self.reporter.lsb_release_filename = self.makeFile(SAMPLE_LSB_RELEASE)
        self.facade.set_arch("arch")

        # Check fetch_async is called with the default url
        hash_id_db_url = "http://fake.url/path/hash-id-databases/" \
                         "uuid_codename_arch"
        fetch_async_mock = self.mocker.replace("landscape.lib."
                                               "fetch.fetch_async")
        fetch_async_mock(hash_id_db_url, cainfo=self.config.ssl_public_key)
        fetch_async_result = Deferred()
        fetch_async_result.callback("hash-ids")
        self.mocker.result(fetch_async_result)

        # Now go!
        self.mocker.replay()
        result = self.reporter.fetch_hash_id_db()

        return result

    def test_wb_apt_sources_have_changed(self):
        """
        The L{PackageReporter._apt_sources_have_changed} method returns a bool
        indicating if the APT sources list file has changed recently.
        """
        self.reporter.sources_list_filename = "/I/Dont/Exist"
        self.reporter.sources_list_directory = "/I/Dont/Exist/At/All"
        self.assertFalse(self.reporter._apt_sources_have_changed())
        self.reporter.sources_list_filename = self.makeFile("foo")
        self.assertTrue(self.reporter._apt_sources_have_changed())
        os.utime(self.reporter.sources_list_filename, (-1, time.time() - 1799))
        self.assertTrue(self.reporter._apt_sources_have_changed())
        os.utime(self.reporter.sources_list_filename, (-1, time.time() - 1800))
        self.assertFalse(self.reporter._apt_sources_have_changed())

    def test_wb_apt_sources_have_changed_with_directory(self):
        """
        The L{PackageReporter._apt_sources_have_changed} checks also for
        possible additional sources files under /etc/apt/sources.d.
        """
        self.reporter.sources_list_filename = "/I/Dont/Exist/At/All"
        self.reporter.sources_list_directory = self.makeDir()
        self.makeFile(dirname=self.reporter.sources_list_directory,
                      content="deb http://foo ./")
        self.assertTrue(self.reporter._apt_sources_have_changed())

    def test_remove_expired_hash_id_request(self):
        request = self.store.add_hash_id_request(["hash1"])
        request.message_id = 9999

        request.timestamp -= HASH_ID_REQUEST_TIMEOUT

        def got_result(result):
            self.assertRaises(UnknownHashIDRequest,
                              self.store.get_hash_id_request, request.id)

        result = self.reporter.remove_expired_hash_id_requests()
        return result.addCallback(got_result)

    def test_remove_expired_hash_id_request_wont_remove_before_timeout(self):
        request1 = self.store.add_hash_id_request(["hash1"])
        request1.message_id = 9999
        request1.timestamp -= HASH_ID_REQUEST_TIMEOUT / 2

        initial_timestamp = request1.timestamp

        def got_result(result):
            request2 = self.store.get_hash_id_request(request1.id)
            self.assertTrue(request2)

            # Shouldn't update timestamp when already delivered.
            self.assertEqual(request2.timestamp, initial_timestamp)

        result = self.reporter.remove_expired_hash_id_requests()
        return result.addCallback(got_result)

    def test_remove_expired_hash_id_request_updates_timestamps(self):
        request = self.store.add_hash_id_request(["hash1"])
        message_store = self.broker_service.message_store
        message_id = message_store.add({"type": "add-packages",
                                        "packages": [],
                                        "request-id": request.id})
        request.message_id = message_id
        initial_timestamp = request.timestamp

        def got_result(result):
            self.assertTrue(request.timestamp > initial_timestamp)

        result = self.reporter.remove_expired_hash_id_requests()
        return result.addCallback(got_result)

    def test_remove_expired_hash_id_request_removes_when_no_message_id(self):
        request = self.store.add_hash_id_request(["hash1"])

        def got_result(result):
            self.assertRaises(UnknownHashIDRequest,
                              self.store.get_hash_id_request, request.id)

        result = self.reporter.remove_expired_hash_id_requests()
        return result.addCallback(got_result)

    def test_request_unknown_hashes(self):
        self.store.set_hash_ids({HASH2: 123})

        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["unknown-package-hashes"])

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(),
                                [{"hashes": EqualsHashes(HASH1, HASH3),
                                  "request-id": 1,
                                  "type": "unknown-package-hashes"}])

            message = message_store.get_pending_messages()[0]

            request = self.store.get_hash_id_request(1)
            self.assertEqual(request.hashes, message["hashes"])

            self.assertTrue(message_store.is_pending(request.message_id))

        result = self.reporter.request_unknown_hashes()
        return result.addCallback(got_result)

    def test_request_unknown_hashes_limits_number_of_packages(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["unknown-package-hashes"])

        self.addCleanup(setattr, reporter, "MAX_UNKNOWN_HASHES_PER_REQUEST",
                        reporter.MAX_UNKNOWN_HASHES_PER_REQUEST)

        reporter.MAX_UNKNOWN_HASHES_PER_REQUEST = 2

        def got_result1(result):
            # The first message sent should send any 2 of the 3 hashes.
            self.assertEqual(len(message_store.get_pending_messages()), 1)
            message = message_store.get_pending_messages()[-1]
            self.assertEqual(len(message["hashes"]), 2)

            result2 = self.reporter.request_unknown_hashes()
            result2.addCallback(got_result2, message["hashes"])

            return result2

        def got_result2(result, hashes):
            # The second message sent should send the missing hash.
            self.assertEqual(len(message_store.get_pending_messages()), 2)
            message = message_store.get_pending_messages()[-1]
            self.assertEqual(len(message["hashes"]), 1)
            self.assertNotIn(message["hashes"][0], hashes)

        result1 = self.reporter.request_unknown_hashes()
        result1.addCallback(got_result1)

        return result1

    def test_request_unknown_hashes_with_previously_requested(self):
        """
        In this test we'll pretend that a couple of hashes were
        previously requested, and there's one new hash to be requested.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["unknown-package-hashes"])

        self.store.add_hash_id_request([HASH1, HASH3])

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(),
                                [{"hashes": [HASH2],
                                  "request-id": 2,
                                  "type": "unknown-package-hashes"}])

            message = message_store.get_pending_messages()[0]

            request = self.store.get_hash_id_request(2)
            self.assertEqual(request.hashes, message["hashes"])

            self.assertTrue(message_store.is_pending(request.message_id))

        result = self.reporter.request_unknown_hashes()
        return result.addCallback(got_result)

    def test_request_unknown_hashes_with_all_previously_requested(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["unknown-package-hashes"])

        self.store.add_hash_id_request([HASH1, HASH2, HASH3])

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(), [])

        result = self.reporter.request_unknown_hashes()
        return result.addCallback(got_result)

    def test_request_unknown_hashes_with_failing_send_message(self):
        """
        When broker.send_message() fails, the hash_id_request shouldn't
        even be stored, because we have no message_id.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["unknown-package-hashes"])

        class Boom(Exception):
            pass

        deferred = Deferred()
        deferred.errback(Boom())

        remote_mock = self.mocker.patch(self.reporter._broker)
        remote_mock.send_message(ANY, ANY, True)
        self.mocker.result(deferred)
        self.mocker.replay()

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(), [])
            self.assertEqual(list(self.store.iter_hash_id_requests()), [])

        result = self.reporter.request_unknown_hashes()

        self.assertFailure(result, Boom)

        return result.addCallback(got_result)

    def test_detect_packages_creates_stamp_file(self):
        """
        When the detect_packages method computes package changes, it creates
        a stamp file after sending the message.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})

        self.assertFalse(os.path.exists(self.check_stamp_file))

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(),
                                [{"type": "packages", "available": [(1, 3)]}])
            self.assertTrue(os.path.exists(self.check_stamp_file))

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    def test_detect_packages_changes_with_available(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(),
                                [{"type": "packages", "available": [(1, 3)]}])

            self.assertEqual(sorted(self.store.get_available()), [1, 2, 3])

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    def test_detect_packages_changes_with_available_and_unknown_hash(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        self.store.set_hash_ids({HASH1: 1, HASH3: 3})

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(),
                                [{"type": "packages", "available": [1, 3]}])

            self.assertEqual(sorted(self.store.get_available()), [1, 3])

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    def test_detect_packages_changes_with_available_and_previously_known(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})
        self.store.add_available([1, 3])

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(),
                                [{"type": "packages", "available": [2]}])

            self.assertEqual(sorted(self.store.get_available()), [1, 2, 3])

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    def test_detect_packages_changes_with_not_available(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        self._clear_repository()

        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})
        self.store.add_available([1, 2, 3])

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(),
                                [{"type": "packages",
                                  "not-available": [(1, 3)]}])

            self.assertEqual(self.store.get_available(), [])

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    def test_detect_packages_changes_with_installed(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})
        self.store.add_available([1, 2, 3])

        self.set_pkg1_installed()

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(),
                                [{"type": "packages", "installed": [1]}])

            self.assertEqual(self.store.get_installed(), [1])

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    def test_detect_packages_changes_with_installed_already_known(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})
        self.store.add_available([1, 2, 3])
        self.store.add_installed([1])

        self.set_pkg1_installed()

        def got_result(result):
            self.assertFalse(result)
            self.assertMessages(message_store.get_pending_messages(), [])

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    def test_detect_packages_changes_with_not_installed(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})
        self.store.add_available([1, 2, 3])
        self.store.add_installed([1])

        def got_result(result):
            self.assertTrue(result)
            self.assertMessages(message_store.get_pending_messages(),
                                [{"type": "packages", "not-installed": [1]}])

            self.assertEqual(self.store.get_installed(), [])

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    def test_detect_packages_changes_with_upgrade_but_not_installed(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        upgrade_hash = self.set_pkg1_upgradable()
        self.store.set_hash_ids({HASH1: 1, upgrade_hash: 2, HASH3: 3})
        self.store.add_available([1, 2, 3])

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(), [])

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    def test_detect_packages_changes_with_upgrade(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        upgrade_hash = self.set_pkg1_upgradable()
        self.set_pkg1_installed()
        self.facade.reload_channels()

        self.store.set_hash_ids(
            {HASH1: 1, upgrade_hash: 2, HASH3: 3})
        self.store.add_available([1, 2, 3])
        self.store.add_installed([1])

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(),
                                [{"type": "packages",
                                  "available-upgrades": [2]}])

            self.assertEqual(self.store.get_available_upgrades(), [2])

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    def test_detect_packages_changes_with_not_upgrade(self):
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})
        self.store.add_available([1, 2, 3])
        self.store.add_available_upgrades([2])

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(),
                                [{"type": "packages",
                                  "not-available-upgrades": [2]}])

            self.assertEqual(self.store.get_available_upgrades(), [])

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    @inlineCallbacks
    def test_detect_packages_after_tasks(self):
        """
        When the L{PackageReporter} got a task to handle, it forces itself to
        detect package changes, not checking the local state of package.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})

        touch_file(self.check_stamp_file)

        self.store.add_task("reporter",
                            {"type": "package-ids", "ids": [123, 456],
                             "request-id": 123})

        yield self.reporter.handle_tasks()

        yield self.reporter.detect_packages_changes()

        # We check that detect changes run by looking at messages
        self.assertMessages(message_store.get_pending_messages(),
                            [{"type": "packages", "available": [(1, 3)]}])

    def test_detect_packages_changes_with_not_locked_and_ranges(self):
        """
        Ranges are used when reporting changes to 3 or more not locked packages
        having consecutive ids.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["packages"])

        self.store.add_locked([1, 2, 3])

        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})
        self.store.add_available([1, 2, 3])

        def got_result(result):
            self.assertMessages(message_store.get_pending_messages(),
                                [{"type": "packages", "not-locked": [(1, 3)]}])
            self.assertEqual(sorted(self.store.get_locked()), [])

        result = self.reporter.detect_packages_changes()
        return result.addCallback(got_result)

    def test_detect_changes_considers_packages_changes(self):
        """
        The L{PackageReporter.detect_changes} method package changes.
        """
        reporter_mock = self.mocker.patch(self.reporter)
        reporter_mock.detect_packages_changes()
        self.mocker.result(succeed(True))

        self.mocker.replay()
        return self.reporter.detect_changes()

    def test_detect_changes_fires_package_data_changed(self):
        """
        The L{PackageReporter.detect_changes} method fires an event of
        type 'package-data-changed' if we detected something has changed
        with respect to our previous run.
        """
        reporter_mock = self.mocker.patch(self.reporter)
        reporter_mock.detect_packages_changes()
        self.mocker.result(succeed(True))
        callback = self.mocker.mock()
        callback()
        self.mocker.replay()

        self.broker_service.reactor.call_on("package-data-changed", callback)
        return self.reporter.detect_changes()

    def test_run(self):
        reporter_mock = self.mocker.patch(self.reporter)

        self.mocker.order()

        results = [Deferred() for i in range(7)]

        reporter_mock.run_apt_update()
        self.mocker.result(results[0])

        reporter_mock.fetch_hash_id_db()
        self.mocker.result(results[1])

        reporter_mock.use_hash_id_db()
        self.mocker.result(results[2])

        reporter_mock.handle_tasks()
        self.mocker.result(results[3])

        reporter_mock.remove_expired_hash_id_requests()
        self.mocker.result(results[4])

        reporter_mock.request_unknown_hashes()
        self.mocker.result(results[5])

        reporter_mock.detect_changes()
        self.mocker.result(results[6])

        self.mocker.replay()

        self.reporter.run()

        # It must raise an error because deferreds weren't yet fired.
        self.assertRaises(AssertionError, self.mocker.verify)

        # Call them in reversed order. It must not make a difference because
        # Twisted is ensuring that things run in the proper order.
        for deferred in reversed(results):
            deferred.callback(None)

    def test_main(self):
        run_task_handler = self.mocker.replace("landscape.package.taskhandler"
                                               ".run_task_handler",
                                               passthrough=False)
        run_task_handler(PackageReporter, ["ARGS"])
        self.mocker.result("RESULT")
        self.mocker.replay()

        self.assertEqual(main(["ARGS"]), "RESULT")

    def test_find_reporter_command(self):
        dirname = self.makeDir()
        filename = self.makeFile("", dirname=dirname,
                                 basename="landscape-package-reporter")

        saved_argv = sys.argv
        try:
            sys.argv = [os.path.join(dirname, "landscape-monitor")]

            command = find_reporter_command()

            self.assertEqual(command, filename)
        finally:
            sys.argv = saved_argv

    def test_resynchronize(self):
        """
        When a resynchronize task arrives, the reporter should clear
        out all the data in the package store, except the hash ids and
        the hash ids requests.
        This is done in the reporter so that we know it happens when
        no other reporter is possibly running at the same time.
        """
        self._add_system_package("foo")
        self.facade.reload_channels()
        [foo] = self.facade.get_packages_by_name("foo")
        foo_hash = self.facade.get_package_hash(foo)
        self.facade.set_package_hold(foo)
        self.facade.reload_channels()
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-locks"])
        self.store.set_hash_ids({foo_hash: 3, HASH2: 4})
        self.store.add_available([1])
        self.store.add_available_upgrades([2])
        self.store.add_installed([2])
        self.store.add_locked([3])
        request1 = self.store.add_hash_id_request(["hash3"])
        request2 = self.store.add_hash_id_request(["hash4"])

        # Set the message id to avoid the requests being deleted by the
        # L{PackageReporter.remove_expired_hash_id_requests} method.
        request1.message_id = 1
        request2.message_id = 2

        # Let's make sure the data is there.
        self.assertEqual(self.store.get_available_upgrades(), [2])
        self.assertEqual(self.store.get_available(), [1])
        self.assertEqual(self.store.get_installed(), [2])
        self.assertEqual(self.store.get_hash_id_request(request1.id).id,
                         request1.id)

        self.store.add_task("reporter", {"type": "resynchronize"})

        deferred = self.reporter.run()

        def check_result(result):

            # The hashes should not go away.
            hash1 = self.store.get_hash_id(foo_hash)
            hash2 = self.store.get_hash_id(HASH2)
            self.assertEqual([hash1, hash2], [3, 4])

            # But the other data should.
            self.assertEqual(self.store.get_available_upgrades(), [])

            # After running the resychronize task, detect_packages_changes is
            # called, and the existing known hashes are made available.
            self.assertEqual(self.store.get_available(), [4])
            self.assertEqual(self.store.get_installed(), [3])
            self.assertEqual(self.store.get_locked(), [3])

            # The two original hash id requests should be still there, and
            # a new hash id request should also be detected for HASH3.
            requests_count = 0
            new_request_found = False
            for request in self.store.iter_hash_id_requests():
                requests_count += 1
                if request.id == request1.id:
                    self.assertEqual(request.hashes, ["hash3"])
                elif request.id == request2.id:
                    self.assertEqual(request.hashes, ["hash4"])
                elif not new_request_found:
                    self.assertEqual(request.hashes, [HASH3, HASH1])
                else:
                    self.fail("Unexpected hash-id request!")
            self.assertEqual(requests_count, 3)

        deferred.addCallback(check_result)
        return deferred

    def test_run_apt_update(self):
        """
        The L{PackageReporter.run_apt_update} method should run apt-update.
        """
        self.reporter.sources_list_filename = "/I/Dont/Exist"
        self.reporter.sources_list_directory = "/I/Dont/Exist"
        self._make_fake_apt_update()
        debug_mock = self.mocker.replace("logging.debug")
        debug_mock("'%s' exited with status 0 (out='output', err='error')" %
                   self.reporter.apt_update_filename)
        warning_mock = self.mocker.replace("logging.warning")
        self.expect(warning_mock(ANY)).count(0)
        self.mocker.replay()
        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback((out, err, code)):
                self.assertEqual("output", out)
                self.assertEqual("error", err)
                self.assertEqual(0, code)
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_with_force_apt_update(self):
        """
        L{PackageReporter.run_apt_update} forces an apt-update run if the
        '--force-apt-update' command line option was passed.
        """
        self.makeFile("", path=self.config.update_stamp_filename)
        self.config.load(["--force-apt-update"])
        self._make_fake_apt_update()

        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback((out, err, code)):
                self.assertEqual("output", out)
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_with_force_apt_update_if_sources_changed(self):
        """
        L{PackageReporter.run_apt_update} forces an apt-update run if the APT
        sources.list file has changed.

        """
        self.assertEqual(self.reporter.sources_list_filename,
                         "/etc/apt/sources.list")
        self.reporter.sources_list_filename = self.makeFile("deb ftp://url ./")
        self._make_fake_apt_update()

        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback((out, err, code)):
                self.assertEqual("output", out)
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_warns_about_failures(self):
        """
        The L{PackageReporter.run_apt_update} method should log a warning in
        case apt-update terminates with a non-zero exit code.
        """
        self._make_fake_apt_update(code=2)
        logging_mock = self.mocker.replace("logging.warning")
        logging_mock("'%s' exited with status 2"
                     " (error)" % self.reporter.apt_update_filename)
        self.mocker.replay()
        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback((out, err, code)):
                self.assertEqual("output", out)
                self.assertEqual("error", err)
                self.assertEqual(2, code)
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_report_apt_failure(self):
        """
        If L{PackageReporter.run_apt_update} fails, a message is sent to the
        server reporting the error, to be able to fix the problem centrally.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-reporter-result"])
        self._make_fake_apt_update(code=2)
        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback(ignore):
                self.assertMessages(
                    message_store.get_pending_messages(),
                    [{"type": "package-reporter-result",
                      "code": 2, "err": u"error"}])
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_report_no_sources(self):
        """
        L{PackageReporter.run_apt_update} reports a failure if apt succeeds but
        there are no APT sources defined. APT doesn't fail if there are no
        sources, but we fake a failure in order to re-use the
        PackageReporterAlert on the server.
        """
        self.facade.reset_channels()
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-reporter-result"])
        self._make_fake_apt_update()
        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback(ignore):
                error = "There are no APT sources configured in %s or %s." % (
                    self.reporter.sources_list_filename,
                    self.reporter.sources_list_directory)
                self.assertMessages(
                    message_store.get_pending_messages(),
                    [{"type": "package-reporter-result",
                      "code": 1, "err": error}])
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_report_apt_failure_no_sources(self):
        """
        If L{PackageReporter.run_apt_update} fails and there are no
        APT sources configured, the APT error takes precedence.
        """
        self.facade.reset_channels()
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-reporter-result"])
        self._make_fake_apt_update(code=2)
        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback(ignore):
                self.assertMessages(
                    message_store.get_pending_messages(),
                    [{"type": "package-reporter-result",
                      "code": 2, "err": u"error"}])
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_report_success(self):
        """
        L{PackageReporter.run_apt_update} also reports success to be able to
        know the proper state of the client.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-reporter-result"])
        self._make_fake_apt_update(err="message")
        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback(ignore):
                self.assertMessages(
                    message_store.get_pending_messages(),
                    [{"type": "package-reporter-result",
                      "code": 0, "err": u"message"}])
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_no_run_in_interval(self):
        """
        The L{PackageReporter.run_apt_update} logs a debug message if
        apt-update doesn't run because interval has not passed.
        """
        self.reporter._apt_sources_have_changed = lambda: False
        self.makeFile("", path=self.config.update_stamp_filename)

        logging_mock = self.mocker.replace("logging.debug")
        logging_mock("'%s' didn't run, update interval has not passed" %
                     self.reporter.apt_update_filename)
        self.mocker.replay()
        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback((out, err, code)):
                self.assertEqual("", out)
                self.assertEqual("", err)
                self.assertEqual(0, code)
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_no_run_update_notifier_stamp_in_interval(self):
        """
        The L{PackageReporter.run_apt_update} doesn't runs apt-update if the
        interval is passed but the stamp file from update-notifier-common
        reports that 'apt-get update' has been run in the interval.
        """
        self.reporter._apt_sources_have_changed = lambda: False

        # The interval for the apt-update stamp file is expired.
        self.makeFile("", path=self.config.update_stamp_filename)
        expired_time = time.time() - self.config.apt_update_interval - 1
        os.utime(
            self.config.update_stamp_filename, (expired_time, expired_time))
        # The interval for the update-notifier-common stamp file is not
        # expired.
        self.reporter.update_notifier_stamp = self.makeFile("")

        logging_mock = self.mocker.replace("logging.debug")
        logging_mock("'%s' didn't run, update interval has not passed" %
                     self.reporter.apt_update_filename)
        self.mocker.replay()
        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback((out, err, code)):
                self.assertEqual("", out)
                self.assertEqual("", err)
                self.assertEqual(0, code)
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_runs_interval_expired(self):
        """
        L{PackageReporter.run_apt_update} runs if both apt-update and
        update-notifier-common stamp files are present and the time
        interval has passed.
        """
        self.reporter._apt_sources_have_changed = lambda: False

        expired_time = time.time() - self.config.apt_update_interval - 1
        # The interval for both stamp files is expired.
        self.makeFile("", path=self.config.update_stamp_filename)
        os.utime(
            self.config.update_stamp_filename, (expired_time, expired_time))
        self.reporter.update_notifier_stamp = self.makeFile("")
        os.utime(
            self.reporter.update_notifier_stamp, (expired_time, expired_time))

        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-reporter-result"])
        self._make_fake_apt_update(err="message")
        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback(ignore):
                self.assertMessages(
                    message_store.get_pending_messages(),
                    [{"type": "package-reporter-result",
                      "code": 0, "err": u"message"}])
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_touches_stamp_file(self):
        """
        The L{PackageReporter.run_apt_update} method touches a stamp file
        after running the apt-update wrapper.
        """
        self.reporter.sources_list_filename = "/I/Dont/Exist"
        self._make_fake_apt_update()
        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback(ignored):
                self.assertTrue(
                    os.path.exists(self.config.update_stamp_filename))
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_error_on_cache_file(self):
        """
        L{PackageReporter.run_apt_update} succeeds if the command fails because
        one of the cache files is not found. This generally occurs if 'apt-get
        clean' has been concurrently run with 'apt-get update'.  This is not an
        issue for the package lists update.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-reporter-result"])
        self._make_fake_apt_update(
            code=2, out="not important",
            err=("E: Problem renaming the file "
                 "/var/cache/apt/pkgcache.bin.6ZsRSX to "
                 "/var/cache/apt/pkgcache.bin - rename (2: No such file "
                 "or directory)\n"
                 "W: You may want to run apt-get update to correct these "
                 "problems"))
        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback(ignore):
                self.assertMessages(
                    message_store.get_pending_messages(),
                    [{"type": "package-reporter-result",
                      "code": 0, "err": u""}])
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_run_apt_update_error_no_cache_files(self):
        """
        L{PackageReporter.run_apt_update} succeeds if the command fails because
        cache files are not found.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-reporter-result"])
        self._make_fake_apt_update(
            code=2, out="not important",
            err=("E: Problem renaming the file "
                 "/var/cache/apt/srcpkgcache.bin.Pw1Zxy to "
                 "/var/cache/apt/srcpkgcache.bin - rename (2: No such file "
                 "or directory)\n"
                 "E: Problem renaming the file "
                 "/var/cache/apt/pkgcache.bin.wz8ooS to "
                 "/var/cache/apt/pkgcache.bin - rename (2: No such file "
                 "or directory)\n"
                 "E: The package lists or status file could not be parsed "
                 "or opened."))

        deferred = Deferred()

        def do_test():
            result = self.reporter.run_apt_update()

            def callback(ignore):
                self.assertMessages(
                    message_store.get_pending_messages(),
                    [{"type": "package-reporter-result",
                      "code": 0, "err": u""}])
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_config_apt_update_interval(self):
        """
        L{PackageReporter} uses the C{apt_update_interval} configuration
        parameter to check the age of the update stamp file.
        """
        self.config.apt_update_interval = 1234
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-reporter-result"])
        intervals = []

        def apt_update_timeout_expired(interval):
            intervals.append(interval)
            return False

        deferred = Deferred()

        self.reporter._apt_sources_have_changed = lambda: False
        self.reporter._apt_update_timeout_expired = apt_update_timeout_expired

        def do_test():
            result = self.reporter.run_apt_update()

            def callback(ignore):
                self.assertMessages(message_store.get_pending_messages(), [])
                self.assertEqual([1234], intervals)
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred

    def test_detect_packages_doesnt_creates_stamp_files(self):
        """
        Stamp file is created if not present, and the method returns
        that the information changed in that case.
        """
        result = self.reporter._package_state_has_changed()
        self.assertTrue(result)
        self.assertFalse(os.path.exists(self.check_stamp_file))

    def test_detect_packages_changes_returns_false_if_unchanged(self):
        """
        If a monitored file is not changed (touched), the method returns
        False.
        """
        touch_file(self.check_stamp_file, offset_seconds=2)
        result = self.reporter._package_state_has_changed()
        self.assertFalse(result)

    def test_detect_packages_changes_returns_true_if_changed(self):
        """
        If a monitored file is changed (touched), the method returns True.
        """
        status_file = apt_pkg.config.find_file("dir::state::status")

        touch_file(status_file)
        touch_file(self.check_stamp_file)

        touch_file(status_file)
        result = self.reporter._package_state_has_changed()
        self.assertTrue(result)

    def test_detect_packages_changes_works_for_list_files(self):
        """
        If a list file is touched, the method returns True.
        """
        status_file = apt_pkg.config.find_file("dir::state::status")
        touch_file(status_file)
        touch_file(self.check_stamp_file)

        list_dir = apt_pkg.config.find_dir("dir::state::lists")
        # There are no *Packages files in the fixures, let's create one.
        touch_file(os.path.join(list_dir, "testPackages"))

        result = self.reporter._package_state_has_changed()
        self.assertTrue(result)

    def test_detect_packages_changes_detects_removed_list_file(self):
        """
        If a list file is removed from the system, the method returns True.
        """
        list_dir = apt_pkg.config.find_dir("dir::state::lists")
        test_file = os.path.join(list_dir, "testPackages")
        touch_file(test_file)
        touch_file(self.check_stamp_file)

        os.remove(test_file)
        result = self.reporter._package_state_has_changed()
        self.assertTrue(result)


class GlobalPackageReporterAptTest(LandscapeTest):

    helpers = [AptFacadeHelper, SimpleRepositoryHelper, BrokerServiceHelper]

    def setUp(self):
        super(GlobalPackageReporterAptTest, self).setUp()
        self.store = FakePackageStore(self.makeFile())
        self.config = PackageReporterConfiguration()
        self.reporter = FakeGlobalReporter(
            self.store, self.facade, self.remote, self.config)
        # Assume update-notifier-common stamp file is not present by
        # default.
        self.reporter.update_notifier_stamp = "/Not/Existing"
        self.config.data_path = self.makeDir()
        os.mkdir(self.config.package_directory)

    def test_store_messages(self):
        """
        L{FakeGlobalReporter} stores messages which are sent.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-reporter-result"])
        self.reporter.apt_update_filename = self.makeFile(
            "#!/bin/sh\necho -n error >&2\necho -n output\nexit 0")
        os.chmod(self.reporter.apt_update_filename, 0755)
        deferred = Deferred()

        def do_test():
            self.reporter.get_session_id()
            result = self.reporter.run_apt_update()

            def callback(ignore):
                message = {"type": "package-reporter-result",
                           "code": 0, "err": u"error"}
                self.assertMessages(
                    message_store.get_pending_messages(), [message])
                stored = list(self.store._db.execute(
                    "SELECT id, data FROM message").fetchall())
                self.assertEqual(1, len(stored))
                self.assertEqual(1, stored[0][0])
                self.assertEqual(message, bpickle.loads(str(stored[0][1])))
            result.addCallback(callback)
            result.chainDeferred(deferred)

        reactor.callWhenRunning(do_test)
        return deferred


class FakePackageReporterTest(LandscapeTest):

    helpers = [EnvironSaverHelper, BrokerServiceHelper]

    def setUp(self):
        super(FakePackageReporterTest, self).setUp()
        self.store = FakePackageStore(self.makeFile())
        global_file = self.makeFile()
        self.global_store = FakePackageStore(global_file)
        os.environ["FAKE_PACKAGE_STORE"] = global_file
        self.config = PackageReporterConfiguration()
        self.reporter = FakeReporter(
            self.store, None, self.remote, self.config)
        self.config.data_path = self.makeDir()
        os.mkdir(self.config.package_directory)

    def test_send_messages(self):
        """
        L{FakeReporter} sends messages stored in the global store specified by
        C{FAKE_PACKAGE_STORE}.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-reporter-result"])
        message = {"type": "package-reporter-result",
                   "code": 0, "err": u"error"}
        self.global_store.save_message(message)

        def check(ignore):
            messages = message_store.get_pending_messages()
            self.assertMessages(
                messages, [message])
            stored = list(self.store._db.execute(
                "SELECT id FROM message").fetchall())
            self.assertEqual(1, len(stored))
            self.assertEqual(1, stored[0][0])

        deferred = self.reporter.run()
        deferred.addCallback(check)
        return deferred

    def test_filter_message_type(self):
        """
        L{FakeReporter} only sends one message of each type per run.
        """
        message_store = self.broker_service.message_store
        message_store.set_accepted_types(["package-reporter-result"])
        message1 = {"type": "package-reporter-result",
                    "code": 0, "err": u"error"}
        self.global_store.save_message(message1)
        message2 = {"type": "package-reporter-result",
                    "code": 1, "err": u"error"}
        self.global_store.save_message(message2)

        def check1(ignore):
            self.assertMessages(
                message_store.get_pending_messages(), [message1])
            stored = list(self.store._db.execute(
                "SELECT id FROM message").fetchall())
            self.assertEqual(1, stored[0][0])
            return self.reporter.run().addCallback(check2)

        def check2(ignore):
            self.assertMessages(
                message_store.get_pending_messages(), [message1, message2])
            stored = list(self.store._db.execute(
                "SELECT id FROM message").fetchall())
            self.assertEqual(2, len(stored))
            self.assertEqual(1, stored[0][0])
            self.assertEqual(2, stored[1][0])

        return self.reporter.run().addCallback(check1)


class EqualsHashes(object):

    def __init__(self, *hashes):
        self._hashes = sorted(hashes)

    def __eq__(self, other):
        return self._hashes == sorted(other)
