#!/usr/bin/env python3
######################################################################
#
# File: test/integration/test_b2_command_line.py
#
# Copyright 2019 Backblaze Inc. All Rights Reserved.
#
# License https://www.backblaze.com/using_b2_code.html
#
######################################################################
from __future__ import annotations

import base64
import contextlib
import hashlib
import itertools
import json
import os
import os.path
import pathlib
import re
import sys
import time
from pathlib import Path
from tempfile import mkdtemp

import pytest
from b2sdk.v2 import (
    B2_ACCOUNT_INFO_ENV_VAR,
    SSE_C_KEY_ID_FILE_INFO_KEY_NAME,
    UNKNOWN_FILE_RETENTION_SETTING,
    EncryptionMode,
    EncryptionSetting,
    FileRetentionSetting,
    LegalHold,
    RetentionMode,
    SqliteAccountInfo,
    fix_windows_path_limit,
)
from b2sdk.v2.exception import MissingAccountData

from b2._internal._cli.const import (
    B2_APPLICATION_KEY_ENV_VAR,
    B2_APPLICATION_KEY_ID_ENV_VAR,
    B2_ENVIRONMENT_ENV_VAR,
)
from b2._internal.console_tool import current_time_millis

from ..helpers import skip_on_windows
from .helpers import (
    ONE_DAY_MILLIS,
    ONE_HOUR_MILLIS,
    SSE_B2_AES,
    SSE_C_AES,
    SSE_C_AES_2,
    SSE_NONE,
    TempDir,
    file_mod_time_millis,
    random_hex,
    read_file,
    set_file_mod_time_millis,
    should_equal,
    write_file,
)


def test_authorize_account_via_params_saving_credentials(
    b2_tool,
    realm,
    application_key,
    application_key_id,
    account_info_file,
):
    """
    When calling `authorize-account` and passing credentials as params,
    we want the credentials to be saved.
    """

    b2_tool.should_succeed(['clear-account'])

    assert B2_APPLICATION_KEY_ID_ENV_VAR not in os.environ
    assert B2_APPLICATION_KEY_ENV_VAR not in os.environ

    b2_tool.should_succeed(
        ['authorize-account', '--environment', realm, application_key_id, application_key]
    )

    assert account_info_file.exists()
    account_info = SqliteAccountInfo()
    assert account_info.get_application_key() == application_key
    assert account_info.get_application_key_id() == application_key_id


def test_authorize_account_via_env_vars_saving_credentials(
    b2_tool,
    realm,
    application_key,
    application_key_id,
    account_info_file,
):
    """
    When calling `authorize-account` and passing credentials
    via env vars, we still want the credentials to be saved.
    """

    b2_tool.should_succeed(['clear-account'])

    assert B2_APPLICATION_KEY_ID_ENV_VAR not in os.environ
    assert B2_APPLICATION_KEY_ENV_VAR not in os.environ

    b2_tool.should_succeed(
        ['authorize-account'],
        additional_env={
            B2_ENVIRONMENT_ENV_VAR: realm,
            B2_APPLICATION_KEY_ID_ENV_VAR: application_key_id,
            B2_APPLICATION_KEY_ENV_VAR: application_key,
        }
    )

    assert account_info_file.exists()
    account_info = SqliteAccountInfo()
    assert account_info.get_application_key() == application_key
    assert account_info.get_application_key_id() == application_key_id


def test_clear_account_with_env_vars(
    b2_tool,
    realm,
    application_key,
    application_key_id,
    account_info_file,
):
    """
    When calling `clear-account` and passing credentials via env vars,
    we want the credentials to be removed from the file.
    """

    assert account_info_file.exists()
    account_info = SqliteAccountInfo()
    assert account_info.get_application_key() == application_key
    assert account_info.get_application_key_id() == application_key_id

    b2_tool.should_succeed(
        ['clear-account'],
        additional_env={
            B2_ENVIRONMENT_ENV_VAR: realm,
            B2_APPLICATION_KEY_ID_ENV_VAR: application_key_id,
            B2_APPLICATION_KEY_ENV_VAR: application_key,
        }
    )

    assert account_info_file.exists()
    account_info = SqliteAccountInfo()
    with pytest.raises(MissingAccountData):
        account_info.get_application_key()
    with pytest.raises(MissingAccountData):
        account_info.get_application_key_id()


@pytest.mark.apiver(to_ver=3)
def test_command_with_env_vars_saving_credentials(
    b2_tool,
    realm,
    application_key,
    application_key_id,
    account_info_file,
    bucket_name,
    b2_uri_args,
):
    """
    When calling any command other then `authorize-account` and passing credentials
    via env vars, we don't want them to be saved.
    """

    b2_tool.should_succeed(['clear-account'])

    assert B2_APPLICATION_KEY_ID_ENV_VAR not in os.environ
    assert B2_APPLICATION_KEY_ENV_VAR not in os.environ

    b2_tool.should_succeed(
        ['ls', '--long', *b2_uri_args(bucket_name)],
        additional_env={
            B2_ENVIRONMENT_ENV_VAR: realm,
            B2_APPLICATION_KEY_ID_ENV_VAR: application_key_id,
            B2_APPLICATION_KEY_ENV_VAR: application_key,
        }
    )

    assert account_info_file.exists()
    account_info = SqliteAccountInfo()
    assert account_info.get_application_key() == application_key
    assert account_info.get_application_key_id() == application_key_id


@pytest.mark.apiver(from_ver=4)
def test_command_with_env_vars_not_saving_credentials(
    b2_tool,
    realm,
    application_key,
    application_key_id,
    account_info_file,
    bucket_name,
    b2_uri_args,
):
    """
    When calling any command other then `authorize-account` and passing credentials
    via env vars, we don't want them to be saved.
    """

    b2_tool.should_succeed(['clear-account'])

    assert B2_APPLICATION_KEY_ID_ENV_VAR not in os.environ
    assert B2_APPLICATION_KEY_ENV_VAR not in os.environ

    b2_tool.should_succeed(
        ['ls', '--long', *b2_uri_args(bucket_name)],
        additional_env={
            B2_ENVIRONMENT_ENV_VAR: realm,
            B2_APPLICATION_KEY_ID_ENV_VAR: application_key_id,
            B2_APPLICATION_KEY_ENV_VAR: application_key,
        }
    )

    assert account_info_file.exists()
    account_info = SqliteAccountInfo()
    with pytest.raises(MissingAccountData):
        account_info.get_application_key()
    with pytest.raises(MissingAccountData):
        account_info.get_application_key_id()


@pytest.mark.apiver(from_ver=4)
def test_command_with_env_vars_reusing_existing_account_info(
    b2_tool,
    realm,
    application_key,
    application_key_id,
    account_info_file,
    bucket_name,
    b2_uri_args,
):
    """
    When calling any command with credentials passed via env vars, and the account
    info file already contains the same credentials, we want to use filesystem for
    storing cache, not the in-memory cache.
    """

    assert B2_APPLICATION_KEY_ID_ENV_VAR not in os.environ
    assert B2_APPLICATION_KEY_ENV_VAR not in os.environ

    assert account_info_file.exists()
    account_info = SqliteAccountInfo()
    assert account_info.get_application_key() == application_key
    assert account_info.get_application_key_id() == application_key_id

    account_info.remove_bucket_name(bucket_name)
    assert account_info.get_bucket_id_or_none_from_bucket_name(bucket_name) is None

    b2_tool.should_succeed(
        ['ls', '--long', *b2_uri_args(bucket_name)],
        additional_env={
            B2_ENVIRONMENT_ENV_VAR: realm,
            B2_APPLICATION_KEY_ID_ENV_VAR: application_key_id,
            B2_APPLICATION_KEY_ENV_VAR: application_key,
        }
    )

    assert account_info_file.exists()
    account_info = SqliteAccountInfo()
    assert account_info.get_bucket_id_or_none_from_bucket_name(bucket_name) is not None


@pytest.fixture
def uploaded_sample_file(b2_tool, bucket_name, sample_filepath):
    return b2_tool.should_succeed_json(
        ['upload-file', '--quiet', bucket_name,
         str(sample_filepath), 'sample_file']
    )


def test_download(b2_tool, bucket_name, sample_filepath, uploaded_sample_file, tmp_path):
    output_a = tmp_path / 'a'
    b2_tool.should_succeed(
        [
            'download-file', '--quiet', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}",
            str(output_a)
        ]
    )
    assert output_a.read_text() == sample_filepath.read_text()

    output_b = tmp_path / 'b'
    b2_tool.should_succeed(
        ['download-file', '--quiet', f"b2id://{uploaded_sample_file['fileId']}",
         str(output_b)]
    )
    assert output_b.read_text() == sample_filepath.read_text()


def test_basic(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args):

    file_mod_time_str = str(file_mod_time_millis(sample_file))

    file_data = read_file(sample_file)
    hex_sha1 = hashlib.sha1(file_data).hexdigest()

    list_of_buckets = b2_tool.should_succeed_json(['list-buckets', '--json'])
    should_equal(
        [bucket_name], [b['bucketName'] for b in list_of_buckets if b['bucketName'] == bucket_name]
    )

    b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, sample_file, 'a'])
    b2_tool.should_succeed(['ls', '--long', '--replication', *b2_uri_args(bucket_name)])
    b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'a'])
    b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'b/1'])
    b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'b/2'])
    b2_tool.should_succeed(
        [
            'upload-file', '--no-progress', '--sha1', hex_sha1, '--info', 'foo=bar=baz', '--info',
            'color=blue', bucket_name, sample_file, 'c'
        ]
    )
    b2_tool.should_fail(
        [
            'upload-file', '--no-progress', '--sha1', hex_sha1, '--info', 'foo-bar', '--info',
            'color=blue', bucket_name, sample_file, 'c'
        ], r'ERROR: Bad file info: foo-bar'
    )
    b2_tool.should_succeed(
        [
            'upload-file', '--no-progress', '--content-type', 'text/plain', bucket_name,
            sample_file, 'd'
        ]
    )

    b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'rm'])
    b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'rm1'])
    # with_wildcard allows us to target a single file. rm will be removed, rm1 will be left alone
    b2_tool.should_succeed(
        ['rm', '--recursive', '--with-wildcard', *b2_uri_args(bucket_name, 'rm')]
    )
    list_of_files = b2_tool.should_succeed_json(
        ['ls', '--json', '--recursive', '--with-wildcard', *b2_uri_args(bucket_name, 'rm*')]
    )
    should_equal(['rm1'], [f['fileName'] for f in list_of_files])
    b2_tool.should_succeed(
        ['rm', '--recursive', '--with-wildcard', *b2_uri_args(bucket_name, 'rm1')]
    )

    b2_tool.should_succeed(['download-file', '--quiet', f'b2://{bucket_name}/b/1', tmp_path / 'a'])

    b2_tool.should_succeed(['hide-file', bucket_name, 'c'])

    list_of_files = b2_tool.should_succeed_json(
        ['ls', '--json', '--recursive', *b2_uri_args(bucket_name)]
    )
    should_equal(['a', 'b/1', 'b/2', 'd'], [f['fileName'] for f in list_of_files])

    list_of_files = b2_tool.should_succeed_json(
        ['ls', '--json', '--recursive', '--versions', *b2_uri_args(bucket_name)]
    )
    should_equal(['a', 'a', 'b/1', 'b/2', 'c', 'c', 'd'], [f['fileName'] for f in list_of_files])
    should_equal(
        ['upload', 'upload', 'upload', 'upload', 'hide', 'upload', 'upload'],
        [f['action'] for f in list_of_files]
    )

    first_a_version = list_of_files[0]

    first_c_version = list_of_files[4]
    second_c_version = list_of_files[5]
    list_of_files = b2_tool.should_succeed_json(
        ['ls', '--json', '--recursive', '--versions', *b2_uri_args(bucket_name, 'c')]
    )
    should_equal([], [f['fileName'] for f in list_of_files])

    b2_tool.should_succeed(['copy-file-by-id', first_a_version['fileId'], bucket_name, 'x'])

    b2_tool.should_succeed(['ls', *b2_uri_args(bucket_name)], '^a{0}b/{0}d{0}'.format(os.linesep))
    # file_id, action, date, time, size(, replication), name
    b2_tool.should_succeed(
        ['ls', '--long', *b2_uri_args(bucket_name)],
        '^4_z.* upload .* {1}  a{0}.* - .* b/{0}4_z.* upload .* {1}  d{0}'.format(
            os.linesep, len(file_data)
        )
    )
    b2_tool.should_succeed(
        ['ls', '--long', '--replication', *b2_uri_args(bucket_name)],
        '^4_z.* upload .* {1}  -  a{0}.* - .*  -  b/{0}4_z.* upload .* {1}  -  d{0}'.format(
            os.linesep, len(file_data)
        )
    )
    b2_tool.should_succeed(
        ['ls', '--versions', *b2_uri_args(bucket_name)],
        f'^a{os.linesep}a{os.linesep}b/{os.linesep}c{os.linesep}c{os.linesep}d{os.linesep}'
    )
    b2_tool.should_succeed(
        ['ls', *b2_uri_args(bucket_name, 'b')], f'^b/1{os.linesep}b/2{os.linesep}'
    )
    b2_tool.should_succeed(
        ['ls', *b2_uri_args(bucket_name, 'b/')], f'^b/1{os.linesep}b/2{os.linesep}'
    )

    file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{second_c_version['fileId']}"])
    expected_info = {
        'color': 'blue',
        'foo': 'bar=baz',
        'src_last_modified_millis': file_mod_time_str
    }
    should_equal(expected_info, file_info['fileInfo'])

    b2_tool.should_succeed(['delete-file-version', 'c', first_c_version['fileId']])
    b2_tool.should_succeed(
        ['ls', *b2_uri_args(bucket_name)], f'^a{os.linesep}b/{os.linesep}c{os.linesep}d{os.linesep}'
    )

    b2_tool.should_succeed(['get-url', f"b2id://{second_c_version['fileId']}"])

    b2_tool.should_succeed(
        ['get-url', f"b2://{bucket_name}/any-file-name"],
        '^https://.*/file/{}/{}\r?$'.format(
            bucket_name,
            'any-file-name',
        ),
    )  # \r? is for Windows, as $ doesn't match \r\n


@pytest.mark.apiver(from_ver=4)
def test_ls_b2id(b2_tool, uploaded_sample_file):
    b2_tool.should_succeed(
        ['ls', f"b2id://{uploaded_sample_file['fileId']}"],
        expected_pattern=f"^{uploaded_sample_file['fileName']}",
    )


@pytest.mark.apiver(from_ver=4)
def test_rm_b2id(b2_tool, bucket_name, uploaded_sample_file):
    # remove the file by id
    b2_tool.should_succeed(['rm', f"b2id://{uploaded_sample_file['fileId']}"])

    # check that the file is gone
    b2_tool.should_succeed(
        ['ls', f'b2://{bucket_name}'],
        expected_pattern='^$',
    )


def test_debug_logs(b2_tool, is_running_on_docker, tmp_path):
    to_be_removed_bucket_name = b2_tool.generate_bucket_name()
    b2_tool.should_succeed(
        [
            'create-bucket',
            to_be_removed_bucket_name,
            'allPublic',
            *b2_tool.get_bucket_info_args(),
        ],
    )
    b2_tool.should_succeed(['delete-bucket', to_be_removed_bucket_name],)
    b2_tool.should_fail(
        ['delete-bucket', to_be_removed_bucket_name],
        re.compile(r'^ERROR: Bucket with id=\w* not found\s*$')
    )
    # Check logging settings
    if not is_running_on_docker:  # It's difficult to read the log in docker in CI
        b2_tool.should_fail(
            ['delete-bucket', to_be_removed_bucket_name, '--debug-logs'],
            re.compile(r'^ERROR: Bucket with id=\w* not found\s*$')
        )
        stack_trace_in_log = r'Traceback \(most recent call last\):.*Bucket with id=\w* not found'

        # the two regexes below depend on log message from urllib3, which is not perfect, but this test needs to
        # check global logging settings
        stderr_regex = re.compile(
            r'DEBUG:urllib3.connectionpool:.* "POST /b2api/v2/b2_delete_bucket HTTP'
            r'.*' + stack_trace_in_log,
            re.DOTALL,
        )
        log_file_regex = re.compile(
            r'urllib3.connectionpool\tDEBUG\t.* "POST /b2api/v2/b2_delete_bucket HTTP'
            r'.*' + stack_trace_in_log,
            re.DOTALL,
        )
        with open('b2_cli.log') as logfile:
            log = logfile.read()
            assert re.search(log_file_regex, log), log
        os.remove('b2_cli.log')

        b2_tool.should_fail(['delete-bucket', to_be_removed_bucket_name, '--verbose'], stderr_regex)
        assert not os.path.exists('b2_cli.log')

        b2_tool.should_fail(
            ['delete-bucket', to_be_removed_bucket_name, '--verbose', '--debug-logs'], stderr_regex
        )
        with open('b2_cli.log') as logfile:
            log = logfile.read()
            assert re.search(log_file_regex, log), log


def test_bucket(b2_tool, bucket_name):
    rule = """{
        "daysFromHidingToDeleting": 1,
        "daysFromUploadingToHiding": null,
        "fileNamePrefix": ""
    }"""
    output = b2_tool.should_succeed_json(
        [
            'update-bucket', '--lifecycle-rule', rule, bucket_name, 'allPublic',
            *b2_tool.get_bucket_info_args()
        ],
    )

    ########## // doesn't happen on production, but messes up some tests \\ ##########
    for key in output['lifecycleRules'][0]:
        if key[8] == 'S' and len(key) == 47:
            del output['lifecycleRules'][0][key]
            break
    ########## \\ doesn't happen on production, but messes up some tests // ##########

    assert output["lifecycleRules"] == [
        {
            "daysFromHidingToDeleting": 1,
            "daysFromUploadingToHiding": None,
            "fileNamePrefix": ""
        }
    ]


def test_key_restrictions(b2_tool, bucket_name, sample_file, bucket_factory, b2_uri_args):
    # A single file for rm to fail on.
    b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'test'])

    key_one_name = 'clt-testKey-01' + random_hex(6)
    created_key_stdout = b2_tool.should_succeed(
        [
            'create-key',
            key_one_name,
            'listFiles,listBuckets,readFiles,writeKeys',
        ]
    )
    key_one_id, key_one = created_key_stdout.split()

    b2_tool.should_succeed(
        ['authorize-account', '--environment', b2_tool.realm, key_one_id, key_one],
    )

    b2_tool.should_succeed(['get-bucket', bucket_name],)
    second_bucket_name = bucket_factory().name
    b2_tool.should_succeed(['get-bucket', second_bucket_name],)

    key_two_name = 'clt-testKey-02' + random_hex(6)
    created_key_two_stdout = b2_tool.should_succeed(
        [
            'create-key',
            '--bucket',
            bucket_name,
            key_two_name,
            'listFiles,listBuckets,readFiles',
        ]
    )
    key_two_id, key_two = created_key_two_stdout.split()

    b2_tool.should_succeed(
        ['authorize-account', '--environment', b2_tool.realm, key_two_id, key_two],
    )
    b2_tool.should_succeed(['get-bucket', bucket_name],)
    b2_tool.should_succeed(['ls', *b2_uri_args(bucket_name)],)

    # Capabilities can be listed in any order. While this regex doesn't confirm that all three are present,
    # in ensures that there are three in total.
    failed_bucket_err = r'Deletion of file "test" \([^\)]+\) failed: unauthorized for ' \
                        r'application key with capabilities ' \
                        r"'(.*listFiles.*|.*listBuckets.*|.*readFiles.*){3}', " \
                        r"restricted to bucket '%s' \(unauthorized\)" % bucket_name
    b2_tool.should_fail(
        ['rm', '--recursive', '--no-progress', *b2_uri_args(bucket_name)], failed_bucket_err
    )

    failed_bucket_err = r'ERROR: Application key is restricted to bucket: ' + bucket_name
    b2_tool.should_fail(['get-bucket', second_bucket_name], failed_bucket_err)

    failed_list_files_err = r'ERROR: Application key is restricted to bucket: ' + bucket_name
    b2_tool.should_fail(['ls', *b2_uri_args(second_bucket_name)], failed_list_files_err)

    failed_list_files_err = r'ERROR: Application key is restricted to bucket: ' + bucket_name
    b2_tool.should_fail(['rm', *b2_uri_args(second_bucket_name)], failed_list_files_err)

    # reauthorize with more capabilities for clean up
    b2_tool.should_succeed(
        [
            'authorize-account', '--environment', b2_tool.realm, b2_tool.account_id,
            b2_tool.application_key
        ]
    )
    b2_tool.should_succeed(['delete-key', key_one_id])
    b2_tool.should_succeed(['delete-key', key_two_id])


def test_delete_bucket(b2_tool, bucket_name):
    b2_tool.should_succeed(['delete-bucket', bucket_name])
    b2_tool.should_fail(
        ['delete-bucket', bucket_name], re.compile(r'^ERROR: Bucket with id=\w* not found\s*$')
    )


def test_rapid_bucket_operations(b2_tool):
    new_bucket_name = b2_tool.generate_bucket_name()
    bucket_info_args = b2_tool.get_bucket_info_args()
    # apparently server behaves erratically when we delete a bucket and recreate it right away
    b2_tool.should_succeed(['create-bucket', new_bucket_name, 'allPrivate', *bucket_info_args])
    b2_tool.should_succeed(['update-bucket', new_bucket_name, 'allPublic'])
    b2_tool.should_succeed(['delete-bucket', new_bucket_name])


def test_account(b2_tool, cli_version, apiver_int, monkeypatch):

    with monkeypatch.context() as mp:
        account_info_file_path = os.path.join(mkdtemp(), 'b2_account_info')
        mp.setenv(B2_ACCOUNT_INFO_ENV_VAR, account_info_file_path)

        b2_tool.should_succeed(['clear-account'])
        bad_application_key = random_hex(len(b2_tool.application_key))
        b2_tool.should_fail(
            ['authorize-account', b2_tool.account_id, bad_application_key], r'unauthorized'
        )  # this call doesn't use --environment on purpose, so that we check that it is non-mandatory
        b2_tool.should_succeed(
            [
                'authorize-account',
                '--environment',
                b2_tool.realm,
                b2_tool.account_id,
                b2_tool.application_key,
            ]
        )

    # Testing (B2_APPLICATION_KEY, B2_APPLICATION_KEY_ID) for commands other than authorize-account
    with monkeypatch.context() as mp:
        account_info_file_path = os.path.join(mkdtemp(), 'b2_account_info')
        mp.setenv(B2_ACCOUNT_INFO_ENV_VAR, account_info_file_path)

        # first, let's make sure "create-bucket" doesn't work without auth data - i.e. that the sqlite file has been
        # successfully removed
        bucket_name = b2_tool.generate_bucket_name()
        b2_tool.should_fail(
            ['create-bucket', bucket_name, 'allPrivate'],
            r'ERROR: Missing account data: \'NoneType\' object is not subscriptable (\(key 0\) )? '
            fr'Use: {cli_version}(\.(exe|EXE))? authorize-account or provide auth data with \'B2_APPLICATION_KEY_ID\' and '
            r'\'B2_APPLICATION_KEY\' environment variables'
        )

    with monkeypatch.context() as mp:
        account_info_file_path = os.path.join(mkdtemp(), 'b2_account_info')
        mp.setenv(B2_ACCOUNT_INFO_ENV_VAR, account_info_file_path)

        # then, let's see that auth data from env vars works
        os.environ['B2_APPLICATION_KEY'] = os.environ['B2_TEST_APPLICATION_KEY']
        os.environ['B2_APPLICATION_KEY_ID'] = os.environ['B2_TEST_APPLICATION_KEY_ID']
        os.environ['B2_ENVIRONMENT'] = b2_tool.realm

        bucket_name = b2_tool.generate_bucket_name()
        b2_tool.should_succeed(
            ['create-bucket', bucket_name, 'allPrivate', *b2_tool.get_bucket_info_args()]
        )
        b2_tool.should_succeed(['delete-bucket', bucket_name])

        if apiver_int >= 4:
            assert not os.path.exists(
                account_info_file_path
            ), 'sqlite file was created while it shouldn\'t'
        else:
            assert os.path.exists(account_info_file_path), 'sqlite file was not created'
            account_info = SqliteAccountInfo(account_info_file_path)
            assert account_info.get_application_key_id() == os.environ['B2_TEST_APPLICATION_KEY_ID']
            assert account_info.get_application_key() == os.environ['B2_TEST_APPLICATION_KEY']

        os.environ.pop('B2_APPLICATION_KEY')
        os.environ.pop('B2_APPLICATION_KEY_ID')

        # last, let's see that providing only one of the env vars results in a failure
        os.environ['B2_APPLICATION_KEY'] = os.environ['B2_TEST_APPLICATION_KEY']
        b2_tool.should_fail(
            ['create-bucket', bucket_name, 'allPrivate'],
            r'Please provide both "B2_APPLICATION_KEY" and "B2_APPLICATION_KEY_ID" environment variables or none of them'
        )
        os.environ.pop('B2_APPLICATION_KEY')

        os.environ['B2_APPLICATION_KEY_ID'] = os.environ['B2_TEST_APPLICATION_KEY_ID']
        b2_tool.should_fail(
            ['create-bucket', bucket_name, 'allPrivate'],
            r'Please provide both "B2_APPLICATION_KEY" and "B2_APPLICATION_KEY_ID" environment variables or none of them'
        )
        os.environ.pop('B2_APPLICATION_KEY_ID')


def file_version_summary(list_of_files):
    """
    Given the result of list-file-versions, returns a list
    of all file versions, with "+" for upload and "-" for
    hide, looking like this:

       ['+ photos/a.jpg', '- photos/b.jpg', '+ photos/c.jpg']

    """
    return [filename_summary(f) for f in list_of_files]


def filename_summary(file_):
    return ('+ ' if (file_['action'] == 'upload') else '- ') + file_['fileName']


def file_version_summary_with_encryption(list_of_files):
    """
    Given the result of list-file-versions, returns a list
    of all file versions, with "+" for upload and "-" for
    hide, with information about encryption, looking like this:

       [
           ('+ photos/a.jpg', 'SSE-C:AES256?sse_c_key_id=user-generated-key-id'),
           ('+ photos/a.jpg', 'SSE-B2:AES256'),
           ('- photos/b.jpg', None),
           ('+ photos/c.jpg', 'none'),
       ]
    """
    result = []
    for f in list_of_files:
        entry = filename_summary(f)
        encryption = encryption_summary(f['serverSideEncryption'], f['fileInfo'])
        result.append((entry, encryption))
    return result


def find_file_id(list_of_files, file_name):
    for file in list_of_files:
        if file['fileName'] == file_name:
            return file['fileId']
    assert False, f'file not found: {file_name}'


def encryption_summary(sse_dict, file_info):
    if isinstance(sse_dict, EncryptionSetting):
        sse_dict = sse_dict.as_dict()
    encryption = sse_dict['mode']
    assert encryption in (
        EncryptionMode.NONE.value, EncryptionMode.SSE_B2.value, EncryptionMode.SSE_C.value
    )
    algorithm = sse_dict.get('algorithm')
    if algorithm is not None:
        encryption += ':' + algorithm
    if sse_dict['mode'] == 'SSE-C':
        sse_c_key_id = file_info.get(SSE_C_KEY_ID_FILE_INFO_KEY_NAME)
        encryption += f'?{SSE_C_KEY_ID_FILE_INFO_KEY_NAME}={sse_c_key_id}'

    return encryption


def test_sync_up(b2_tool, bucket_name):
    sync_up_helper(b2_tool, bucket_name, 'sync')


def test_sync_up_sse_b2(b2_tool, bucket_name):
    sync_up_helper(b2_tool, bucket_name, 'sync', encryption=SSE_B2_AES)


def test_sync_up_sse_c(b2_tool, bucket_name):
    sync_up_helper(b2_tool, bucket_name, 'sync', encryption=SSE_C_AES)


def test_sync_up_no_prefix(b2_tool, bucket_name):
    sync_up_helper(b2_tool, bucket_name, '')


def sync_up_helper(b2_tool, bucket_name, dir_, encryption=None):
    sync_point_parts = [bucket_name]
    if dir_:
        sync_point_parts.append(dir_)
        prefix = dir_ + '/'
    else:
        prefix = ''
    b2_sync_point = 'b2:' + '/'.join(sync_point_parts)

    with TempDir() as dir_path:
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal([], file_version_summary(file_versions))

        write_file(dir_path / 'a', b'hello')
        write_file(dir_path / 'b', b'hello')
        write_file(dir_path / 'c', b'hello')

        # simulate action (nothing should be uploaded)
        b2_tool.should_succeed(['sync', '--no-progress', '--dry-run', dir_path, b2_sync_point])
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal([], file_version_summary(file_versions))

        #
        # A note about OSError: [WinError 1314]
        #
        # If you are seeing this, then probably you ran the integration test suite from
        # a non-admin account which on Windows doesn't by default get to create symlinks.
        # A special permission is needed. Now maybe there is a way to give that permission,
        # but it didn't work for me, so I just ran it as admin. A guide that I've found
        # recommended to go to Control Panel, Administrative Tools, Local Security Policy,
        # Local Policies, User Rights Assignment and there you can find a permission to
        # create symbilic links. Add your user to it (or a group that the user is in).
        #
        # Finally in order to apply the new policy, run `cmd` and execute
        # ``gpupdate /force``.
        #
        # Again, if it still doesn't work, consider just running the shell you are
        # launching ``nox`` as admin.

        os.symlink('broken', dir_path / 'd')  # OSError: [WinError 1314] ? See the comment above

        additional_env = None

        # now upload
        if encryption is None:
            command = ['sync', '--no-progress', dir_path, b2_sync_point]
            expected_encryption = SSE_NONE
            expected_encryption_str = encryption_summary(expected_encryption.as_dict(), {})
        elif encryption == SSE_B2_AES:
            command = [
                'sync', '--no-progress', '--destination-server-side-encryption', 'SSE-B2', dir_path,
                b2_sync_point
            ]
            expected_encryption = encryption
            expected_encryption_str = encryption_summary(expected_encryption.as_dict(), {})
        elif encryption == SSE_C_AES:
            command = [
                'sync', '--no-progress', '--destination-server-side-encryption', 'SSE-C', dir_path,
                b2_sync_point
            ]
            expected_encryption = encryption
            additional_env = {
                'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(SSE_C_AES.key.secret).decode(),
                'B2_DESTINATION_SSE_C_KEY_ID': SSE_C_AES.key.key_id,
            }
            expected_encryption_str = encryption_summary(
                expected_encryption.as_dict(),
                {SSE_C_KEY_ID_FILE_INFO_KEY_NAME: SSE_C_AES.key.key_id}
            )
        else:
            raise NotImplementedError('unsupported encryption mode: %s' % encryption)

        b2_tool.should_succeed(
            command, expected_pattern="d could not be accessed", additional_env=additional_env
        )
        file_versions = b2_tool.list_file_versions(bucket_name)

        should_equal(
            [
                ('+ ' + prefix + 'a', expected_encryption_str),
                ('+ ' + prefix + 'b', expected_encryption_str),
                ('+ ' + prefix + 'c', expected_encryption_str),
            ],
            file_version_summary_with_encryption(file_versions),
        )
        if encryption and encryption.mode == EncryptionMode.SSE_C:
            b2_tool.should_fail(
                command,
                expected_pattern="ValueError: Using SSE-C requires providing an encryption key via "
                "B2_DESTINATION_SSE_C_KEY_B64 env var"
            )
        if encryption is not None:
            return  # that's enough, we've checked that encryption works, no need to repeat the whole sync suite

        c_id = find_file_id(file_versions, prefix + 'c')
        file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{c_id}"])['fileInfo']
        should_equal(
            file_mod_time_millis(dir_path / 'c'), int(file_info['src_last_modified_millis'])
        )

        os.unlink(dir_path / 'b')
        write_file(dir_path / 'c', b'hello world')

        b2_tool.should_succeed(
            ['sync', '--no-progress', '--keep-days', '10', dir_path, b2_sync_point]
        )
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal(
            [
                '+ ' + prefix + 'a',
                '- ' + prefix + 'b',
                '+ ' + prefix + 'b',
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'c',
            ], file_version_summary(file_versions)
        )

        os.unlink(dir_path / 'a')

        b2_tool.should_succeed(['sync', '--no-progress', '--delete', dir_path, b2_sync_point])
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal([
            '+ ' + prefix + 'c',
        ], file_version_summary(file_versions))

        # test --compare-threshold with file size
        write_file(dir_path / 'c', b'hello world!')

        # should not upload new version of c
        b2_tool.should_succeed(
            [
                'sync', '--no-progress', '--keep-days', '10', '--compare-versions', 'size',
                '--compare-threshold', '1', dir_path, b2_sync_point
            ]
        )
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal([
            '+ ' + prefix + 'c',
        ], file_version_summary(file_versions))

        # should upload new version of c
        b2_tool.should_succeed(
            [
                'sync', '--no-progress', '--keep-days', '10', '--compare-versions', 'size',
                dir_path, b2_sync_point
            ]
        )
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal(
            [
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'c',
            ], file_version_summary(file_versions)
        )

        set_file_mod_time_millis(dir_path / 'c', file_mod_time_millis(dir_path / 'c') + 2000)

        # test --compare-threshold with modTime
        # should not upload new version of c
        b2_tool.should_succeed(
            [
                'sync', '--no-progress', '--keep-days', '10', '--compare-versions', 'modTime',
                '--compare-threshold', '2000', dir_path, b2_sync_point
            ]
        )
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal(
            [
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'c',
            ], file_version_summary(file_versions)
        )

        # should upload new version of c
        b2_tool.should_succeed(
            [
                'sync', '--no-progress', '--keep-days', '10', '--compare-versions', 'modTime',
                dir_path, b2_sync_point
            ]
        )
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal(
            [
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'c',
            ], file_version_summary(file_versions)
        )

        # create one more file
        write_file(dir_path / 'linktarget', b'hello')
        mod_time = str((file_mod_time_millis(dir_path / 'linktarget') - 10) / 1000)

        # exclude last created file because of mtime
        b2_tool.should_succeed(
            [
                'sync', '--no-progress', '--exclude-if-modified-after', mod_time, dir_path,
                b2_sync_point
            ]
        )
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal(
            [
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'c',
            ],
            file_version_summary(file_versions),
        )

        # confirm symlink is skipped
        os.symlink('linktarget', dir_path / 'alink')

        b2_tool.should_succeed(
            ['sync', '--no-progress', '--exclude-all-symlinks', dir_path, b2_sync_point],
        )
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal(
            [
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'linktarget',
            ],
            file_version_summary(file_versions),
        )

        # confirm symlink target is uploaded (with symlink's name)
        b2_tool.should_succeed(['sync', '--no-progress', dir_path, b2_sync_point])
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal(
            [
                '+ ' + prefix + 'alink',
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'c',
                '+ ' + prefix + 'linktarget',
            ],
            file_version_summary(file_versions),
        )


def test_sync_down(b2_tool, bucket_name, sample_file):
    sync_down_helper(b2_tool, bucket_name, 'sync', sample_file)


def test_sync_down_no_prefix(b2_tool, bucket_name, sample_file):
    sync_down_helper(b2_tool, bucket_name, '', sample_file)


def test_sync_down_sse_c_no_prefix(b2_tool, bucket_name, sample_file):
    sync_down_helper(b2_tool, bucket_name, '', sample_file, SSE_C_AES)


def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, sample_file, encryption=None):

    b2_sync_point = 'b2:%s' % bucket_name
    if folder_in_bucket:
        b2_sync_point += '/' + folder_in_bucket
        b2_file_prefix = folder_in_bucket + '/'
    else:
        b2_file_prefix = ''

    if encryption is None or encryption.mode in (EncryptionMode.NONE, EncryptionMode.SSE_B2):
        upload_encryption_args = []
        upload_additional_env = {}
        sync_encryption_args = []
        sync_additional_env = {}
    elif encryption.mode == EncryptionMode.SSE_C:
        upload_encryption_args = ['--destination-server-side-encryption', 'SSE-C']
        upload_additional_env = {
            'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(encryption.key.secret).decode(),
            'B2_DESTINATION_SSE_C_KEY_ID': encryption.key.key_id,
        }
        sync_encryption_args = ['--source-server-side-encryption', 'SSE-C']
        sync_additional_env = {
            'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(encryption.key.secret).decode(),
            'B2_SOURCE_SSE_C_KEY_ID': encryption.key.key_id,
        }
    else:
        raise NotImplementedError(encryption)

    with TempDir() as local_path:
        # Sync from an empty "folder" as a source.
        b2_tool.should_succeed(['sync', b2_sync_point, local_path])
        should_equal([], sorted(local_path.iterdir()))

        # Put a couple files in B2
        b2_tool.should_succeed(
            ['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'a'] +
            upload_encryption_args,
            additional_env=upload_additional_env,
        )
        b2_tool.should_succeed(
            ['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'b'] +
            upload_encryption_args,
            additional_env=upload_additional_env,
        )
        b2_tool.should_succeed(
            ['sync', b2_sync_point, local_path] + sync_encryption_args,
            additional_env=sync_additional_env,
        )
        should_equal(['a', 'b'], sorted(os.listdir(local_path)))

        b2_tool.should_succeed(
            ['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'c'] +
            upload_encryption_args,
            additional_env=upload_additional_env,
        )

        # Sync the files with one file being excluded because of mtime
        mod_time = str((file_mod_time_millis(sample_file) - 10) / 1000)
        b2_tool.should_succeed(
            [
                'sync', '--no-progress', '--exclude-if-modified-after', mod_time, b2_sync_point,
                local_path
            ] + sync_encryption_args,
            additional_env=sync_additional_env,
        )
        should_equal(['a', 'b'], sorted(os.listdir(local_path)))
        # Sync all the files
        b2_tool.should_succeed(
            ['sync', '--no-progress', b2_sync_point, local_path] + sync_encryption_args,
            additional_env=sync_additional_env,
        )
        should_equal(['a', 'b', 'c'], sorted(os.listdir(local_path)))
    with TempDir() as new_local_path:
        if encryption and encryption.mode == EncryptionMode.SSE_C:
            b2_tool.should_fail(
                ['sync', '--no-progress', b2_sync_point, new_local_path] + sync_encryption_args,
                expected_pattern='ValueError: Using SSE-C requires providing an encryption key via '
                'B2_SOURCE_SSE_C_KEY_B64 env var',
            )
            b2_tool.should_fail(
                ['sync', '--no-progress', b2_sync_point, new_local_path],
                expected_pattern=
                'b2sdk._internal.exception.BadRequest: The object was stored using a form of Server Side '
                'Encryption. The correct parameters must be provided to retrieve the object. '
                r'\(bad_request\)',
            )


def test_sync_copy(bucket_factory, b2_tool, bucket_name, sample_file):
    prepare_and_run_sync_copy_tests(
        bucket_factory, b2_tool, bucket_name, 'sync', sample_file=sample_file
    )


def test_sync_copy_no_prefix_default_encryption(bucket_factory, b2_tool, bucket_name, sample_file):
    prepare_and_run_sync_copy_tests(
        bucket_factory,
        b2_tool,
        bucket_name,
        '',
        sample_file=sample_file,
        destination_encryption=None,
        expected_encryption=SSE_NONE
    )


def test_sync_copy_no_prefix_no_encryption(bucket_factory, b2_tool, bucket_name, sample_file):
    prepare_and_run_sync_copy_tests(
        bucket_factory,
        b2_tool,
        bucket_name,
        '',
        sample_file=sample_file,
        destination_encryption=SSE_NONE,
        expected_encryption=SSE_NONE
    )


def test_sync_copy_no_prefix_sse_b2(bucket_factory, b2_tool, bucket_name, sample_file):
    prepare_and_run_sync_copy_tests(
        bucket_factory,
        b2_tool,
        bucket_name,
        '',
        sample_file=sample_file,
        destination_encryption=SSE_B2_AES,
        expected_encryption=SSE_B2_AES,
    )


def test_sync_copy_no_prefix_sse_c(bucket_factory, b2_tool, bucket_name, sample_file):
    prepare_and_run_sync_copy_tests(
        bucket_factory,
        b2_tool,
        bucket_name,
        '',
        sample_file=sample_file,
        destination_encryption=SSE_C_AES,
        expected_encryption=SSE_C_AES,
        source_encryption=SSE_C_AES_2,
    )


def test_sync_copy_sse_c_single_bucket(b2_tool, bucket_name, sample_file):
    run_sync_copy_with_basic_checks(
        b2_tool=b2_tool,
        b2_file_prefix='first_folder/',
        b2_sync_point=f'b2:{bucket_name}/first_folder',
        bucket_name=bucket_name,
        other_b2_sync_point=f'b2:{bucket_name}/second_folder',
        destination_encryption=SSE_C_AES_2,
        source_encryption=SSE_C_AES,
        sample_file=sample_file,
    )
    expected_encryption_first = encryption_summary(
        SSE_C_AES.as_dict(),
        {SSE_C_KEY_ID_FILE_INFO_KEY_NAME: SSE_C_AES.key.key_id},
    )
    expected_encryption_second = encryption_summary(
        SSE_C_AES_2.as_dict(),
        {SSE_C_KEY_ID_FILE_INFO_KEY_NAME: SSE_C_AES_2.key.key_id},
    )

    file_versions = b2_tool.list_file_versions(bucket_name)
    should_equal(
        [
            ('+ first_folder/a', expected_encryption_first),
            ('+ first_folder/b', expected_encryption_first),
            ('+ second_folder/a', expected_encryption_second),
            ('+ second_folder/b', expected_encryption_second),
        ],
        file_version_summary_with_encryption(file_versions),
    )


def prepare_and_run_sync_copy_tests(
    bucket_factory,
    b2_tool,
    bucket_name,
    folder_in_bucket,
    sample_file,
    destination_encryption=None,
    expected_encryption=SSE_NONE,
    source_encryption=None,
):
    b2_sync_point = 'b2:%s' % bucket_name
    if folder_in_bucket:
        b2_sync_point += '/' + folder_in_bucket
        b2_file_prefix = folder_in_bucket + '/'
    else:
        b2_file_prefix = ''

    other_bucket_name = bucket_factory().name

    other_b2_sync_point = 'b2:%s' % other_bucket_name
    if folder_in_bucket:
        other_b2_sync_point += '/' + folder_in_bucket

    run_sync_copy_with_basic_checks(
        b2_tool=b2_tool,
        b2_file_prefix=b2_file_prefix,
        b2_sync_point=b2_sync_point,
        bucket_name=bucket_name,
        other_b2_sync_point=other_b2_sync_point,
        destination_encryption=destination_encryption,
        source_encryption=source_encryption,
        sample_file=sample_file,
    )

    if destination_encryption is None or destination_encryption in (SSE_NONE, SSE_B2_AES):
        encryption_file_info = {}
    elif destination_encryption.mode == EncryptionMode.SSE_C:
        encryption_file_info = {SSE_C_KEY_ID_FILE_INFO_KEY_NAME: destination_encryption.key.key_id}
    else:
        raise NotImplementedError(destination_encryption)

    file_versions = b2_tool.list_file_versions(other_bucket_name)
    expected_encryption_str = encryption_summary(
        expected_encryption.as_dict(), encryption_file_info
    )
    should_equal(
        [
            ('+ ' + b2_file_prefix + 'a', expected_encryption_str),
            ('+ ' + b2_file_prefix + 'b', expected_encryption_str),
        ],
        file_version_summary_with_encryption(file_versions),
    )


def run_sync_copy_with_basic_checks(
    b2_tool,
    b2_file_prefix,
    b2_sync_point,
    bucket_name,
    other_b2_sync_point,
    destination_encryption,
    source_encryption,
    sample_file,
):
    # Put a couple files in B2
    if source_encryption is None or source_encryption.mode in (
        EncryptionMode.NONE, EncryptionMode.SSE_B2
    ):
        b2_tool.should_succeed(
            [
                'upload-file', '--no-progress', '--destination-server-side-encryption', 'SSE-B2',
                bucket_name, sample_file, b2_file_prefix + 'a'
            ]
        )
        b2_tool.should_succeed(
            ['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'b']
        )
    elif source_encryption.mode == EncryptionMode.SSE_C:
        for suffix in ['a', 'b']:
            b2_tool.should_succeed(
                [
                    'upload-file', '--no-progress', '--destination-server-side-encryption', 'SSE-C',
                    bucket_name, sample_file, b2_file_prefix + suffix
                ],
                additional_env={
                    'B2_DESTINATION_SSE_C_KEY_B64':
                        base64.b64encode(source_encryption.key.secret).decode(),
                    'B2_DESTINATION_SSE_C_KEY_ID':
                        source_encryption.key.key_id,
                },
            )
    else:
        raise NotImplementedError(source_encryption)

    # Sync all the files
    if destination_encryption is None or destination_encryption == SSE_NONE:
        b2_tool.should_succeed(['sync', '--no-progress', b2_sync_point, other_b2_sync_point])
    elif destination_encryption == SSE_B2_AES:
        b2_tool.should_succeed(
            [
                'sync', '--no-progress', '--destination-server-side-encryption',
                destination_encryption.mode.value, b2_sync_point, other_b2_sync_point
            ]
        )
    elif destination_encryption.mode == EncryptionMode.SSE_C:
        b2_tool.should_fail(
            [
                'sync', '--no-progress', '--destination-server-side-encryption',
                destination_encryption.mode.value, b2_sync_point, other_b2_sync_point
            ],
            additional_env={
                'B2_DESTINATION_SSE_C_KEY_B64':
                    base64.b64encode(destination_encryption.key.secret).decode(),
                'B2_DESTINATION_SSE_C_KEY_ID':
                    destination_encryption.key.key_id,
            },
            expected_pattern=
            'b2sdk._internal.exception.BadRequest: The object was stored using a form of Server Side '
            'Encryption. The correct parameters must be provided to retrieve the object. '
            r'\(bad_request\)'
        )
        b2_tool.should_succeed(
            [
                'sync', '--no-progress', '--destination-server-side-encryption',
                destination_encryption.mode.value, '--source-server-side-encryption',
                source_encryption.mode.value, b2_sync_point, other_b2_sync_point
            ],
            additional_env={
                'B2_DESTINATION_SSE_C_KEY_B64':
                    base64.b64encode(destination_encryption.key.secret).decode(),
                'B2_DESTINATION_SSE_C_KEY_ID':
                    destination_encryption.key.key_id,
                'B2_SOURCE_SSE_C_KEY_B64':
                    base64.b64encode(source_encryption.key.secret).decode(),
                'B2_SOURCE_SSE_C_KEY_ID':
                    source_encryption.key.key_id,
            }
        )

    else:
        raise NotImplementedError(destination_encryption)


def test_sync_long_path(b2_tool, bucket_name):
    """
    test sync with very long path (overcome windows 260 character limit)
    """
    b2_sync_point = 'b2://' + bucket_name

    long_path = '/'.join(
        (
            'extremely_long_path_which_exceeds_windows_unfortunate_260_character_path_limit',
            'and_needs_special_prefixes_containing_backslashes_added_to_overcome_this_limitation',
            'when_doing_so_beware_leaning_toothpick_syndrome_as_it_can_cause_frustration',
            'see_also_xkcd_1638'
        )
    )

    with TempDir() as dir_path:
        local_long_path = (dir_path / long_path).resolve()
        fixed_local_long_path = Path(fix_windows_path_limit(str(local_long_path)))
        os.makedirs(fixed_local_long_path.parent)
        write_file(fixed_local_long_path, b'asdf')

        b2_tool.should_succeed(['sync', '--no-progress', '--delete', dir_path, b2_sync_point])
        file_versions = b2_tool.list_file_versions(bucket_name)
        should_equal(['+ ' + long_path], file_version_summary(file_versions))


def test_default_sse_b2__update_bucket(b2_tool, bucket_name, schedule_bucket_cleanup):
    # Set default encryption via update-bucket
    bucket_info = b2_tool.should_succeed_json(['get-bucket', bucket_name])
    bucket_default_sse = {'mode': 'none'}
    should_equal(bucket_default_sse, bucket_info['defaultServerSideEncryption'])

    bucket_info = b2_tool.should_succeed_json(
        ['update-bucket', '--default-server-side-encryption=SSE-B2', bucket_name]
    )
    bucket_default_sse = {
        'algorithm': 'AES256',
        'mode': 'SSE-B2',
    }
    should_equal(bucket_default_sse, bucket_info['defaultServerSideEncryption'])

    bucket_info = b2_tool.should_succeed_json(['get-bucket', bucket_name])
    bucket_default_sse = {
        'algorithm': 'AES256',
        'mode': 'SSE-B2',
    }
    should_equal(bucket_default_sse, bucket_info['defaultServerSideEncryption'])


def test_default_sse_b2__create_bucket(b2_tool, schedule_bucket_cleanup):
    # Set default encryption via create-bucket
    second_bucket_name = b2_tool.generate_bucket_name()
    schedule_bucket_cleanup(second_bucket_name)
    b2_tool.should_succeed(
        [
            'create-bucket',
            '--default-server-side-encryption=SSE-B2',
            second_bucket_name,
            'allPublic',
            *b2_tool.get_bucket_info_args(),
        ]
    )
    second_bucket_info = b2_tool.should_succeed_json(['get-bucket', second_bucket_name])
    second_bucket_default_sse = {
        'algorithm': 'AES256',
        'mode': 'SSE-B2',
    }
    should_equal(second_bucket_default_sse, second_bucket_info['defaultServerSideEncryption'])


def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args):
    b2_tool.should_succeed(
        [
            'upload-file', '--destination-server-side-encryption=SSE-B2', '--quiet', bucket_name,
            sample_file, 'encrypted'
        ]
    )
    b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, sample_file, 'not_encrypted'])

    b2_tool.should_succeed(
        ['download-file', '--quiet', f'b2://{bucket_name}/encrypted', tmp_path / 'encrypted']
    )
    b2_tool.should_succeed(
        [
            'download-file', '--quiet', f'b2://{bucket_name}/not_encrypted',
            tmp_path / 'not_encrypted'
        ]
    )

    list_of_files = b2_tool.should_succeed_json(
        ['ls', '--json', '--recursive', *b2_uri_args(bucket_name)]
    )
    should_equal(
        [{
            'algorithm': 'AES256',
            'mode': 'SSE-B2'
        }, {
            'mode': 'none'
        }], [f['serverSideEncryption'] for f in list_of_files]
    )

    encrypted_version = list_of_files[0]
    file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{encrypted_version['fileId']}"])
    should_equal({'algorithm': 'AES256', 'mode': 'SSE-B2'}, file_info['serverSideEncryption'])
    not_encrypted_version = list_of_files[1]
    file_info = b2_tool.should_succeed_json(
        ['file-info', f"b2id://{not_encrypted_version['fileId']}"]
    )
    should_equal({'mode': 'none'}, file_info['serverSideEncryption'])

    b2_tool.should_succeed(
        [
            'copy-file-by-id', '--destination-server-side-encryption=SSE-B2',
            encrypted_version['fileId'], bucket_name, 'copied_encrypted'
        ]
    )
    b2_tool.should_succeed(
        ['copy-file-by-id', not_encrypted_version['fileId'], bucket_name, 'copied_not_encrypted']
    )

    list_of_files = b2_tool.should_succeed_json(
        ['ls', '--json', '--recursive', *b2_uri_args(bucket_name)]
    )
    should_equal(
        [{
            'algorithm': 'AES256',
            'mode': 'SSE-B2'
        }, {
            'mode': 'none'
        }] * 2, [f['serverSideEncryption'] for f in list_of_files]
    )

    copied_encrypted_version = list_of_files[2]
    file_info = b2_tool.should_succeed_json(
        ['file-info', f"b2id://{copied_encrypted_version['fileId']}"]
    )
    should_equal({'algorithm': 'AES256', 'mode': 'SSE-B2'}, file_info['serverSideEncryption'])

    copied_not_encrypted_version = list_of_files[3]
    file_info = b2_tool.should_succeed_json(
        ['file-info', f"b2id://{copied_not_encrypted_version['fileId']}"]
    )
    should_equal({'mode': 'none'}, file_info['serverSideEncryption'])


def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path, b2_uri_args):

    sse_c_key_id = 'user-generated-key-id \nąóźćż\nœøΩ≈ç\nßäöü'
    if is_running_on_docker:
        # TODO: fix this once we figure out how to pass env vars with \n in them to docker, docker-compose should work
        sse_c_key_id = sse_c_key_id.replace('\n', '')

    secret = os.urandom(32)

    b2_tool.should_fail(
        [
            'upload-file', '--no-progress', '--quiet', '--destination-server-side-encryption',
            'SSE-C', bucket_name, sample_file, 'gonna-fail-anyway'
        ],
        'Using SSE-C requires providing an encryption key via B2_DESTINATION_SSE_C_KEY_B64 env var'
    )
    file_version_info = b2_tool.should_succeed_json(
        [
            'upload-file', '--no-progress', '--quiet', '--destination-server-side-encryption',
            'SSE-C', bucket_name, sample_file, 'uploaded_encrypted'
        ],
        additional_env={
            'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(secret).decode(),
            'B2_DESTINATION_SSE_C_KEY_ID': sse_c_key_id,
        }
    )
    should_equal(
        {
            "algorithm": "AES256",
            "customerKey": "******",
            "customerKeyMd5": "******",
            "mode": "SSE-C"
        }, file_version_info['serverSideEncryption']
    )
    should_equal(sse_c_key_id, file_version_info['fileInfo'][SSE_C_KEY_ID_FILE_INFO_KEY_NAME])

    b2_tool.should_fail(
        ['download-file', '--quiet', f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway'],
        expected_pattern='ERROR: The object was stored using a form of Server Side Encryption. The '
        r'correct parameters must be provided to retrieve the object. \(bad_request\)'
    )
    b2_tool.should_fail(
        [
            'download-file', '--quiet', '--source-server-side-encryption', 'SSE-C',
            f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway'
        ],
        expected_pattern='ValueError: Using SSE-C requires providing an encryption key via '
        'B2_SOURCE_SSE_C_KEY_B64 env var'
    )
    b2_tool.should_fail(
        [
            'download-file', '--quiet', '--source-server-side-encryption', 'SSE-C',
            f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway'
        ],
        expected_pattern='ERROR: Wrong or no SSE-C key provided when reading a file.',
        additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(os.urandom(32)).decode()}
    )
    with contextlib.nullcontext(tmp_path) as dir_path:
        b2_tool.should_succeed(
            [
                'download-file',
                '--no-progress',
                '--quiet',
                '--source-server-side-encryption',
                'SSE-C',
                f'b2://{bucket_name}/uploaded_encrypted',
                dir_path / 'a',
            ],
            additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()}
        )
        assert read_file(dir_path / 'a') == read_file(sample_file)
        b2_tool.should_succeed(
            [
                'download-file',
                '--no-progress',
                '--quiet',
                '--source-server-side-encryption',
                'SSE-C',
                f"b2id://{file_version_info['fileId']}",
                dir_path / 'b',
            ],
            additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()}
        )
        assert read_file(dir_path / 'b') == read_file(sample_file)

    b2_tool.should_fail(
        ['copy-file-by-id', file_version_info['fileId'], bucket_name, 'gonna-fail-anyway'],
        expected_pattern=
        'ERROR: The object was stored using a form of Server Side Encryption. The correct '
        r'parameters must be provided to retrieve the object. \(bad_request\)'
    )
    b2_tool.should_fail(
        [
            'copy-file-by-id', '--source-server-side-encryption=SSE-C', file_version_info['fileId'],
            bucket_name, 'gonna-fail-anyway'
        ],
        expected_pattern='ValueError: Using SSE-C requires providing an encryption key via '
        'B2_SOURCE_SSE_C_KEY_B64 env var'
    )
    b2_tool.should_fail(
        [
            'copy-file-by-id', '--source-server-side-encryption=SSE-C',
            '--destination-server-side-encryption=SSE-C', file_version_info['fileId'], bucket_name,
            'gonna-fail-anyway'
        ],
        expected_pattern='ValueError: Using SSE-C requires providing an encryption key via '
        'B2_DESTINATION_SSE_C_KEY_B64 env var',
        additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()}
    )
    b2_tool.should_fail(
        [
            'copy-file-by-id', '--source-server-side-encryption=SSE-C', file_version_info['fileId'],
            bucket_name, 'gonna-fail-anyway'
        ],
        additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()},
        expected_pattern=
        'Attempting to copy file with metadata while either source or destination uses '
        'SSE-C. Use --fetch-metadata to fetch source file metadata before copying.',
    )
    b2_tool.should_succeed(
        [
            'copy-file-by-id',
            '--source-server-side-encryption=SSE-C',
            file_version_info['fileId'],
            bucket_name,
            'not_encrypted_copied_from_encrypted_metadata_replace',
            '--info',
            'a=b',
            '--content-type',
            'text/plain',
        ],
        additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()}
    )
    b2_tool.should_succeed(
        [
            'copy-file-by-id',
            '--source-server-side-encryption=SSE-C',
            file_version_info['fileId'],
            bucket_name,
            'not_encrypted_copied_from_encrypted_metadata_replace_empty',
            '--no-info',
            '--content-type',
            'text/plain',
        ],
        additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()}
    )
    b2_tool.should_succeed(
        [
            'copy-file-by-id',
            '--source-server-side-encryption=SSE-C',
            file_version_info['fileId'],
            bucket_name,
            'not_encrypted_copied_from_encrypted_metadata_pseudo_copy',
            '--fetch-metadata',
        ],
        additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()}
    )
    b2_tool.should_succeed(
        [
            'copy-file-by-id',
            '--source-server-side-encryption=SSE-C',
            '--destination-server-side-encryption=SSE-C',
            file_version_info['fileId'],
            bucket_name,
            'encrypted_no_id_copied_from_encrypted',
            '--fetch-metadata',
        ],
        additional_env={
            'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode(),
            'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(os.urandom(32)).decode(),
        }
    )
    b2_tool.should_succeed(
        [
            'copy-file-by-id',
            '--source-server-side-encryption=SSE-C',
            '--destination-server-side-encryption=SSE-C',
            file_version_info['fileId'],
            bucket_name,
            'encrypted_with_id_copied_from_encrypted_metadata_replace',
            '--no-info',
            '--content-type',
            'text/plain',
        ],
        additional_env={
            'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode(),
            'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(os.urandom(32)).decode(),
            'B2_DESTINATION_SSE_C_KEY_ID': 'another-user-generated-key-id',
        }
    )
    b2_tool.should_succeed(
        [
            'copy-file-by-id',
            '--source-server-side-encryption=SSE-C',
            '--destination-server-side-encryption=SSE-C',
            file_version_info['fileId'],
            bucket_name,
            'encrypted_with_id_copied_from_encrypted_metadata_pseudo_copy',
            '--fetch-metadata',
        ],
        additional_env={
            'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode(),
            'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(os.urandom(32)).decode(),
            'B2_DESTINATION_SSE_C_KEY_ID': 'another-user-generated-key-id',
        }
    )
    list_of_files = b2_tool.should_succeed_json(
        ['ls', '--json', '--recursive', *b2_uri_args(bucket_name)]
    )
    should_equal(
        [
            {
                'file_name': 'encrypted_no_id_copied_from_encrypted',
                'sse_c_key_id': 'missing_key',
                'serverSideEncryption':
                    {
                        "algorithm": "AES256",
                        "customerKey": "******",
                        "customerKeyMd5": "******",
                        "mode": "SSE-C"
                    },
            },
            {
                'file_name': 'encrypted_with_id_copied_from_encrypted_metadata_pseudo_copy',
                'sse_c_key_id': 'another-user-generated-key-id',
                'serverSideEncryption':
                    {
                        'algorithm': 'AES256',
                        "customerKey": "******",
                        "customerKeyMd5": "******",
                        'mode': 'SSE-C',
                    },
            },
            {
                'file_name': 'encrypted_with_id_copied_from_encrypted_metadata_replace',
                'sse_c_key_id': 'another-user-generated-key-id',
                'serverSideEncryption':
                    {
                        'algorithm': 'AES256',
                        "customerKey": "******",
                        "customerKeyMd5": "******",
                        'mode': 'SSE-C',
                    },
            },
            {
                'file_name': 'not_encrypted_copied_from_encrypted_metadata_pseudo_copy',
                'sse_c_key_id': 'missing_key',
                'serverSideEncryption': {
                    'mode': 'none',
                },
            },
            {
                'file_name': 'not_encrypted_copied_from_encrypted_metadata_replace',
                'sse_c_key_id': 'missing_key',
                'serverSideEncryption': {
                    'mode': 'none',
                },
            },
            {
                'file_name': 'not_encrypted_copied_from_encrypted_metadata_replace_empty',
                'sse_c_key_id': 'missing_key',
                'serverSideEncryption': {
                    'mode': 'none',
                },
            },
            {
                'file_name': 'uploaded_encrypted',
                'sse_c_key_id': sse_c_key_id,
                'serverSideEncryption':
                    {
                        "algorithm": "AES256",
                        "customerKey": "******",
                        "customerKeyMd5": "******",
                        "mode": "SSE-C"
                    },
            },
        ],
        sorted(
            [
                {
                    'sse_c_key_id':
                        f['fileInfo'].get(SSE_C_KEY_ID_FILE_INFO_KEY_NAME, 'missing_key'),
                    'serverSideEncryption':
                        f['serverSideEncryption'],
                    'file_name':
                        f['fileName']
                } for f in list_of_files
            ],
            key=lambda r: r['file_name']
        )
    )


@pytest.mark.skipif(
    (sys.version_info.major, sys.version_info.minor) < (3, 8),
    reason="License extraction doesn't work on older versions, and we're only "
    "obliged to provide this "
    "data in bundled and built packages."
)
@pytest.mark.parametrize('with_packages', [True, False])
def test_license(b2_tool, with_packages, cli_version):
    license_text = b2_tool.should_succeed(
        ['license'] + (['--with-packages'] if with_packages else [])
    )

    if with_packages:
        # In the case of e.g.: docker image, it has a license built-in with a `b2`.
        # It also is unable to generate this license because it lacks required packages.
        # Thus, I'm allowing here for the test of licenses to pass whenever
        # the binary is named `b2` or with the proper cli version string (e.g. `_b2v4` or `b2v3`).
        full_license_re = re.compile(
            fr'Licenses of all modules used by ({cli_version}|b2)(\.EXE)?, shipped with it in binary form:\r?\n'
            r'\+-*\+-*\+\r?\n'
            r'\|\s*Module name\s*\|\s*License text\s*\|\r?\n'
            r'.*'
            r'\+-*\+-*\+\r?\n', re.MULTILINE + re.DOTALL
        )
        full_license_text = next(full_license_re.finditer(license_text), None)
        assert full_license_text, license_text
        assert len(
            full_license_text.group(0)
        ) > 140_000  # we should know if the length of this block changes dramatically
        # Note that GitHub CI adds additional packages:
        # 'colorlog', 'virtualenv', 'nox', 'packaging', 'argcomplete', 'filelock'
        # that sum up to around 50k characters. Tests ran from docker image are unaffected.

        # See the explanation above for why both `b2` and `cli_version` are allowed here.
        license_summary_re = re.compile(
            fr'Summary of all modules used by ({cli_version}|b2)(\.EXE)?, shipped with it in binary form:\r?\n'
            r'\+-*\+-*\+-*\+-*\+-*\+\r?\n'
            r'\|\s*Module name\s*\|\s*Version\s*\|\s*License\s*\|\s*Author\s*\|\s*URL\s*\|\r?\n'
            r'.*'
            r'\+-*\+-*\+-*\+-*\+-*\+\r?\n', re.MULTILINE + re.DOTALL
        )
        license_summary_text = next(license_summary_re.finditer(license_text), None)
        assert license_summary_text, license_text
        assert len(
            license_summary_text.group(0)
        ) > 6_300  # we should know if the length of this block changes dramatically

    assert """ license:
Backblaze wants developers and organization to copy and re-use our
code examples, so we make the samples available by several different
licenses.  One option is the MIT license (below).  Other options are
available here:

    https://www.backblaze.com/using_b2_code.html


The MIT License (MIT)

Copyright (c) 2015 Backblaze

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.""" in license_text.replace(os.linesep, '\n'), repr(license_text[-2000:])


def test_file_lock(
    b2_tool, application_key_id, application_key, sample_file, bucket_factory,
    schedule_bucket_cleanup
):
    lock_disabled_bucket_name = bucket_factory(bucket_type='allPrivate').name

    now_millis = current_time_millis()

    not_lockable_file = b2_tool.should_succeed_json(  # file in a lock disabled bucket
        ['upload-file', '--quiet', lock_disabled_bucket_name, sample_file, 'a']
    )

    _assert_file_lock_configuration(
        b2_tool,
        not_lockable_file['fileId'],
        retention_mode=RetentionMode.NONE,
        legal_hold=LegalHold.UNSET
    )

    b2_tool.should_fail(
        [
            'upload-file',
            '--quiet',
            lock_disabled_bucket_name,
            sample_file,
            'a',
            '--file-retention-mode',
            'governance',
            '--retain-until',
            str(now_millis + 1.5 * ONE_HOUR_MILLIS),
            '--legal-hold',
            'on',
        ], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)'
    )

    b2_tool.should_fail(
        [
            'update-bucket', lock_disabled_bucket_name, 'allPrivate', '--default-retention-mode',
            'compliance'
        ], 'ValueError: must specify period for retention mode RetentionMode.COMPLIANCE'
    )
    b2_tool.should_fail(
        [
            'update-bucket', lock_disabled_bucket_name, 'allPrivate', '--default-retention-mode',
            'compliance', '--default-retention-period', '7 days'
        ], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)'
    )
    lock_enabled_bucket_name = b2_tool.generate_bucket_name()
    schedule_bucket_cleanup(lock_enabled_bucket_name)
    b2_tool.should_succeed(
        [
            'create-bucket',
            lock_enabled_bucket_name,
            'allPrivate',
            '--file-lock-enabled',
            *b2_tool.get_bucket_info_args(),
        ],
    )
    updated_bucket = b2_tool.should_succeed_json(
        [
            'update-bucket',
            lock_enabled_bucket_name,
            'allPrivate',
            '--default-retention-mode',
            'governance',
            '--default-retention-period',
            '1 days',
        ],
    )
    assert updated_bucket['defaultRetention'] == {
        'mode': 'governance',
        'period': {
            'duration': 1,
            'unit': 'days',
        },
    }

    lockable_file = b2_tool.should_succeed_json(  # file in a lock enabled bucket
        ['upload-file', '--no-progress', '--quiet', lock_enabled_bucket_name, sample_file, 'a']
    )

    b2_tool.should_fail(
        [
            'update-file-retention', not_lockable_file['fileName'], not_lockable_file['fileId'],
            'governance', '--retain-until',
            str(now_millis + ONE_DAY_MILLIS + ONE_HOUR_MILLIS)
        ], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)'
    )

    b2_tool.should_succeed(  # first let's try with a file name
        ['update-file-retention', lockable_file['fileName'], lockable_file['fileId'], 'governance',
         '--retain-until', str(now_millis + ONE_DAY_MILLIS + ONE_HOUR_MILLIS)]
    )

    _assert_file_lock_configuration(
        b2_tool,
        lockable_file['fileId'],
        retention_mode=RetentionMode.GOVERNANCE,
        retain_until=now_millis + ONE_DAY_MILLIS + ONE_HOUR_MILLIS
    )

    b2_tool.should_succeed(  # and now without a file name
        ['update-file-retention', lockable_file['fileId'], 'governance',
         '--retain-until', str(now_millis + ONE_DAY_MILLIS + 2 * ONE_HOUR_MILLIS)]
    )

    _assert_file_lock_configuration(
        b2_tool,
        lockable_file['fileId'],
        retention_mode=RetentionMode.GOVERNANCE,
        retain_until=now_millis + ONE_DAY_MILLIS + 2 * ONE_HOUR_MILLIS
    )

    b2_tool.should_fail(
        [
            'update-file-retention', lockable_file['fileName'], lockable_file['fileId'],
            'governance', '--retain-until',
            str(now_millis + ONE_HOUR_MILLIS)
        ],
        "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or "
        "bypassGovernance=true parameter missing",
    )
    b2_tool.should_succeed(
        [
            'update-file-retention', lockable_file['fileName'], lockable_file['fileId'],
            'governance', '--retain-until',
            str(now_millis + ONE_HOUR_MILLIS), '--bypass-governance'
        ],
    )

    _assert_file_lock_configuration(
        b2_tool,
        lockable_file['fileId'],
        retention_mode=RetentionMode.GOVERNANCE,
        retain_until=now_millis + ONE_HOUR_MILLIS
    )

    b2_tool.should_fail(
        ['update-file-retention', lockable_file['fileName'], lockable_file['fileId'], 'none'],
        "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or "
        "bypassGovernance=true parameter missing",
    )
    b2_tool.should_succeed(
        [
            'update-file-retention', lockable_file['fileName'], lockable_file['fileId'], 'none',
            '--bypass-governance'
        ],
    )

    _assert_file_lock_configuration(
        b2_tool, lockable_file['fileId'], retention_mode=RetentionMode.NONE
    )

    b2_tool.should_fail(
        ['update-file-legal-hold', not_lockable_file['fileId'], 'on'],
        r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)'
    )

    b2_tool.should_succeed(  # first let's try with a file name
        ['update-file-legal-hold', lockable_file['fileName'], lockable_file['fileId'], 'on'],
    )

    _assert_file_lock_configuration(b2_tool, lockable_file['fileId'], legal_hold=LegalHold.ON)

    b2_tool.should_succeed(  # and now without a file name
        ['update-file-legal-hold', lockable_file['fileId'], 'off'],
    )

    _assert_file_lock_configuration(b2_tool, lockable_file['fileId'], legal_hold=LegalHold.OFF)

    updated_bucket = b2_tool.should_succeed_json(
        [
            'update-bucket',
            lock_enabled_bucket_name,
            'allPrivate',
            '--default-retention-mode',
            'none',
        ],
    )
    assert updated_bucket['defaultRetention'] == {'mode': None}

    b2_tool.should_fail(
        [
            'upload-file',
            '--no-progress',
            '--quiet',
            lock_enabled_bucket_name,
            sample_file,
            'a',
            '--file-retention-mode',
            'governance',
            '--retain-until',
            str(now_millis - 1.5 * ONE_HOUR_MILLIS),
        ],
        r'ERROR: The retainUntilTimestamp must be in future \(retain_until_timestamp_must_be_in_future\)',
    )

    uploaded_file = b2_tool.should_succeed_json(
        [
            'upload-file',
            '--no-progress',
            '--quiet',
            lock_enabled_bucket_name,
            sample_file,
            'a',
            '--file-retention-mode',
            'governance',
            '--retain-until',
            str(now_millis + 1.5 * ONE_HOUR_MILLIS),
            '--legal-hold',
            'on',
        ]
    )

    _assert_file_lock_configuration(
        b2_tool,
        uploaded_file['fileId'],
        retention_mode=RetentionMode.GOVERNANCE,
        retain_until=now_millis + 1.5 * ONE_HOUR_MILLIS,
        legal_hold=LegalHold.ON
    )

    b2_tool.should_fail(
        [
            'copy-file-by-id',
            lockable_file['fileId'],
            lock_disabled_bucket_name,
            'copied',
            '--file-retention-mode',
            'governance',
            '--retain-until',
            str(now_millis + 1.25 * ONE_HOUR_MILLIS),
            '--legal-hold',
            'off',
        ], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)'
    )

    copied_file = b2_tool.should_succeed_json(
        [
            'copy-file-by-id',
            lockable_file['fileId'],
            lock_enabled_bucket_name,
            'copied',
            '--file-retention-mode',
            'governance',
            '--retain-until',
            str(now_millis + 1.25 * ONE_HOUR_MILLIS),
            '--legal-hold',
            'off',
        ]
    )

    _assert_file_lock_configuration(
        b2_tool,
        copied_file['fileId'],
        retention_mode=RetentionMode.GOVERNANCE,
        retain_until=now_millis + 1.25 * ONE_HOUR_MILLIS,
        legal_hold=LegalHold.OFF
    )
    lock_disabled_key_id, lock_disabled_key = make_lock_disabled_key(b2_tool)

    b2_tool.should_succeed(
        [
            'authorize-account', '--environment', b2_tool.realm, lock_disabled_key_id,
            lock_disabled_key
        ],
    )

    file_lock_without_perms_test(
        b2_tool,
        lock_enabled_bucket_name,
        lock_disabled_bucket_name,
        lockable_file['fileId'],
        not_lockable_file['fileId'],
        sample_file=sample_file
    )

    b2_tool.should_succeed(
        ['authorize-account', '--environment', b2_tool.realm, application_key_id, application_key],
    )

    deleting_locked_files(
        b2_tool, lock_enabled_bucket_name, lock_disabled_key_id, lock_disabled_key, sample_file
    )


def make_lock_disabled_key(b2_tool):
    key_name = 'no-perms-for-file-lock' + random_hex(6)
    created_key_stdout = b2_tool.should_succeed(
        [
            'create-key',
            key_name,
            'listFiles,listBuckets,readFiles,writeKeys,deleteFiles',
        ]
    )
    key_id, key = created_key_stdout.split()
    return key_id, key


def file_lock_without_perms_test(
    b2_tool, lock_enabled_bucket_name, lock_disabled_bucket_name, lockable_file_id,
    not_lockable_file_id, sample_file
):

    b2_tool.should_fail(
        [
            'update-bucket', lock_enabled_bucket_name, 'allPrivate', '--default-retention-mode',
            'governance', '--default-retention-period', '1 days'
        ],
        'ERROR: unauthorized for application key with capabilities',
    )

    _assert_file_lock_configuration(
        b2_tool,
        lockable_file_id,
        retention_mode=RetentionMode.UNKNOWN,
        legal_hold=LegalHold.UNKNOWN
    )

    b2_tool.should_fail(
        [
            'update-file-retention', lockable_file_id, 'governance', '--retain-until',
            str(current_time_millis() + 7 * ONE_DAY_MILLIS)
        ],
        "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or "
        "bypassGovernance=true parameter missing",
    )

    b2_tool.should_fail(
        [
            'update-file-retention', not_lockable_file_id, 'governance', '--retain-until',
            str(current_time_millis() + 7 * ONE_DAY_MILLIS)
        ],
        "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or "
        "bypassGovernance=true parameter missing",
    )

    b2_tool.should_fail(
        ['update-file-legal-hold', lockable_file_id, 'on'],
        "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or "
        "bypassGovernance=true parameter missing",
    )

    b2_tool.should_fail(
        ['update-file-legal-hold', not_lockable_file_id, 'on'],
        "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or "
        "bypassGovernance=true parameter missing",
    )

    b2_tool.should_fail(
        [
            'upload-file',
            '--no-progress',
            '--quiet',
            lock_enabled_bucket_name,
            sample_file,
            'bound_to_fail_anyway',
            '--file-retention-mode',
            'governance',
            '--retain-until',
            str(current_time_millis() + ONE_HOUR_MILLIS),
            '--legal-hold',
            'on',
        ],
        "unauthorized for application key with capabilities",
    )

    b2_tool.should_fail(
        [
            'upload-file',
            '--no-progress',
            '--quiet',
            lock_disabled_bucket_name,
            sample_file,
            'bound_to_fail_anyway',
            '--file-retention-mode',
            'governance',
            '--retain-until',
            str(current_time_millis() + ONE_HOUR_MILLIS),
            '--legal-hold',
            'on',
        ],
        "unauthorized for application key with capabilities",
    )

    b2_tool.should_fail(
        [
            'copy-file-by-id',
            lockable_file_id,
            lock_enabled_bucket_name,
            'copied',
            '--file-retention-mode',
            'governance',
            '--retain-until',
            str(current_time_millis() + ONE_HOUR_MILLIS),
            '--legal-hold',
            'off',
        ],
        'ERROR: unauthorized for application key with capabilities',
    )

    b2_tool.should_fail(
        [
            'copy-file-by-id',
            lockable_file_id,
            lock_disabled_bucket_name,
            'copied',
            '--file-retention-mode',
            'governance',
            '--retain-until',
            str(current_time_millis() + ONE_HOUR_MILLIS),
            '--legal-hold',
            'off',
        ],
        'ERROR: unauthorized for application key with capabilities',
    )


def upload_locked_file(b2_tool, bucket_name, sample_file):
    return b2_tool.should_succeed_json(
        [
            'upload-file',
            '--no-progress',
            '--quiet',
            '--file-retention-mode',
            'governance',
            '--retain-until',
            str(int(time.time()) + 1000),
            bucket_name,
            sample_file,
            'a-locked',
        ]
    )


def deleting_locked_files(
    b2_tool, lock_enabled_bucket_name, lock_disabled_key_id, lock_disabled_key, sample_file
):
    locked_file = upload_locked_file(b2_tool, lock_enabled_bucket_name, sample_file)
    b2_tool.should_fail(
        [  # master key
            'delete-file-version',
            locked_file['fileName'],
            locked_file['fileId'],
        ],
        "ERROR: Access Denied for application key "
    )
    b2_tool.should_succeed([  # master key
        'delete-file-version',
        locked_file['fileName'],
        locked_file['fileId'],
        '--bypass-governance'
    ])

    locked_file = upload_locked_file(b2_tool, lock_enabled_bucket_name, sample_file)

    b2_tool.should_succeed(
        [
            'authorize-account', '--environment', b2_tool.realm, lock_disabled_key_id,
            lock_disabled_key
        ],
    )
    b2_tool.should_fail([  # lock disabled key
        'delete-file-version',
        locked_file['fileName'],
        locked_file['fileId'],
        '--bypass-governance',
    ], "ERROR: unauthorized for application key with capabilities '")


def test_profile_switch(b2_tool):
    # this test could be unit, but it adds a lot of complexity because of
    # necessity to pass mocked B2Api to ConsoleTool; it's much easier to
    # just have an integration test instead

    MISSING_ACCOUNT_PATTERN = 'Missing account data'

    b2_tool.should_succeed(
        [
            'authorize-account',
            '--environment',
            b2_tool.realm,
            b2_tool.account_id,
            b2_tool.application_key,
        ]
    )
    b2_tool.should_succeed(['get-account-info'])
    b2_tool.should_succeed(['clear-account'])
    b2_tool.should_fail(['get-account-info'], expected_pattern=MISSING_ACCOUNT_PATTERN)

    # in order to use --profile flag, we need to temporary
    # delete B2_ACCOUNT_INFO_ENV_VAR
    B2_ACCOUNT_INFO = os.environ.pop(B2_ACCOUNT_INFO_ENV_VAR, None)

    # now authorize a different account
    profile = 'profile-for-test-' + random_hex(6)
    b2_tool.should_fail(
        ['get-account-info', '--profile', profile],
        expected_pattern=MISSING_ACCOUNT_PATTERN,
    )
    b2_tool.should_succeed(
        [
            'authorize-account',
            '--environment',
            b2_tool.realm,
            '--profile',
            profile,
            b2_tool.account_id,
            b2_tool.application_key,
        ]
    )

    account_info = b2_tool.should_succeed_json(['get-account-info', '--profile', profile])
    account_file_path = account_info['accountFilePath']
    assert profile in account_file_path, \
        f'accountFilePath "{account_file_path}" should contain profile name "{profile}"'

    b2_tool.should_succeed(['clear-account', '--profile', profile])
    b2_tool.should_fail(
        ['get-account-info', '--profile', profile],
        expected_pattern=MISSING_ACCOUNT_PATTERN,
    )
    os.remove(account_file_path)

    # restore B2_ACCOUNT_INFO_ENV_VAR, if existed
    if B2_ACCOUNT_INFO:
        os.environ[B2_ACCOUNT_INFO_ENV_VAR] = B2_ACCOUNT_INFO


def test_replication_basic(b2_tool, bucket_name, schedule_bucket_cleanup):
    key_one_name = 'clt-testKey-01' + random_hex(6)
    created_key_stdout = b2_tool.should_succeed(
        [
            'create-key',
            key_one_name,
            'listBuckets,readFiles',
        ]
    )
    key_one_id, _ = created_key_stdout.split()

    key_two_name = 'clt-testKey-02' + random_hex(6)
    created_key_stdout = b2_tool.should_succeed(
        [
            'create-key',
            key_two_name,
            'listBuckets,writeFiles',
        ]
    )
    key_two_id, _ = created_key_stdout.split()

    destination_bucket_name = bucket_name
    destination_bucket = b2_tool.should_succeed_json(['get-bucket', destination_bucket_name])

    # test that by default there's no `replicationConfiguration` key
    assert 'replicationConfiguration' not in destination_bucket

    # ---------------- set up replication destination ----------------

    # update destination bucket info
    destination_replication_configuration = {
        'asReplicationSource': None,
        'asReplicationDestination': {
            'sourceToDestinationKeyMapping': {
                key_one_id: key_two_id,
            },
        },
    }
    destination_replication_configuration_json = json.dumps(destination_replication_configuration)
    destination_bucket = b2_tool.should_succeed_json(
        [
            'update-bucket',
            destination_bucket_name,
            'allPublic',
            '--replication',
            destination_replication_configuration_json,
        ]
    )

    # test that destination bucket is registered as replication destination
    assert destination_bucket['replication'].get('asReplicationSource') is None
    assert destination_bucket['replication'
                             ]['asReplicationDestination'
                              ] == destination_replication_configuration['asReplicationDestination']

    # ---------------- set up replication source ----------------
    source_replication_configuration = {
        "asReplicationSource":
            {
                "replicationRules":
                    [
                        {
                            "destinationBucketId": destination_bucket['bucketId'],
                            "fileNamePrefix": "one/",
                            "includeExistingFiles": False,
                            "isEnabled": True,
                            "priority": 1,
                            "replicationRuleName": "replication-one"
                        }, {
                            "destinationBucketId": destination_bucket['bucketId'],
                            "fileNamePrefix": "two/",
                            "includeExistingFiles": False,
                            "isEnabled": True,
                            "priority": 2,
                            "replicationRuleName": "replication-two"
                        }
                    ],
                "sourceApplicationKeyId": key_one_id,
            },
    }
    source_replication_configuration_json = json.dumps(source_replication_configuration)

    # create a source bucket and set up replication to destination bucket
    source_bucket_name = b2_tool.generate_bucket_name()
    schedule_bucket_cleanup(source_bucket_name)
    b2_tool.should_succeed(
        [
            'create-bucket',
            source_bucket_name,
            'allPublic',
            '--replication',
            source_replication_configuration_json,
            *b2_tool.get_bucket_info_args(),
        ]
    )
    source_bucket = b2_tool.should_succeed_json(['get-bucket', source_bucket_name])

    # test that all replication rules are present in source bucket
    assert source_bucket['replication']['asReplicationSource'
                                       ] == source_replication_configuration['asReplicationSource']

    # test that source bucket is not mentioned as replication destination
    assert source_bucket['replication'].get('asReplicationDestination') is None

    # ---------------- attempt enabling object lock  ----------------
    b2_tool.should_fail(
        ['update-bucket', source_bucket_name, '--file-lock-enabled'],
        'ERROR: Operation not supported for buckets with source replication'
    )

    # ---------------- remove replication source ----------------

    no_replication_configuration = {
        'asReplicationSource': None,
        'asReplicationDestination': None,
    }
    no_replication_configuration_json = json.dumps(no_replication_configuration)
    source_bucket = b2_tool.should_succeed_json(
        [
            'update-bucket', source_bucket_name, 'allPublic', '--replication',
            no_replication_configuration_json
        ]
    )

    # test that source bucket replication is removed
    assert source_bucket['replication'] == {
        'asReplicationDestination': None,
        'asReplicationSource': None
    }

    # ---------------- remove replication destination ----------------

    destination_bucket = b2_tool.should_succeed_json(
        [
            'update-bucket',
            destination_bucket_name,
            'allPublic',
            '--replication',
            '{}',
        ]
    )

    # test that destination bucket replication is removed
    assert destination_bucket['replication'] == {
        'asReplicationDestination': None,
        'asReplicationSource': None
    }

    b2_tool.should_succeed(['delete-key', key_one_id])
    b2_tool.should_succeed(['delete-key', key_two_id])


def test_replication_setup(b2_tool, bucket_name, schedule_bucket_cleanup):
    source_bucket_name = b2_tool.generate_bucket_name()
    schedule_bucket_cleanup(source_bucket_name)
    b2_tool.should_succeed(
        [
            'create-bucket',
            source_bucket_name,
            'allPublic',
            '--file-lock-enabled',
            *b2_tool.get_bucket_info_args(),
        ]
    )
    destination_bucket_name = bucket_name
    b2_tool.should_succeed(['replication-setup', source_bucket_name, destination_bucket_name])
    destination_bucket_old = b2_tool.should_succeed_json(['get-bucket', destination_bucket_name])

    b2_tool.should_succeed(
        [
            'replication-setup',
            '--priority',
            '132',
            '--file-name-prefix',
            'foo',
            '--name',
            'my-replication-rule',
            source_bucket_name,
            destination_bucket_name,
        ]
    )
    source_bucket = b2_tool.should_succeed_json(['get-bucket', source_bucket_name])
    destination_bucket = b2_tool.should_succeed_json(['get-bucket', destination_bucket_name])
    assert source_bucket['replication']['asReplicationSource']['replicationRules'] == [
        {
            "destinationBucketId": destination_bucket['bucketId'],
            "fileNamePrefix": "",
            "includeExistingFiles": False,
            "isEnabled": True,
            "priority": 128,
            "replicationRuleName": destination_bucket['bucketName'],
        },
        {
            "destinationBucketId": destination_bucket['bucketId'],
            "fileNamePrefix": "foo",
            "includeExistingFiles": False,
            "isEnabled": True,
            "priority": 132,
            "replicationRuleName": "my-replication-rule",
        },
    ]

    for key_one_id, key_two_id in destination_bucket['replication']['asReplicationDestination'][
        'sourceToDestinationKeyMapping'].items():
        b2_tool.should_succeed(['delete-key', key_one_id])
        b2_tool.should_succeed(['delete-key', key_two_id])
    assert destination_bucket_old['replication']['asReplicationDestination'][
        'sourceToDestinationKeyMapping'] == destination_bucket['replication'][
            'asReplicationDestination']['sourceToDestinationKeyMapping']


def test_replication_monitoring(b2_tool, bucket_name, sample_file, schedule_bucket_cleanup):

    # ---------------- set up keys ----------------
    key_one_name = 'clt-testKey-01' + random_hex(6)
    created_key_stdout = b2_tool.should_succeed(
        [
            'create-key',
            key_one_name,
            'listBuckets,readFiles',
        ]
    )
    key_one_id, _ = created_key_stdout.split()

    key_two_name = 'clt-testKey-02' + random_hex(6)
    created_key_stdout = b2_tool.should_succeed(
        [
            'create-key',
            key_two_name,
            'listBuckets,writeFiles',
        ]
    )
    key_two_id, _ = created_key_stdout.split()

    # ---------------- add test data ----------------
    destination_bucket_name = bucket_name
    uploaded_a = b2_tool.should_succeed_json(
        ['upload-file', '--quiet', destination_bucket_name, sample_file, 'one/a']
    )

    # ---------------- set up replication destination ----------------

    # update destination bucket info
    destination_replication_configuration = {
        'asReplicationSource': None,
        'asReplicationDestination': {
            'sourceToDestinationKeyMapping': {
                key_one_id: key_two_id,
            },
        },
    }
    destination_replication_configuration_json = json.dumps(destination_replication_configuration)
    destination_bucket = b2_tool.should_succeed_json(
        [
            'update-bucket',
            destination_bucket_name,
            'allPublic',
            '--replication',
            destination_replication_configuration_json,
        ]
    )

    # ---------------- set up replication source ----------------
    source_replication_configuration = {
        "asReplicationSource":
            {
                "replicationRules":
                    [
                        {
                            "destinationBucketId": destination_bucket['bucketId'],
                            "fileNamePrefix": "one/",
                            "includeExistingFiles": False,
                            "isEnabled": True,
                            "priority": 1,
                            "replicationRuleName": "replication-one"
                        }, {
                            "destinationBucketId": destination_bucket['bucketId'],
                            "fileNamePrefix": "two/",
                            "includeExistingFiles": False,
                            "isEnabled": True,
                            "priority": 2,
                            "replicationRuleName": "replication-two"
                        }
                    ],
                "sourceApplicationKeyId": key_one_id,
            },
    }
    source_replication_configuration_json = json.dumps(source_replication_configuration)

    # create a source bucket and set up replication to destination bucket
    source_bucket_name = b2_tool.generate_bucket_name()
    schedule_bucket_cleanup(source_bucket_name)
    b2_tool.should_succeed(
        [
            'create-bucket',
            source_bucket_name,
            'allPublic',
            '--file-lock-enabled',
            '--replication',
            source_replication_configuration_json,
            *b2_tool.get_bucket_info_args(),
        ]
    )

    # make test data
    uploaded_a = b2_tool.should_succeed_json(
        ['upload-file', '--quiet', source_bucket_name, sample_file, 'one/a']
    )
    b2_tool.should_succeed_json(
        [
            'upload-file',
            '--quiet',
            source_bucket_name,
            '--legal-hold',
            'on',
            sample_file,
            'two/b',
        ]
    )

    # encryption
    # SSE-B2
    upload_encryption_args = ['--destination-server-side-encryption', 'SSE-B2']
    upload_additional_env = {}
    b2_tool.should_succeed_json(
        ['upload-file', '--quiet', source_bucket_name, sample_file, 'two/c'] +
        upload_encryption_args,
        additional_env=upload_additional_env,
    )

    # SSE-C
    upload_encryption_args = ['--destination-server-side-encryption', 'SSE-C']
    upload_additional_env = {
        'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(SSE_C_AES.key.secret).decode(),
        'B2_DESTINATION_SSE_C_KEY_ID': SSE_C_AES.key.key_id,
    }
    b2_tool.should_succeed_json(
        ['upload-file', '--quiet', source_bucket_name, sample_file, 'two/d'] +
        upload_encryption_args,
        additional_env=upload_additional_env,
    )

    # encryption + legal hold
    b2_tool.should_succeed_json(
        [
            'upload-file',
            '--quiet',
            source_bucket_name,
            sample_file,
            'two/e',
            '--legal-hold',
            'on',
        ] + upload_encryption_args,
        additional_env=upload_additional_env,
    )

    # there is just one file, so clean after itself for faster execution
    b2_tool.should_succeed(['delete-file-version', uploaded_a['fileName'], uploaded_a['fileId']])

    # run stats command
    replication_status_json = b2_tool.should_succeed_json(
        [
            'replication-status',
            # '--destination-profile',
            # profile,
            '--no-progress',
            # '--columns=count, hash differs',
            '--output-format',
            'json',
            source_bucket_name,
        ]
    )

    assert replication_status_json in [
        {
            "replication-one":
                [
                    {
                        "count": 1,
                        "destination_replication_status": None,
                        "hash_differs": None,
                        "metadata_differs": None,
                        "source_has_file_retention": None,
                        "source_has_hide_marker": None,
                        "source_has_large_metadata": None,
                        "source_has_legal_hold": None,
                        "source_encryption_mode": None,
                        "source_replication_status": None,
                    }
                ],
            "replication-two":
                [
                    {
                        "count": 1,
                        "destination_replication_status": None,
                        "hash_differs": None,
                        "metadata_differs": None,
                        "source_has_file_retention": False,
                        "source_has_hide_marker": False,
                        "source_has_large_metadata": False,
                        "source_has_legal_hold": True,
                        "source_encryption_mode": 'none',
                        "source_replication_status": first,
                    }, {
                        "count": 1,
                        "destination_replication_status": None,
                        "hash_differs": None,
                        "metadata_differs": None,
                        "source_has_file_retention": False,
                        "source_has_hide_marker": False,
                        "source_has_large_metadata": False,
                        "source_has_legal_hold": False,
                        "source_encryption_mode": 'SSE-B2',
                        "source_replication_status": second,
                    }, {
                        "count": 1,
                        "destination_replication_status": None,
                        "hash_differs": None,
                        "metadata_differs": None,
                        "source_has_file_retention": False,
                        "source_has_hide_marker": False,
                        "source_has_large_metadata": False,
                        "source_has_legal_hold": False,
                        "source_encryption_mode": 'SSE-C',
                        "source_replication_status": None,
                    }, {
                        "count": 1,
                        "destination_replication_status": None,
                        "hash_differs": None,
                        "metadata_differs": None,
                        "source_has_file_retention": False,
                        "source_has_hide_marker": False,
                        "source_has_large_metadata": False,
                        "source_has_legal_hold": True,
                        "source_encryption_mode": 'SSE-C',
                        "source_replication_status": None,
                    }
                ]
        } for first, second in itertools.product(['FAILED', 'PENDING'], ['FAILED', 'PENDING'])
    ]


def test_enable_file_lock_first_retention_second(b2_tool, bucket_name):
    # enable file lock only
    b2_tool.should_succeed(['update-bucket', bucket_name, '--file-lock-enabled'])

    # set retention with file lock already enabled
    b2_tool.should_succeed(
        [
            'update-bucket', bucket_name, '--default-retention-mode', 'compliance',
            '--default-retention-period', '7 days'
        ]
    )

    # attempt to re-enable should be a noop
    b2_tool.should_succeed(['update-bucket', bucket_name, '--file-lock-enabled'])


def test_enable_file_lock_and_set_retention_at_once(b2_tool, bucket_name):
    # attempt setting retention without file lock enabled
    b2_tool.should_fail(
        [
            'update-bucket', bucket_name, '--default-retention-mode', 'compliance',
            '--default-retention-period', '7 days'
        ], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)'
    )

    # enable file lock and set retention at once
    b2_tool.should_succeed(
        [
            'update-bucket', bucket_name, '--default-retention-mode', 'compliance',
            '--default-retention-period', '7 days', '--file-lock-enabled'
        ]
    )

    # attempt to re-enable should be a noop
    b2_tool.should_succeed(['update-bucket', bucket_name, '--file-lock-enabled'])


def _assert_file_lock_configuration(
    b2_tool,
    file_id,
    retention_mode: RetentionMode | None = None,
    retain_until: int | None = None,
    legal_hold: LegalHold | None = None
):

    file_version = b2_tool.should_succeed_json(['file-info', f"b2id://{file_id}"])
    if retention_mode is not None:
        if file_version['fileRetention']['mode'] == 'unknown':
            actual_file_retention = UNKNOWN_FILE_RETENTION_SETTING
        else:
            actual_file_retention = FileRetentionSetting.from_file_retention_value_dict(
                file_version['fileRetention']
            )
        expected_file_retention = FileRetentionSetting(retention_mode, retain_until)
        assert expected_file_retention == actual_file_retention
    if legal_hold is not None:
        if file_version['legalHold'] == 'unknown':
            actual_legal_hold = LegalHold.UNKNOWN
        else:
            actual_legal_hold = LegalHold.from_string_or_none(file_version['legalHold'])
        assert legal_hold == actual_legal_hold


def test_upload_file__custom_upload_time(b2_tool, bucket_name, sample_file, b2_uri_args):
    file_data = read_file(sample_file)
    cut = 12345
    cut_printable = '1970-01-01  00:00:12'
    args = [
        'upload-file',
        '--no-progress',
        '--custom-upload-time',
        str(cut),
        '--quiet',
        bucket_name,
        sample_file,
        'a',
    ]
    succeeded, stdout = b2_tool.run_command(args)
    if not succeeded:
        b2_tool.should_fail(args, 'custom_timestamp_not_allowed')
    else:
        # file_id, action, date, time, size(, replication), name
        b2_tool.should_succeed(
            ['ls', '--long', *b2_uri_args(bucket_name)], '^4_z.*  upload  {} +{}  a'.format(
                cut_printable,
                len(file_data),
            )
        )
        # file_id, action, date, time, size(, replication), name
        b2_tool.should_succeed(
            ['ls', '--long', '--replication', *b2_uri_args(bucket_name)],
            f'^4_z.*  upload  {cut_printable} +{len(file_data)}  -  a'
        )


@skip_on_windows
def test_upload_file__stdin_pipe_operator(request, bash_runner, b2_tool, bucket_name):
    """Test upload-file from stdin using pipe operator."""
    content = request.node.name
    run = bash_runner(
        f'echo -n {content!r} '
        f'| '
        f'{" ".join(b2_tool.parse_command(b2_tool.prepare_env()))} upload-file {bucket_name} - {request.node.name}.txt'
    )
    assert hashlib.sha1(content.encode()).hexdigest() in run.stdout


@skip_on_windows
def test_upload_unbound_stream__redirect_operator(
    request, bash_runner, b2_tool, bucket_name, is_running_on_docker
):
    """Test upload-unbound-stream from stdin using redirect operator."""
    if is_running_on_docker:
        pytest.skip('Not supported on Docker')
    content = request.node.name
    command = request.config.getoption('--sut')
    run = bash_runner(
        f'{command} upload-unbound-stream {bucket_name} <(echo -n {content}) {request.node.name}.txt'
    )
    assert hashlib.sha1(content.encode()).hexdigest() in run.stdout


def test_download_file_stdout(
    b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file
):
    assert b2_tool.should_succeed(
        ['download-file', '--quiet', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}", '-'],
    ) == sample_filepath.read_text()
    assert b2_tool.should_succeed(
        ['download-file', '--quiet', f"b2id://{uploaded_sample_file['fileId']}", '-'],
    ) == sample_filepath.read_text()


def test_download_file_to_directory(
    b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file
):
    downloads_directory = 'downloads'
    target_directory = tmp_path / downloads_directory
    target_directory.mkdir()
    filename_as_path = pathlib.Path(uploaded_sample_file['fileName'])

    sample_file_content = sample_filepath.read_text()
    b2_tool.should_succeed(
        [
            'download-file',
            '--quiet',
            f"b2://{bucket_name}/{uploaded_sample_file['fileName']}",
            str(target_directory),
        ],
    )
    downloaded_file = target_directory / filename_as_path
    assert downloaded_file.read_text() == sample_file_content, \
        f'{downloaded_file}, {downloaded_file.read_text()}, {sample_file_content}'

    b2_tool.should_succeed(
        [
            'download-file',
            '--quiet',
            f"b2id://{uploaded_sample_file['fileId']}",
            str(target_directory),
        ],
    )
    # A second file should be created.
    new_files = [
        filepath
        for filepath in target_directory.glob(f'{filename_as_path.stem}*{filename_as_path.suffix}')
        if filepath.name != filename_as_path.name
    ]
    assert len(new_files) == 1, f'{new_files}'
    assert new_files[0].read_text() == sample_file_content, \
        f'{new_files}, {new_files[0].read_text()}, {sample_file_content}'


def test_cat(b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file):
    assert b2_tool.should_succeed(
        ['cat', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}"],
    ) == sample_filepath.read_text()
    assert b2_tool.should_succeed(['cat', f"b2id://{uploaded_sample_file['fileId']}"]
                                 ) == sample_filepath.read_text()


def test_header_arguments(b2_tool, bucket_name, sample_filepath, tmp_path):
    # yapf: disable
    args = [
        '--cache-control', 'max-age=3600',
        '--content-disposition', 'attachment',
        '--content-encoding', 'gzip',
        '--content-language', 'en',
        '--expires', 'Thu, 01 Dec 2050 16:00:00 GMT',
    ]
    # yapf: enable
    expected_file_info = {
        'b2-cache-control': 'max-age=3600',
        'b2-content-disposition': 'attachment',
        'b2-content-encoding': 'gzip',
        'b2-content-language': 'en',
        'b2-expires': 'Thu, 01 Dec 2050 16:00:00 GMT',
    }

    def assert_expected(file_info, expected=expected_file_info):
        for key, val in expected.items():
            assert file_info[key] == val

    status, stdout, stderr = b2_tool.execute(
        [
            'upload-file',
            '--quiet',
            '--no-progress',
            bucket_name,
            str(sample_filepath),
            'sample_file',
            *args,
            '--info',
            'b2-content-disposition=will-be-overwritten',
        ]
    )
    assert status == 0
    file_version = json.loads(stdout)
    assert_expected(file_version['fileInfo'])

    # Since we used both --info and --content-disposition to set b2-content-disposition,
    # a warning should be emitted
    assert 'will be overwritten' in stderr and 'b2-content-disposition = attachment' in stderr

    copied_version = b2_tool.should_succeed_json(
        [
            'copy-file-by-id', '--quiet', *args, '--content-type', 'text/plain',
            file_version['fileId'], bucket_name, 'copied_file'
        ]
    )
    assert_expected(copied_version['fileInfo'])

    download_output = b2_tool.should_succeed(
        ['download-file', f"b2id://{file_version['fileId']}", tmp_path / 'downloaded_file']
    )
    assert re.search(r'CacheControl: *max-age=3600', download_output)
    assert re.search(r'ContentDisposition: *attachment', download_output)
    assert re.search(r'ContentEncoding: *gzip', download_output)
    assert re.search(r'ContentLanguage: *en', download_output)
    assert re.search(r'Expires: *Thu, 01 Dec 2050 16:00:00 GMT', download_output)


def test_notification_rules(b2_tool, bucket_name):
    auth_dict = b2_tool.should_succeed_json(['get-account-info'])
    if 'writeBucketNotifications' not in auth_dict['allowed']['capabilities']:
        pytest.skip('Test account does not have writeBucketNotifications capability')

    private_preview_pattern = re.compile(r'FeaturePreviewWarning')
    assert b2_tool.should_succeed_json(
        ["notification-rules", "list", f"b2://{bucket_name}", "--json"],
        expected_stderr_pattern=private_preview_pattern
    ) == []

    notification_rule = {
        "eventTypes": ["b2:ObjectCreated:*"],
        "isEnabled": True,
        "name": "test-rule",
        "objectNamePrefix": "",
        "targetConfiguration":
            {
                "customHeaders": None,
                "hmacSha256SigningSecret": None,
                "targetType": "webhook",
                "url": "https://example.com/webhook",
            }
    }
    # add rule
    created_rule = b2_tool.should_succeed_json(
        [
            "notification-rules",
            "create",
            "--json",
            f"b2://{bucket_name}",
            "test-rule",
            "--webhook-url",
            "https://example.com/webhook",
            "--event-type",
            "b2:ObjectCreated:*",
        ],
        expected_stderr_pattern=private_preview_pattern
    )
    expected_rules = [{**notification_rule, "isSuspended": False, "suspensionReason": ""}]
    assert created_rule == expected_rules[0]

    # modify rule
    secret = "0testSecret000000000000000000032"
    modified_rule = b2_tool.should_succeed_json(
        [
            "notification-rules",
            "update",
            "--json",
            f"b2://{bucket_name}/prefix",
            "test-rule",
            "--disable",
            "--sign-secret",
            secret,
        ],
        expected_stderr_pattern=private_preview_pattern
    )
    expected_rules[0].update({"objectNamePrefix": "prefix", "isEnabled": False})
    expected_rules[0]["targetConfiguration"]["hmacSha256SigningSecret"] = secret
    assert modified_rule == expected_rules[0]

    # read updated rules
    assert b2_tool.should_succeed_json(
        ["notification-rules", "list", f"b2://{bucket_name}", "--json"],
        expected_stderr_pattern=private_preview_pattern
    ) == expected_rules

    # delete rule by name
    assert b2_tool.should_succeed(
        ["notification-rules", "delete", f"b2://{bucket_name}", "test-rule"],
        expected_stderr_pattern=private_preview_pattern
    ) == f"Rule 'test-rule' has been deleted from b2://{bucket_name}/\n"
    assert b2_tool.should_succeed_json(
        ["notification-rules", "list", f"b2://{bucket_name}", "--json"],
        expected_stderr_pattern=private_preview_pattern
    ) == []
