# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import os

from awscrt.s3 import S3RequestType

from awscli.compat import BytesIO
from awscli.customizations.s3.utils import relative_path
from awscli.testutils import cd, mock, skip_if_case_sensitive, skip_if_windows
from tests.functional.s3 import (
    BaseCRTTransferClientTest,
    BaseS3CLIRunnerTest,
    BaseS3TransferCommandTest,
)


class TestSyncCommand(BaseS3TransferCommandTest):
    prefix = 's3 sync '

    def test_website_redirect_ignore_paramfile(self):
        full_path = self.files.create_file('foo.txt', 'mycontent')
        cmdline = '%s %s s3://bucket/key.txt --website-redirect %s' % (
            self.prefix,
            self.files.rootdir,
            'http://someserver',
        )
        self.parsed_responses = [
            {"CommonPrefixes": [], "Contents": []},
            {'ETag': '"c8afdb36c52cf4727836669019e69222"'},
        ]
        self.run_cmd(cmdline, expected_rc=0)

        # The only operations we should have called are ListObjectsV2/PutObject.
        self.assertEqual(
            len(self.operations_called), 2, self.operations_called
        )
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
        # Make sure that the specified web address is used as opposed to the
        # contents of the web address when uploading the object
        self.assertEqual(
            self.operations_called[1][1]['WebsiteRedirectLocation'],
            'http://someserver',
        )

    def test_no_recursive_option(self):
        cmdline = '. s3://mybucket --recursive'
        # Return code will be 252 for invalid parameter ``--recursive``
        self.run_cmd(cmdline, expected_rc=252)

    def test_sync_from_non_existant_directory(self):
        non_existant_directory = os.path.join(self.files.rootdir, 'fakedir')
        cmdline = '%s %s s3://bucket/' % (self.prefix, non_existant_directory)
        self.parsed_responses = [{"CommonPrefixes": [], "Contents": []}]
        _, stderr, _ = self.run_cmd(cmdline, expected_rc=255)
        self.assertIn('does not exist', stderr)

    def test_sync_to_non_existant_directory(self):
        key = 'foo.txt'
        non_existant_directory = os.path.join(self.files.rootdir, 'fakedir')
        cmdline = '%s s3://bucket/ %s' % (self.prefix, non_existant_directory)
        self.parsed_responses = [
            {
                "CommonPrefixes": [],
                "Contents": [
                    {
                        "Key": key,
                        "Size": 3,
                        "LastModified": "2014-01-09T20:45:49.000Z",
                        "ETag": '"c8afdb36c52cf4727836669019e69222-"',
                    }
                ],
            },
            {
                'ETag': '"c8afdb36c52cf4727836669019e69222-"',
                'Body': BytesIO(b'foo'),
            },
        ]
        self.run_cmd(cmdline, expected_rc=0)
        # Make sure the file now exists.
        self.assertTrue(
            os.path.exists(os.path.join(non_existant_directory, key))
        )

    def test_dryrun_sync(self):
        self.parsed_responses = [
            self.list_objects_response([]),
        ]
        full_path = self.files.create_file('file.txt', 'mycontent')
        cmdline = f'{self.prefix} {self.files.rootdir} s3://bucket/ --dryrun'
        stdout, _, _ = self.run_cmd(cmdline, expected_rc=0)
        self.assert_operations_called(
            [
                (
                    'ListObjectsV2',
                    {
                        'Bucket': 'bucket',
                        'Prefix': '',
                    },
                ),
            ]
        )
        self.assertIn(
            f'(dryrun) upload: {relative_path(full_path)} to '
            f's3://bucket/file.txt',
            stdout,
        )

    def test_glacier_sync_with_force_glacier(self):
        self.parsed_responses = [
            {
                'Contents': [
                    {
                        'Key': 'foo/bar.txt',
                        'ContentLength': '100',
                        'LastModified': '00:00:00Z',
                        'StorageClass': 'GLACIER',
                        'Size': 100,
                        'ETag': '"foo-1"',
                    },
                ],
                'CommonPrefixes': [],
            },
            {'ETag': '"foo-1"', 'Body': BytesIO(b'foo')},
        ]
        cmdline = '%s s3://bucket/foo %s --force-glacier-transfer' % (
            self.prefix,
            self.files.rootdir,
        )
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(
            len(self.operations_called), 2, self.operations_called
        )
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'GetObject')

    def test_handles_glacier_incompatible_operations(self):
        self.parsed_responses = [
            {
                'Contents': [
                    {
                        'Key': 'foo',
                        'Size': 100,
                        'LastModified': '00:00:00Z',
                        'StorageClass': 'GLACIER',
                    },
                    {
                        'Key': 'bar',
                        'Size': 100,
                        'LastModified': '00:00:00Z',
                        'StorageClass': 'DEEP_ARCHIVE',
                    },
                ]
            }
        ]
        cmdline = '%s s3://bucket/ %s' % (self.prefix, self.files.rootdir)
        _, stderr, _ = self.run_cmd(cmdline, expected_rc=2)
        # There should not have been a download attempted because the
        # operation was skipped because it is glacier and glacier
        # deep archive incompatible.
        self.assertEqual(len(self.operations_called), 1)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertIn('GLACIER', stderr)
        self.assertIn('s3://bucket/foo', stderr)
        self.assertIn('s3://bucket/bar', stderr)

    def test_turn_off_glacier_warnings(self):
        self.parsed_responses = [
            {
                'Contents': [
                    {
                        'Key': 'foo',
                        'Size': 100,
                        'LastModified': '00:00:00Z',
                        'StorageClass': 'GLACIER',
                    },
                    {
                        'Key': 'bar',
                        'Size': 100,
                        'LastModified': '00:00:00Z',
                        'StorageClass': 'DEEP_ARCHIVE',
                    },
                ]
            }
        ]
        cmdline = '%s s3://bucket/ %s --ignore-glacier-warnings' % (
            self.prefix,
            self.files.rootdir,
        )
        _, stderr, _ = self.run_cmd(cmdline, expected_rc=0)
        # There should not have been a download attempted because the
        # operation was skipped because it is glacier incompatible.
        self.assertEqual(len(self.operations_called), 1)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual('', stderr)

    def test_warning_on_invalid_timestamp(self):
        full_path = self.files.create_file('foo.txt', 'mycontent')

        cmdline = '%s %s s3://bucket/key.txt' % (
            self.prefix,
            self.files.rootdir,
        )
        self.parsed_responses = [
            {"CommonPrefixes": [], "Contents": []},
            {'ETag': '"c8afdb36c52cf4727836669019e69222"'},
        ]
        # Patch get_file_stat to return a value indicating that an invalid
        # timestamp was loaded. It is impossible to set an invalid timestamp
        # on all OSes so it has to be patched.
        # TODO: find another method to test this behavior without patching.
        with mock.patch(
            'awscli.customizations.s3.filegenerator.get_file_stat',
            return_value=(None, None),
        ):
            self.run_cmd(cmdline, expected_rc=2)

        # We should still have put the object
        self.assertEqual(
            len(self.operations_called), 2, self.operations_called
        )
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')

    def test_sync_with_delete_on_downloads(self):
        full_path = self.files.create_file('foo.txt', 'mycontent')
        cmdline = '%s s3://bucket %s --delete' % (
            self.prefix,
            self.files.rootdir,
        )
        self.parsed_responses = [
            {"CommonPrefixes": [], "Contents": []},
            {'ETag': '"c8afdb36c52cf4727836669019e69222"'},
        ]
        self.run_cmd(cmdline, expected_rc=0)

        # The only operations we should have called are ListObjectsV2.
        self.assertEqual(
            len(self.operations_called), 1, self.operations_called
        )
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')

        self.assertFalse(os.path.exists(full_path))

    # When a file has been deleted after listing,
    # awscli.customizations.s3.utils.get_file_stat may raise either some kind
    # of OSError, or a ValueError, depending on the environment. In both cases,
    # the behaviour should be the same: skip the file and emit a warning.
    #
    # This test covers the case where a ValueError is emitted.
    def test_sync_skips_over_files_deleted_between_listing_and_transfer_valueerror(
        self,
    ):
        full_path = self.files.create_file('foo.txt', 'mycontent')
        cmdline = '%s %s s3://bucket/' % (self.prefix, self.files.rootdir)

        # FileGenerator.list_files should skip over files that cause an
        # IOError to be raised because they are missing when we try to
        # get their stats. This IOError is translated to a ValueError in
        # awscli.customizations.s3.utils.get_file_stat.
        def side_effect(_):
            os.remove(full_path)
            raise ValueError()

        with mock.patch(
            'awscli.customizations.s3.filegenerator.get_file_stat',
            side_effect=side_effect,
        ):
            self.run_cmd(cmdline, expected_rc=2)

        # We should not call PutObject because the file was deleted
        # before we could transfer it
        self.assertEqual(
            len(self.operations_called), 1, self.operations_called
        )
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')

    # This test covers the case where an OSError is emitted.
    def test_sync_skips_over_files_deleted_between_listing_and_transfer_oserror(
        self,
    ):
        full_path = self.files.create_file('foo.txt', 'mycontent')
        cmdline = '%s %s s3://bucket/' % (self.prefix, self.files.rootdir)

        # FileGenerator.list_files should skip over files that cause an
        # OSError to be raised because they are missing when we try to
        # get their stats.
        def side_effect(_):
            os.remove(full_path)
            raise OSError()

        with mock.patch(
            'awscli.customizations.s3.filegenerator.get_file_stat',
            side_effect=side_effect,
        ):
            self.run_cmd(cmdline, expected_rc=2)

        # We should not call PutObject because the file was deleted
        # before we could transfer it
        self.assertEqual(
            len(self.operations_called), 1, self.operations_called
        )
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')

    def test_request_payer(self):
        cmdline = '%s s3://sourcebucket/ s3://mybucket --request-payer' % (
            self.prefix
        )
        self.parsed_responses = [
            # Response for ListObjects on source bucket
            self.list_objects_response(['mykey']),
            # Response for ListObjects on destination bucket
            self.list_objects_response([]),
            self.copy_object_response(),
        ]
        self.run_cmd(cmdline, expected_rc=0)
        self.assert_operations_called(
            [
                self.list_objects_request(
                    'sourcebucket', RequestPayer='requester'
                ),
                self.list_objects_request(
                    'mybucket', RequestPayer='requester'
                ),
                self.copy_object_request(
                    'sourcebucket',
                    'mykey',
                    'mybucket',
                    'mykey',
                    RequestPayer='requester',
                ),
            ]
        )

    def test_s3s3_sync_with_destination_sse_c(self):
        cmdline = (
            '%s s3://sourcebucket/ s3://mybucket '
            '--sse-c AES256 --sse-c-key destination-key' % self.prefix
        )
        self.parsed_responses = [
            self.list_objects_response(['mykey']),
            self.list_objects_response([]),
            self.copy_object_response(),
        ]
        self.run_cmd(cmdline, expected_rc=0)
        self.assert_operations_called(
            [
                self.list_objects_request('sourcebucket'),
                self.list_objects_request('mybucket'),
                self.copy_object_request(
                    'sourcebucket',
                    'mykey',
                    'mybucket',
                    'mykey',
                    SSECustomerAlgorithm='AES256',
                    SSECustomerKey='destination-key',
                ),
            ]
        )

    def test_s3s3_sync_with_destination_sse_c_multipart(self):
        mp_threshold = 8 * 1024 * 1024
        cmdline = (
            '%s s3://sourcebucket/ s3://mybucket '
            '--sse-c AES256 --sse-c-key destination-key' % self.prefix
        )
        self.parsed_responses = [
            self.list_objects_response(['mykey'], Size=mp_threshold),
            self.list_objects_response([]),
            # HeadObject from SetMetadataDirectivePropsSubscriber
            self.head_object_response(ContentLength=mp_threshold),
            self.get_object_tagging_response({}),
        ] + self.mp_copy_responses()
        self.run_cmd(cmdline, expected_rc=0)
        self.assert_operations_called(
            [
                self.list_objects_request('sourcebucket'),
                self.list_objects_request('mybucket'),
                self.head_object_request(
                    'sourcebucket',
                    'mykey',
                    # no SSE-C params — source is unencrypted
                ),
                self.get_object_tagging_request('sourcebucket', 'mykey'),
                self.create_mpu_request(
                    'mybucket',
                    'mykey',
                    SSECustomerAlgorithm='AES256',
                    SSECustomerKey='destination-key',
                ),
                self.upload_part_copy_request(
                    'sourcebucket',
                    'mykey',
                    'mybucket',
                    'mykey',
                    'upload_id',
                    PartNumber=mock.ANY,
                    CopySourceRange=mock.ANY,
                    CopySourceIfMatch=mock.ANY,
                    SSECustomerAlgorithm='AES256',
                    SSECustomerKey='destination-key',
                ),
                self.complete_mpu_request(
                    'mybucket',
                    'mykey',
                    'upload_id',
                    num_parts=1,
                    SSECustomerAlgorithm='AES256',
                    SSECustomerKey='destination-key',
                ),
            ]
        )

    def test_s3s3_sync_with_different_sse_c_keys(self):
        cmdline = (
            '%s s3://sourcebucket/ s3://mybucket '
            '--sse-c-copy-source AES256 --sse-c-copy-source-key source-key '
            '--sse-c AES256 --sse-c-key destination-key' % self.prefix
        )
        self.parsed_responses = [
            self.list_objects_response(['mykey']),
            self.list_objects_response([]),
            self.copy_object_response(),
        ]
        self.run_cmd(cmdline, expected_rc=0)
        self.assert_operations_called(
            [
                self.list_objects_request('sourcebucket'),
                self.list_objects_request('mybucket'),
                self.copy_object_request(
                    'sourcebucket',
                    'mykey',
                    'mybucket',
                    'mykey',
                    SSECustomerAlgorithm='AES256',
                    SSECustomerKey='destination-key',
                    CopySourceSSECustomerAlgorithm='AES256',
                    CopySourceSSECustomerKey='source-key',
                ),
            ]
        )

    def test_s3s3_sync_with_different_sse_c_keys_multipart(self):
        mp_threshold = 8 * 1024 * 1024
        cmdline = (
            '%s s3://sourcebucket/ s3://mybucket '
            '--sse-c-copy-source AES256 --sse-c-copy-source-key source-key '
            '--sse-c AES256 --sse-c-key destination-key' % self.prefix
        )
        self.parsed_responses = [
            self.list_objects_response(['mykey'], Size=mp_threshold),
            self.list_objects_response([]),
            # HeadObject from SetMetadataDirectivePropsSubscriber
            self.head_object_response(ContentLength=mp_threshold),
            self.get_object_tagging_response({}),
        ] + self.mp_copy_responses()
        self.run_cmd(cmdline, expected_rc=0)
        self.assert_operations_called(
            [
                self.list_objects_request('sourcebucket'),
                self.list_objects_request('mybucket'),
                self.head_object_request(
                    'sourcebucket',
                    'mykey',
                    SSECustomerAlgorithm='AES256',
                    SSECustomerKey='source-key',
                ),
                self.get_object_tagging_request('sourcebucket', 'mykey'),
                self.create_mpu_request(
                    'mybucket',
                    'mykey',
                    SSECustomerAlgorithm='AES256',
                    SSECustomerKey='destination-key',
                ),
                self.upload_part_copy_request(
                    'sourcebucket',
                    'mykey',
                    'mybucket',
                    'mykey',
                    'upload_id',
                    PartNumber=mock.ANY,
                    CopySourceRange=mock.ANY,
                    CopySourceIfMatch=mock.ANY,
                    SSECustomerAlgorithm='AES256',
                    SSECustomerKey='destination-key',
                    CopySourceSSECustomerAlgorithm='AES256',
                    CopySourceSSECustomerKey='source-key',
                ),
                self.complete_mpu_request(
                    'mybucket',
                    'mykey',
                    'upload_id',
                    num_parts=1,
                    SSECustomerAlgorithm='AES256',
                    SSECustomerKey='destination-key',
                ),
            ]
        )

    def test_request_payer_with_deletes(self):
        cmdline = '%s s3://sourcebucket/ s3://mybucket' % self.prefix
        cmdline += ' --request-payer'
        cmdline += ' --delete'
        self.parsed_responses = [
            # Response for ListObjects on source bucket
            self.list_objects_response([]),
            # Response for ListObjects on destination bucket
            self.list_objects_response(['key-to-delete']),
            self.delete_object_response(),
        ]
        self.run_cmd(cmdline, expected_rc=0)
        self.assert_operations_called(
            [
                self.list_objects_request(
                    'sourcebucket', RequestPayer='requester'
                ),
                self.list_objects_request(
                    'mybucket', RequestPayer='requester'
                ),
                self.delete_object_request(
                    'mybucket', 'key-to-delete', RequestPayer='requester'
                ),
            ]
        )

    def test_with_accesspoint_arn(self):
        accesspoint_arn = (
            'arn:aws:s3:us-west-2:123456789012:accesspoint/endpoint'
        )
        cmdline = self.prefix
        cmdline += 's3://%s' % accesspoint_arn
        cmdline += ' %s' % self.files.rootdir
        self.parsed_responses = [
            self.list_objects_response(['mykey']),
            self.get_object_response(),
        ]
        self.run_cmd(cmdline, expected_rc=0)
        self.assert_operations_called(
            [
                self.list_objects_request(accesspoint_arn),
                self.get_object_request(accesspoint_arn, 'mykey'),
            ]
        )

    def test_with_copy_props(self):
        cmdline = self.prefix
        cmdline += 's3://sourcebucket/ s3://bucket/'
        cmdline += ' --copy-props default'

        upload_id = 'upload_id'
        large_tag_set = {'tag-key': 'val' * 3000}
        metadata = {'tag-key': 'tag-value'}
        self.parsed_responses = [
            self.list_objects_response(keys=['key'], Size=8 * 1024**2),
            self.list_objects_response(keys=[]),
            self.head_object_response(
                Metadata=metadata,
            ),
            self.get_object_tagging_response(large_tag_set),
            self.create_mpu_response(upload_id),
            self.upload_part_copy_response(),
            self.complete_mpu_response(),
            self.put_object_tagging_response(),
        ]
        self.run_cmd(cmdline, expected_rc=0)
        self.assert_operations_called(
            [
                self.list_objects_request('sourcebucket'),
                self.list_objects_request('bucket'),
                self.head_object_request('sourcebucket', 'key'),
                self.get_object_tagging_request('sourcebucket', 'key'),
                self.create_mpu_request('bucket', 'key', Metadata=metadata),
                self.upload_part_copy_request(
                    'sourcebucket',
                    'key',
                    'bucket',
                    'key',
                    upload_id,
                    CopySourceRange=mock.ANY,
                    PartNumber=1,
                    CopySourceIfMatch='"foo-1"',
                ),
                self.complete_mpu_request('bucket', 'key', upload_id, 1),
                self.put_object_tagging_request(
                    'bucket', 'key', large_tag_set
                ),
            ]
        )

    def test_upload_with_checksum_algorithm_sha1(self):
        self.files.create_file('foo.txt', 'contents')
        cmdline = f'{self.prefix} {self.files.rootdir} s3://bucket/ --checksum-algorithm SHA1'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
        self.assertEqual(
            self.operations_called[1][1]['ChecksumAlgorithm'], 'SHA1'
        )

    def test_copy_with_checksum_algorithm_update_sha1(self):
        cmdline = f'{self.prefix} s3://src-bucket/ s3://dest-bucket/ --checksum-algorithm SHA1'
        self.parsed_responses = [
            # Response for ListObjects on source bucket
            {
                'Contents': [
                    {
                        'Key': 'mykey',
                        'LastModified': '00:00:00Z',
                        'Size': 100,
                        'ChecksumAlgorithm': 'SHA1',
                        'ETag': 'myetag',
                    }
                ],
                'CommonPrefixes': [],
            },
            # Response for ListObjects on destination bucket
            self.list_objects_response([]),
            # Response for CopyObject
            {'ChecksumSHA1': 'sha1-checksum'},
        ]
        self.run_cmd(cmdline, expected_rc=0)
        self.assert_operations_called(
            [
                self.list_objects_request('src-bucket'),
                self.list_objects_request('dest-bucket'),
                (
                    'CopyObject',
                    {
                        'CopySource': {'Bucket': 'src-bucket', 'Key': 'mykey'},
                        'Bucket': 'dest-bucket',
                        'Key': 'mykey',
                        'ChecksumAlgorithm': 'SHA1',
                    },
                ),
            ]
        )

    def test_upload_with_checksum_algorithm_sha256(self):
        self.files.create_file('foo.txt', 'contents')
        cmdline = f'{self.prefix} {self.files.rootdir} s3://bucket/ --checksum-algorithm SHA256'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
        self.assertEqual(
            self.operations_called[1][1]['ChecksumAlgorithm'], 'SHA256'
        )

    def test_upload_with_checksum_algorithm_sha512(self):
        self.files.create_file('foo.txt', 'contents')
        cmdline = f'{self.prefix} {self.files.rootdir} s3://bucket/ --checksum-algorithm SHA512'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
        self.assertEqual(
            self.operations_called[1][1]['ChecksumAlgorithm'], 'SHA512'
        )

    def test_upload_with_checksum_algorithm_crc32(self):
        self.files.create_file('foo.txt', 'contents')
        cmdline = f'{self.prefix} {self.files.rootdir} s3://bucket/ --checksum-algorithm CRC32'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
        self.assertEqual(
            self.operations_called[1][1]['ChecksumAlgorithm'], 'CRC32'
        )

    def test_upload_with_checksum_algorithm_crc32c(self):
        self.files.create_file('foo.txt', 'contents')
        cmdline = f'{self.prefix} {self.files.rootdir} s3://bucket/ --checksum-algorithm CRC32C'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
        self.assertEqual(
            self.operations_called[1][1]['ChecksumAlgorithm'], 'CRC32C'
        )

    def test_upload_with_checksum_algorithm_crc64nvme(self):
        self.files.create_file('foo.txt', 'contents')
        cmdline = f'{self.prefix} {self.files.rootdir} s3://bucket/ --checksum-algorithm CRC64NVME'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
        self.assertEqual(
            self.operations_called[1][1]['ChecksumAlgorithm'], 'CRC64NVME'
        )

    def test_upload_with_checksum_algorithm_xxhash3(self):
        self.files.create_file('foo.txt', 'contents')
        cmdline = f'{self.prefix} {self.files.rootdir} s3://bucket/ --checksum-algorithm XXHASH3'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
        self.assertEqual(
            self.operations_called[1][1]['ChecksumAlgorithm'], 'XXHASH3'
        )

    def test_upload_with_checksum_algorithm_xxhash64(self):
        self.files.create_file('foo.txt', 'contents')
        cmdline = f'{self.prefix} {self.files.rootdir} s3://bucket/ --checksum-algorithm XXHASH64'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
        self.assertEqual(
            self.operations_called[1][1]['ChecksumAlgorithm'], 'XXHASH64'
        )

    def test_upload_with_checksum_algorithm_xxhash128(self):
        self.files.create_file('foo.txt', 'contents')
        cmdline = f'{self.prefix} {self.files.rootdir} s3://bucket/ --checksum-algorithm XXHASH128'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
        self.assertEqual(
            self.operations_called[1][1]['ChecksumAlgorithm'], 'XXHASH128'
        )

    def test_download_with_checksum_mode_sha1(self):
        self.parsed_responses = [
            self.list_objects_response(['bucket']),
            # Mocked GetObject response with a checksum algorithm specified
            {
                'ETag': 'foo-1',
                'ChecksumSHA1': 'checksum',
                'Body': BytesIO(b'foo'),
            },
        ]
        cmdline = f'{self.prefix} s3://bucket/foo {self.files.rootdir} --checksum-mode ENABLED'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'GetObject')
        self.assertIn(
            ('ChecksumMode', 'ENABLED'), self.operations_called[1][1].items()
        )

    def test_download_with_checksum_mode_sha256(self):
        self.parsed_responses = [
            self.list_objects_response(['bucket']),
            # Mocked GetObject response with a checksum algorithm specified
            {
                'ETag': 'foo-1',
                'ChecksumSHA256': 'checksum',
                'Body': BytesIO(b'foo'),
            },
        ]
        cmdline = f'{self.prefix} s3://bucket/foo {self.files.rootdir} --checksum-mode ENABLED'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'GetObject')
        self.assertIn(
            ('ChecksumMode', 'ENABLED'), self.operations_called[1][1].items()
        )

    def test_download_with_checksum_mode_sha512(self):
        self.parsed_responses = [
            self.list_objects_response(['bucket']),
            # Mocked GetObject response with a checksum algorithm specified
            {
                'ETag': 'foo-1',
                'ChecksumSHA512': 'checksum',
                'Body': BytesIO(b'foo'),
            },
        ]
        cmdline = f'{self.prefix} s3://bucket/foo {self.files.rootdir} --checksum-mode ENABLED'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'GetObject')
        self.assertIn(
            ('ChecksumMode', 'ENABLED'), self.operations_called[1][1].items()
        )

    def test_download_with_checksum_mode_crc32(self):
        self.parsed_responses = [
            self.list_objects_response(['bucket']),
            # Mocked GetObject response with a checksum algorithm specified
            {
                'ETag': 'foo-1',
                'ChecksumCRC32': 'checksum',
                'Body': BytesIO(b'foo'),
            },
        ]
        cmdline = f'{self.prefix} s3://bucket/foo {self.files.rootdir} --checksum-mode ENABLED'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'GetObject')
        self.assertIn(
            ('ChecksumMode', 'ENABLED'), self.operations_called[1][1].items()
        )

    def test_download_with_checksum_mode_crc32c(self):
        self.parsed_responses = [
            self.list_objects_response(['bucket']),
            # Mocked GetObject response with a checksum algorithm specified
            {
                'ETag': 'foo-1',
                'ChecksumCRC32C': 'checksum',
                'Body': BytesIO(b'foo'),
            },
        ]
        cmdline = f'{self.prefix} s3://bucket/foo {self.files.rootdir} --checksum-mode ENABLED'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'GetObject')
        self.assertIn(
            ('ChecksumMode', 'ENABLED'), self.operations_called[1][1].items()
        )

    def test_download_with_checksum_mode_crc64nvme(self):
        self.parsed_responses = [
            self.list_objects_response(['bucket']),
            # Mocked GetObject response with a checksum algorithm specified
            {
                'ETag': 'foo-1',
                'ChecksumCRC64NVME': 'checksum',
                'Body': BytesIO(b'foo'),
            },
        ]
        cmdline = f'{self.prefix} s3://bucket/foo {self.files.rootdir} --checksum-mode ENABLED'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'GetObject')
        self.assertIn(
            ('ChecksumMode', 'ENABLED'), self.operations_called[1][1].items()
        )

    def test_download_with_checksum_mode_xxhash3(self):
        self.parsed_responses = [
            self.list_objects_response(['bucket']),
            # Mocked GetObject response with a checksum algorithm specified
            {
                'ETag': 'foo-1',
                'ChecksumXXHASH3': 'checksum',
                'Body': BytesIO(b'foo'),
            },
        ]
        cmdline = f'{self.prefix} s3://bucket/foo {self.files.rootdir} --checksum-mode ENABLED'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'GetObject')
        self.assertIn(
            ('ChecksumMode', 'ENABLED'), self.operations_called[1][1].items()
        )

    def test_download_with_checksum_mode_xxhash64(self):
        self.parsed_responses = [
            self.list_objects_response(['bucket']),
            # Mocked GetObject response with a checksum algorithm specified
            {
                'ETag': 'foo-1',
                'ChecksumXXHASH64': 'checksum',
                'Body': BytesIO(b'foo'),
            },
        ]
        cmdline = f'{self.prefix} s3://bucket/foo {self.files.rootdir} --checksum-mode ENABLED'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'GetObject')
        self.assertIn(
            ('ChecksumMode', 'ENABLED'), self.operations_called[1][1].items()
        )

    def test_download_with_checksum_mode_xxhash128(self):
        self.parsed_responses = [
            self.list_objects_response(['bucket']),
            # Mocked GetObject response with a checksum algorithm specified
            {
                'ETag': 'foo-1',
                'ChecksumXXHASH128': 'checksum',
                'Body': BytesIO(b'foo'),
            },
        ]
        cmdline = f'{self.prefix} s3://bucket/foo {self.files.rootdir} --checksum-mode ENABLED'
        self.run_cmd(cmdline, expected_rc=0)

        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'GetObject')
        self.assertIn(
            ('ChecksumMode', 'ENABLED'), self.operations_called[1][1].items()
        )

    def test_sync_upload_with_no_overwrite_when_file_does_not_exist_at_destination(
        self,
    ):
        self.files.create_file("new_file.txt", "mycontent")
        self.parsed_responses = [
            self.list_objects_response(['file.txt']),
            {'ETag': '"c8afdb36c52cf4727836669019e69222"'},
        ]
        cmdline = (
            f'{self.prefix} {self.files.rootdir} s3://bucket --no-overwrite'
        )
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(len(self.operations_called), 2)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'PutObject')
        self.assertEqual(self.operations_called[1][1]['Key'], 'new_file.txt')

    def test_sync_upload_with_no_overwrite_when_file_exists_at_destination(
        self,
    ):
        self.files.create_file("new_file.txt", "mycontent")
        self.parsed_responses = [
            self.list_objects_response(['new_file.txt']),
        ]
        cmdline = (
            f'{self.prefix} {self.files.rootdir} s3://bucket --no-overwrite'
        )
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(len(self.operations_called), 1)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')

    def test_sync_download_with_no_overwrite_file_not_exists_at_destination(
        self,
    ):
        self.parsed_responses = [
            self.list_objects_response(['new_file.txt']),
            self.get_object_response(),
        ]
        cmdline = (
            f'{self.prefix} s3://bucket/ {self.files.rootdir} --no-overwrite'
        )
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(len(self.operations_called), 2)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'GetObject')
        local_file_path = os.path.join(self.files.rootdir, 'new_file.txt')
        self.assertTrue(os.path.exists(local_file_path))

    def test_sync_download_with_no_overwrite_file_exists_at_destination(self):
        self.files.create_file('file.txt', 'My content')
        self.parsed_responses = [
            self.list_objects_response(['file.txt']),
        ]
        cmdline = (
            f'{self.prefix} s3://bucket/ {self.files.rootdir} --no-overwrite'
        )
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(len(self.operations_called), 1)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')

    def test_sync_copy_with_no_overwrite_file_not_exists_at_destination(self):
        self.parsed_responses = [
            self.list_objects_response(['new_file.txt']),
            self.list_objects_response(['file1.txt']),
            self.copy_object_response(),
        ]
        cmdline = f'{self.prefix} s3://bucket/ s3://bucket2/ --no-overwrite'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(len(self.operations_called), 3)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[2][0].name, 'CopyObject')
        self.assertEqual(self.operations_called[2][1]['Key'], 'new_file.txt')

    def test_sync_copy_with_no_overwrite_file_exists_at_destination(self):
        self.parsed_responses = [
            self.list_objects_response(['new_file.txt']),
            self.list_objects_response(['new_file.txt', 'file1.txt']),
        ]
        cmdline = f'{self.prefix} s3://bucket/ s3://bucket2/ --no-overwrite'
        self.run_cmd(cmdline, expected_rc=0)
        self.assertEqual(len(self.operations_called), 2)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')
        self.assertEqual(self.operations_called[1][0].name, 'ListObjectsV2')


class TestSyncSourceRegion(BaseS3CLIRunnerTest):
    def test_respects_source_region(self):
        source_region = 'af-south-1'
        cmdline = [
            's3',
            'sync',
            's3://sourcebucket/',
            's3://bucket/',
            '--region',
            self.region,
            '--source-region',
            source_region,
        ]
        self.add_botocore_list_objects_response(['key'])
        self.add_botocore_list_objects_response([])
        self.add_botocore_copy_object_response()
        result = self.run_command(cmdline)
        self.assert_no_remaining_botocore_responses()
        self.assert_operations_to_endpoints(
            cli_runner_result=result,
            expected_operations_to_endpoints=[
                (
                    'ListObjectsV2',
                    self.get_virtual_s3_host('sourcebucket', source_region),
                ),
                (
                    'ListObjectsV2',
                    self.get_virtual_s3_host('bucket', self.region),
                ),
                (
                    'CopyObject',
                    self.get_virtual_s3_host('bucket', self.region),
                ),
            ],
        )


class TestSyncWithCRTClient(BaseCRTTransferClientTest):
    def test_upload_sync_using_crt_client(self):
        filename = self.files.create_file('myfile', 'mycontent')
        cmdline = [
            's3',
            'sync',
            self.files.rootdir,
            's3://bucket/',
        ]
        self.add_botocore_list_objects_response([])
        self.run_command(cmdline)
        crt_requests = self.get_crt_make_request_calls()
        self.assertEqual(len(crt_requests), 1)
        self.assert_crt_make_request_call(
            crt_requests[0],
            expected_type=S3RequestType.PUT_OBJECT,
            expected_host=self.get_virtual_s3_host('bucket'),
            expected_path='/myfile',
            expected_send_filepath=filename,
        )

    def test_download_sync_using_crt_client(self):
        cmdline = [
            's3',
            'sync',
            's3://bucket/',
            self.files.rootdir,
        ]
        self.add_botocore_list_objects_response(['key'])
        self.run_command(cmdline)
        crt_requests = self.get_crt_make_request_calls()
        self.assertEqual(len(crt_requests), 1)
        self.assert_crt_make_request_call(
            crt_requests[0],
            expected_type=S3RequestType.GET_OBJECT,
            expected_host=self.get_virtual_s3_host('bucket'),
            expected_path='/key',
            expected_recv_startswith=os.path.join(self.files.rootdir, 'key'),
        )

    def test_upload_sync_with_delete_using_crt_client(self):
        filename = self.files.create_file('a-file', 'mycontent')
        cmdline = [
            's3',
            'sync',
            self.files.rootdir,
            's3://bucket/',
            '--delete',
        ]
        self.add_botocore_list_objects_response(['delete-this'])
        self.run_command(cmdline)
        crt_requests = self.get_crt_make_request_calls()
        self.assertEqual(len(crt_requests), 2)
        self.assert_crt_make_request_call(
            crt_requests[0],
            expected_type=S3RequestType.PUT_OBJECT,
            expected_host=self.get_virtual_s3_host('bucket'),
            expected_path='/a-file',
            expected_send_filepath=filename,
        )
        self.assert_crt_make_request_call(
            crt_requests[1],
            expected_type=S3RequestType.DEFAULT,
            expected_host=self.get_virtual_s3_host('bucket'),
            expected_path='/delete-this',
            expected_http_method='DELETE',
        )

    def test_download_sync_with_delete_using_crt_client(self):
        self.files.create_file('delete-this', 'content')
        cmdline = [
            's3',
            'sync',
            's3://bucket/',
            self.files.rootdir,
            '--delete',
        ]
        self.add_botocore_list_objects_response(['key'])
        self.run_command(cmdline)
        crt_requests = self.get_crt_make_request_calls()
        self.assertEqual(len(crt_requests), 1)
        self.assert_crt_make_request_call(
            crt_requests[0],
            expected_type=S3RequestType.GET_OBJECT,
            expected_host=self.get_virtual_s3_host('bucket'),
            expected_path='/key',
            expected_recv_startswith=os.path.join(self.files.rootdir, 'key'),
        )
        self.assertFalse(os.path.exists('delete-this'))

    def test_does_not_use_crt_client_for_copy_syncs(self):
        cmdline = ['s3', 'sync', 's3://bucket/', 's3://otherbucket/']
        self.add_botocore_list_objects_response(['key'])
        self.add_botocore_list_objects_response([])
        self.add_botocore_copy_object_response()
        self.run_command(cmdline)
        self.assertEqual(self.get_crt_make_request_calls(), [])
        self.assert_no_remaining_botocore_responses()


class TestSyncCommandWithS3Express(BaseS3TransferCommandTest):
    prefix = 's3 sync '

    def test_incompatible_with_sync_upload(self):
        cmdline = '%s %s s3://testdirectorybucket--usw2-az1--x-s3/' % (
            self.prefix,
            self.files.rootdir,
        )
        stderr = self.run_cmd(cmdline, expected_rc=252)[1]
        self.assertIn(
            'Cannot use sync command with a directory bucket.', stderr
        )

    def test_incompatible_with_sync_download(self):
        cmdline = '%s s3://testdirectorybucket--usw2-az1--x-s3/ %s' % (
            self.prefix,
            self.files.rootdir,
        )
        stderr = self.run_cmd(cmdline, expected_rc=252)[1]
        self.assertIn(
            'Cannot use sync command with a directory bucket.', stderr
        )

    def test_incompatible_with_sync_copy(self):
        cmdline = (
            '%s s3://bucket/ s3://testdirectorybucket--usw2-az1--x-s3/'
            % self.prefix
        )
        stderr = self.run_cmd(cmdline, expected_rc=252)[1]
        self.assertIn(
            'Cannot use sync command with a directory bucket.', stderr
        )

    def test_incompatible_with_sync_with_delete(self):
        cmdline = (
            '%s s3://bucket/ s3://testdirectorybucket--usw2-az1--x-s3/ --delete'
            % self.prefix
        )
        stderr = self.run_cmd(cmdline, expected_rc=252)[1]
        self.assertIn(
            'Cannot use sync command with a directory bucket.', stderr
        )

    def test_compatible_with_sync_with_local_directory_like_directory_bucket(
        self,
    ):
        self.parsed_responses = [{'Contents': []}]
        cmdline = (
            '%s s3://bucket/ testdirectorybucket--usw2-az1--x-s3/'
            % self.prefix
        )
        with cd(self.files.rootdir):
            _, stderr, _ = self.run_cmd(cmdline)
        # Just asserting that command validated and made an API call
        self.assertEqual(len(self.operations_called), 1)
        self.assertEqual(self.operations_called[0][0].name, 'ListObjectsV2')


class TestSyncCaseConflict(BaseS3TransferCommandTest):
    prefix = 's3 sync '

    def setUp(self):
        super().setUp()
        self.lower_key = 'a.txt'
        self.upper_key = 'A.txt'

    @skip_if_case_sensitive()
    def test_error_with_existing_file(self):
        self.files.create_file(self.lower_key, 'mycontent')
        cmd = (
            f"{self.prefix} s3://bucket {self.files.rootdir} "
            "--case-conflict error"
        )
        self.parsed_responses = [self.list_objects_response([self.upper_key])]
        _, stderr, _ = self.run_cmd(cmd, expected_rc=1)
        assert f"Failed to download bucket/{self.upper_key}" in stderr

    def test_error_with_case_conflicts_in_s3(self):
        cmd = (
            f"{self.prefix} s3://bucket {self.files.rootdir} "
            "--case-conflict error"
        )
        self.parsed_responses = [
            self.list_objects_response([self.upper_key, self.lower_key])
        ]
        _, stderr, _ = self.run_cmd(cmd, expected_rc=1)
        assert f"Failed to download bucket/{self.lower_key}" in stderr

    @skip_if_case_sensitive()
    def test_warn_with_existing_file(self):
        self.files.create_file(self.lower_key, 'mycontent')
        cmd = (
            f"{self.prefix} s3://bucket {self.files.rootdir} "
            "--case-conflict warn"
        )
        self.parsed_responses = [
            self.list_objects_response([self.upper_key]),
            self.get_object_response(),
        ]
        _, stderr, _ = self.run_cmd(cmd, expected_rc=0)
        assert f"warning: Downloading bucket/{self.upper_key}" in stderr

    @skip_if_windows("Can't rename to same file")
    def test_warn_with_case_conflicts_in_s3(self):
        cmd = (
            f"{self.prefix} s3://bucket {self.files.rootdir} "
            "--case-conflict warn"
        )
        self.parsed_responses = [
            self.list_objects_response([self.upper_key, self.lower_key]),
            self.get_object_response(),
            self.get_object_response(),
        ]
        _, stderr, _ = self.run_cmd(cmd, expected_rc=0)
        assert f"warning: Downloading bucket/{self.lower_key}" in stderr

    @skip_if_case_sensitive()
    def test_skip_with_existing_file(self):
        self.files.create_file(self.lower_key, 'mycontent')
        cmd = (
            f"{self.prefix} s3://bucket {self.files.rootdir} "
            "--case-conflict skip"
        )
        self.parsed_responses = [self.list_objects_response([self.upper_key])]
        _, stderr, _ = self.run_cmd(cmd, expected_rc=0)
        assert f"warning: Skipping bucket/{self.upper_key}" in stderr

    def test_skip_with_case_conflicts_in_s3(self):
        cmd = (
            f"{self.prefix} s3://bucket {self.files.rootdir} "
            "--case-conflict skip"
        )
        self.parsed_responses = [
            self.list_objects_response([self.upper_key, self.lower_key]),
            self.get_object_response(),
        ]
        _, stderr, _ = self.run_cmd(cmd, expected_rc=0)
        assert f"warning: Skipping bucket/{self.lower_key}" in stderr

    def test_ignore_with_existing_file(self):
        self.files.create_file(self.lower_key, 'mycontent')
        cmd = (
            f"{self.prefix} s3://bucket {self.files.rootdir} "
            "--case-conflict ignore"
        )
        self.parsed_responses = [
            self.list_objects_response([self.upper_key]),
            self.get_object_response(),
        ]
        self.run_cmd(cmd, expected_rc=0)

    @skip_if_windows("Can't rename to same file")
    def test_ignore_with_case_conflicts_in_s3(self):
        cmd = (
            f"{self.prefix} s3://bucket {self.files.rootdir} "
            "--case-conflict ignore"
        )
        self.parsed_responses = [
            self.list_objects_response([self.upper_key, self.lower_key]),
            self.get_object_response(),
            self.get_object_response(),
        ]
        self.run_cmd(cmd, expected_rc=0)
