From 668c2ca8dfadbfb690637172c743bba74a95f935 Mon Sep 17 00:00:00 2001 From: Lutz Justen Date: Thu, 8 Dec 2022 08:39:43 +0100 Subject: [PATCH] [cdc_rsync] Add integration tests (#42) [cdc_rsync] Add integration tests This CL adds Python integration tests for cdc_rsync. To run the tests, you need to supply a Linux host and proper configuration for cdc_rsync to work: set CDC_SSH_COMMAND=C:\path\to\ssh.exe set CDC_SCP_COMMAND=C:\path\to\scp.exe C:\python38\python.exe -m integration_tests.cdc_rsync.all_tests --binary_path=C:\full\path\to\cdc_rsync.exe --user_host=user@host Ran the tests and made sure they worked. --- .gitignore | 1 + __init__.py | 0 integration_tests/__init__.py | 13 + integration_tests/cdc_rsync/__init__.py | 13 + integration_tests/cdc_rsync/all_tests.py | 44 + .../cdc_rsync/connection_test.py | 131 +++ .../cdc_rsync/deployment_test.py | 110 +++ integration_tests/cdc_rsync/dry_run_test.py | 116 +++ integration_tests/cdc_rsync/output_test.py | 243 +++++ integration_tests/cdc_rsync/test_base.py | 113 +++ integration_tests/cdc_rsync/upload_test.py | 894 ++++++++++++++++++ integration_tests/framework/__init__.py | 13 + integration_tests/framework/test_base.py | 63 ++ integration_tests/framework/test_runner.py | 66 ++ integration_tests/framework/utils.py | 331 +++++++ 15 files changed, 2151 insertions(+) create mode 100644 __init__.py create mode 100644 integration_tests/__init__.py create mode 100644 integration_tests/cdc_rsync/__init__.py create mode 100644 integration_tests/cdc_rsync/all_tests.py create mode 100644 integration_tests/cdc_rsync/connection_test.py create mode 100644 integration_tests/cdc_rsync/deployment_test.py create mode 100644 integration_tests/cdc_rsync/dry_run_test.py create mode 100644 integration_tests/cdc_rsync/output_test.py create mode 100644 integration_tests/cdc_rsync/test_base.py create mode 100644 integration_tests/cdc_rsync/upload_test.py create mode 100644 integration_tests/framework/__init__.py create mode 100644 integration_tests/framework/test_base.py create mode 100644 integration_tests/framework/test_runner.py create mode 100644 integration_tests/framework/utils.py diff --git a/.gitignore b/.gitignore index 59bcc04..d719215 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ dependencies .qtc_clangd bazel-* user.bazelrc +*.pyc \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/integration_tests/__init__.py b/integration_tests/__init__.py new file mode 100644 index 0000000..6d6d126 --- /dev/null +++ b/integration_tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/integration_tests/cdc_rsync/__init__.py b/integration_tests/cdc_rsync/__init__.py new file mode 100644 index 0000000..6d6d126 --- /dev/null +++ b/integration_tests/cdc_rsync/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/integration_tests/cdc_rsync/all_tests.py b/integration_tests/cdc_rsync/all_tests.py new file mode 100644 index 0000000..8f9ee81 --- /dev/null +++ b/integration_tests/cdc_rsync/all_tests.py @@ -0,0 +1,44 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +import unittest + +from integration_tests.cdc_rsync import connection_test +from integration_tests.cdc_rsync import deployment_test +from integration_tests.cdc_rsync import dry_run_test +from integration_tests.cdc_rsync import output_test +from integration_tests.cdc_rsync import upload_test +from integration_tests.framework import test_base + + +# pylint: disable=g-doc-args,g-doc-return-or-yield +def load_tests(loader, unused_tests, unused_pattern): + """Customizes the list of test cases to run. + + See the Python documentation for details: + https://docs.python.org/3/library/unittest.html#load-tests-protocol + """ + suite = unittest.TestSuite() + suite.addTests(loader.loadTestsFromModule(connection_test)) + suite.addTests(loader.loadTestsFromModule(deployment_test)) + suite.addTests(loader.loadTestsFromModule(dry_run_test)) + suite.addTests(loader.loadTestsFromModule(output_test)) + suite.addTests(loader.loadTestsFromModule(upload_test)) + + return suite + + +if __name__ == '__main__': + test_base.main() diff --git a/integration_tests/cdc_rsync/connection_test.py b/integration_tests/cdc_rsync/connection_test.py new file mode 100644 index 0000000..dd3085d --- /dev/null +++ b/integration_tests/cdc_rsync/connection_test.py @@ -0,0 +1,131 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""cdc_rsync connection test.""" + +from concurrent import futures +import socket +import time + +from integration_tests.framework import utils +from integration_tests.cdc_rsync import test_base + +RETURN_CODE_SUCCESS = 0 +RETURN_CODE_GENERIC_ERROR = 1 +RETURN_CODE_CONNECTION_TIMEOUT = 2 +RETURN_CODE_ADDRESS_IN_USE = 4 + +FIRST_PORT = 44450 +LAST_PORT = 44459 + + +class ConnectionTest(test_base.CdcRsyncTest): + """cdc_rsync connection test class.""" + + def test_valid_instance(self): + """Runs rsync with --instance option for a valid id. + + 1) Uploads a file with --instance option instead of --ip --port. + 2) Checks the file exists on the used instance. + """ + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir) + self._assert_rsync_success(res) + self.assertTrue(utils.does_file_exist_remotely(self.remote_data_path)) + + def test_invalid_instance(self): + """Runs rsync with --instance option for an invalid id. + + 1) Uploads a file with --instance option for a non-existing id. + 2) Checks the error message. + """ + bad_host = 'bad_host' + + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, + bad_host + ":" + self.remote_base_dir) + self.assertEqual(res.returncode, RETURN_CODE_GENERIC_ERROR) + self.assertIn('lost connection', str(res.stderr)) + + def test_contimeout(self): + """Runs rsync with --contimeout option for an invalid ip. + + 1) Uploads a file with bad IP address. + 2) Checks the error message and that it timed out after ~5 seconds. + 3) Uploads a file with bad IP address and --contimeout 1. + 4) Checks the error message and that it timed out after ~1 second. + + """ + utils.create_test_file(self.local_data_path, 1024) + bad_host = '192.0.2.1' + start = time.time() + res = utils.run_rsync(self.local_data_path, + bad_host + ":" + self.remote_base_dir) + elapsed_time = time.time() - start + self.assertGreater(elapsed_time, 4.5) + self.assertEqual(res.returncode, RETURN_CODE_CONNECTION_TIMEOUT) + self.assertIn('Error: Server connection timed out', str(res.stderr)) + + start = time.time() + res = utils.run_rsync(self.local_data_path, + bad_host + ":" + self.remote_base_dir, + '--contimeout=1') + elapsed_time = time.time() - start + self.assertLess(elapsed_time, 3) + self.assertEqual(res.returncode, RETURN_CODE_CONNECTION_TIMEOUT) + self.assertIn('Error: Server connection timed out', str(res.stderr)) + + def test_multiple_instances(self): + """Runs multiple instances of rsync at the same time.""" + num_instances = LAST_PORT - FIRST_PORT + 1 + + local_data_paths = [] + for n in range(num_instances): + path = self.local_base_dir + ('testdata_%i.dat' % n) + utils.create_test_file(path, 1024) + local_data_paths.append(path) + + with futures.ThreadPoolExecutor(max_workers=num_instances) as executor: + res = [] + for n in range(num_instances): + res.append( + executor.submit(utils.run_rsync, local_data_paths[n], + self.remote_base_dir)) + for r in res: + self._assert_rsync_success(r.result()) + + def test_address_in_use(self): + """Blocks all ports and checks that rsync fails with the expected error.""" + sockets = [] + try: + # Occupy all ports. + for port in range(FIRST_PORT, LAST_PORT + 1): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sockets.append(s) + s.bind(('127.0.0.1', port)) + s.listen() + + # rsync shouldn't be able to find an available port now. + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir) + self.assertIn('All ports are already in use', str(res.stderr)) + + finally: + for s in sockets: + s.close() + + +if __name__ == '__main__': + test_base.test_base.main() diff --git a/integration_tests/cdc_rsync/deployment_test.py b/integration_tests/cdc_rsync/deployment_test.py new file mode 100644 index 0000000..7a628a9 --- /dev/null +++ b/integration_tests/cdc_rsync/deployment_test.py @@ -0,0 +1,110 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""cdc_rsync deployment test.""" + +from integration_tests.framework import utils +from integration_tests.cdc_rsync import test_base + +REMOTE_FOLDER = '~/.cache/cdc-file-transfer/bin/' + + +class DeploymentTest(test_base.CdcRsyncTest): + """cdc_rsync deployment test class.""" + + def _assert_deployment(self, initial_ts, file, msg): + """Checks rsync and library are uploaded and the given file's timestamp matches initial_ts.""" + res = utils.run_rsync(self.local_data_path, self.remote_base_dir) + self._assert_rsync_success(res) + self.assertIn(msg, str(res.stdout)) + changed_ts = utils.get_ssh_command_output('stat --format=%%y %s' % + REMOTE_FOLDER + file) + self.assertEqual(initial_ts, changed_ts) + + def _change_file_preserve_timestamp(self, file): + """Changes a file preserving it timestamp.""" + utils.get_ssh_command_output( + 'touch -r %s %s' % + (REMOTE_FOLDER + file, REMOTE_FOLDER + file + '.tmp')) + utils.get_ssh_command_output('truncate -s +100 %s' % REMOTE_FOLDER + file) + utils.get_ssh_command_output( + 'touch -r %s %s' % + (REMOTE_FOLDER + file + '.tmp', REMOTE_FOLDER + file)) + utils.get_ssh_command_output('rm %s' % (REMOTE_FOLDER + file + '.tmp')) + + def test_no_server(self): + """Checks that cdc_rsync_server is uploaded if not present on the gamelet. + + 1) Wipes ‘/opt/developer/tools/bin/’ on the gamelet. + 2) Uploads a file. + 3) Verifies that cdc_rsync_server exists in that folder. + """ + utils.get_ssh_command_output('rm -rf %s*' % REMOTE_FOLDER) + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir) + self._assert_rsync_success(res) + self.assertIn('Server not deployed. Deploying...', str(res.stdout)) + self._assert_remote_dir_contains(['cdc_rsync_server'], + remote_dir=REMOTE_FOLDER, + pattern='"*"') + + def test_modified_server(self): + """Checks that cdc_rsync_server is re-uploaded. + + 1) Touches cdc_rsync_server in ‘REMOTE_FOLDER’. + 2) Uploads a file. + 3) Verifies that cdc_rsync_server is re-uploaded. + 4) Appends a few bytes to cdc_rsync_server while keeping its timestamp. + 6) Uploads a file. + 7) Verifies that cdc_rsync_server is re-uploaded. + """ + # To be sure that cdc_rsync_server exist on the remote system + # do an "empty" copy. + utils.run_rsync(self.local_base_dir, self.remote_base_dir) + + remote_server_path = REMOTE_FOLDER + 'cdc_rsync_server' + initial_ts = utils.get_ssh_command_output('stat --format=%%y %s' % + remote_server_path) + utils.get_ssh_command_output('touch -d \'1 November 2020 00:00\' %s' % + remote_server_path) + changed_ts = utils.get_ssh_command_output('stat --format=%%y %s' % + remote_server_path) + self.assertNotEqual(initial_ts, changed_ts) + utils.create_test_file(self.local_data_path, 1024) + self._assert_deployment(initial_ts, 'cdc_rsync_server', + 'Server outdated. Redeploying...') + + self._change_file_preserve_timestamp('cdc_rsync_server') + self._assert_deployment(initial_ts, 'cdc_rsync_server', + 'Server outdated. Redeploying...') + + def test_read_only_server(self): + """Checks that cdc_rsync_server is overwritten if it is read-only.""" + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir) + self._assert_rsync_success(res) + + # Modify cdc_rsync_server and wipe permissions. + remote_server_path = REMOTE_FOLDER + 'cdc_rsync_server' + utils.get_ssh_command_output('echo "xxx" > %s && chmod 0 %s' % + (remote_server_path, remote_server_path)) + + res = utils.run_rsync(self.local_data_path, self.remote_base_dir) + self._assert_rsync_success(res) + self.assertIn('Server failed to start. Redeploying...', str(res.stdout)) + + +if __name__ == '__main__': + test_base.test_base.main() diff --git a/integration_tests/cdc_rsync/dry_run_test.py b/integration_tests/cdc_rsync/dry_run_test.py new file mode 100644 index 0000000..1066f86 --- /dev/null +++ b/integration_tests/cdc_rsync/dry_run_test.py @@ -0,0 +1,116 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""cdc_rsync dry-run test.""" + +from integration_tests.framework import utils +from integration_tests.cdc_rsync import test_base + + +class DryRunTest(test_base.CdcRsyncTest): + """cdc_rsync dry-run test class.""" + + def test_dry_run(self): + """Verifies --dry-run option. + + 1) Uploads file1.txt and file2.txt. + 2) Modifies file2.txt. + 3) Dry-runs file2.txt and file3.txt with --dry-run -r --delete. + Result: a missing (file3.txt), a changed (file2.txt) and an extraneous + (file1.txt) file. No files should be changed on the server. + + """ + + files = ['file1.txt', 'file2.txt', 'file3.txt'] + + for file in files: + utils.create_test_file(self.local_base_dir + file, 987) + + res = utils.run_rsync(self.local_base_dir + 'file1.txt', + self.local_base_dir + 'file2.txt', + self.remote_base_dir, '-v') + self._assert_rsync_success(res) + self._assert_remote_dir_contains(['file1.txt', 'file2.txt']) + + # Dry-run of uploading changed/new/to delete files. + utils.create_test_file(self.local_base_dir + 'file2.txt', 2534) + res = utils.run_rsync(self.local_base_dir + 'file2.txt', + self.local_base_dir + 'file3.txt', + self.remote_base_dir, '-v', '--dry-run', '--delete', + '-r') + self._assert_rsync_success(res) + self.assertTrue( + utils.files_count_is(res, missing=1, changed=1, extraneous=1)) + self._assert_remote_dir_does_not_contain(['file3.txt']) + self._assert_remote_dir_contains(['file1.txt', 'file2.txt']) + self.assertIn('file1.txt', str(res.stdout)) + self.assertIn('deleted 1 / 1', str(res.stdout)) + self.assertIn('file2.txt', str(res.stdout)) + self.assertIn('D100%', str(res.stdout)) + self.assertIn('file3.txt', str(res.stdout)) + self.assertIn('C100%', str(res.stdout)) + self.assertFalse( + utils.sha1_matches(self.local_base_dir + 'file2.txt', + self.remote_base_dir + 'file2.txt')) + + def test_dry_run_sync_folder_when_remote_file_recursive_with_delete(self): + """Dry-runs a recursive upload of a folder while removing a remote file with the same name with --delete.""" + + local_folder = self.local_base_dir + 'foldertocopy\\' + utils.create_test_directory(local_folder) + utils.get_ssh_command_output( + 'mkdir -p %s && touch %s' % + (self.remote_base_dir, self.remote_base_dir + 'foldertocopy')) + + res = utils.run_rsync(self.local_base_dir + 'foldertocopy', + self.remote_base_dir, '-r', '--dry-run', '--delete') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, extraneous=1, missing_dir=1)) + self.assertFalse( + utils.does_directory_exist_remotely(self.remote_base_dir + + 'foldertocopy')) + self.assertTrue( + utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy')) + self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout)) + + def test_dry_run_sync_file_when_remote_folder_recursive_with_delete(self): + """Dry-runs a recursive upload of a file while removing an empty remote folder with the same name with --delete.""" + utils.create_test_file(self.local_data_path, 1024) + utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path) + + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, + '--dry-run', '-r', '--delete') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1)) + self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path)) + self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path)) + self.assertIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout)) + + def test_dry_run_sync_file_when_remote_folder_empty(self): + """Dry-runs a non-recursive upload of a file while there is an empty remote folder with the same name.""" + utils.create_test_file(self.local_data_path, 1024) + utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path) + + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, + '--dry-run') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1)) + self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path)) + self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path)) + self.assertNotIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout)) + + +if __name__ == '__main__': + test_base.test_base.main() diff --git a/integration_tests/cdc_rsync/output_test.py b/integration_tests/cdc_rsync/output_test.py new file mode 100644 index 0000000..1acc1a5 --- /dev/null +++ b/integration_tests/cdc_rsync/output_test.py @@ -0,0 +1,243 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""cdc_rsync output test.""" + +import json + +from integration_tests.framework import utils +from integration_tests.cdc_rsync import test_base + + +class OutputTest(test_base.CdcRsyncTest): + """cdc_rsync output test class.""" + + def test_plain(self): + """Runs rsync and verifies the total progress. + + 1) Uploads a file, verifies that the total progress is shown. + 2) Uploads an empty folder with -r --delete options. + Verifies that the total delete messages are shown. + """ + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir) + self._assert_rsync_success(res) + self.assertIn('100% TOT', str(res.stdout)) + + utils.remove_test_file(self.local_data_path) + res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r', + '--delete') + self._assert_rsync_success(res) + self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout)) + + def test_verbose_1(self): + """Runs rsync with -v option for multiple files. + + 1) Uploads 3 files with ‘-v’. + Verifies that each file is listed in the output as ‘C100%’. + 2) Modifies 3 files, uploads them again with ‘--v’. + Verifies that each file is listed in the output as ‘D100%’. + 3) Uploads an empty folder with -r --delete options. + Verifies that the delete messages are shown. + """ + files = ['file1. txt', 'file2.txt', 'file3.txt'] + for file in files: + utils.create_test_file(self.local_base_dir + file, 1024) + res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r') + self._assert_rsync_success(res) + self.assertEqual(3, str(res.stdout).count('C100%')) + + for file in files: + utils.create_test_file(self.local_base_dir + file, 2048) + res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r') + self._assert_rsync_success(res) + self.assertEqual(3, str(res.stdout).count('D100%')) + + for file in files: + utils.remove_test_file(self.local_base_dir + file) + res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r', + '--delete') + self._assert_rsync_success(res) + self.assertIn('will be deleted due to --delete', str(res.stdout)) + self.assertIn('3/3 file(s) and 0/0 folder(s) deleted', str(res.stdout)) + + def test_verbose_2(self): + """Runs rsync with -vv option. + + 1) Uploads a file with ‘-vv’. + 2) Verifies that additional logs show up. + """ + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vv') + self._assert_rsync_success(res) + output = str(res.stdout) + + # client-side output + self._assert_regex('Starting process', output) + self._assert_not_regex( + r'process\.cc\([0-9]+\): Start\(\): Starting process', output) + + # server-side output + self._assert_regex( + 'INFO Finding all files in destination folder ' + f"'{self.remote_base_dir}'", output) + self.assertNotIn('DEBUG', output) + + def test_verbose_3(self): + """Runs rsync with -vvv option. + + 1) Uploads a file with ‘-vvv’. + Verifies that additional logs show up (LOG_DEBUG logs). + 2) Uploads a file to ‘/invalid’ with ‘-vvv’. + Verifies that error messages including filenames are shown. + """ + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vvv') + self._assert_rsync_success(res) + output = str(res.stdout) + + # client-side output + self._assert_regex( + r'cdc_rsync_client\.cc\([0-9]+\): SendOptions\(\): Sending options', + output) + + # server-side output + self._assert_regex( + r'DEBUG server_socket\.cc\([0-9]+\): Receive\(\): EOF\(\) detected', + output) + + # TODO: Add a check here, as currently the output is misleading + # res = utils.run_rsync(self.local_data_path, '/invalid', '-vvv') + + def test_verbose_4(self): + """Runs rsync with -vvv option. + + 1) Uploads a file with ‘-vvvv’. + 2) Verifies that additional logs show up (LOG_VERBOSE logs). + """ + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vvvv') + self._assert_rsync_success(res) + output = str(res.stdout) + + # client-side output + self._assert_regex( + r'message_pump\.cc\([0-9]+\): ThreadDoSendPacket\(\): Sent packet of size', + output) + + # server-side output + self._assert_regex( + r'VERBOSE message_pump\.cc\([0-9]+\): ThreadDoReceivePacket\(\): Received packet of size', + output) + + def test_quiet(self): + """Runs rsync with -q option. + + 1) Uploads a file with ‘-q’. + 2) Verifies that no output is shown. + """ + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-q') + self._assert_rsync_success(res) + self.assertEqual('\r\n', res.stdout) + + def test_quiet_error(self): + """Runs rsync with -q option still showing errors. + + 1) Uploads a file with ‘-q’ and bad options. + 2) Verifies that an error message is shown. + """ + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-q', + '-t') + self.assertEqual(res.returncode, 1) + self.assertEqual('\r\n', str(res.stdout)) + self.assertIn('Unknown option: \'t\'', str(res.stderr)) + # TODO: Add a test case for the non-existing destination. + + def test_existing_verbose_1(self): + """Runs rsync with -v --existing.""" + + files = ['file1.txt', 'file2.txt'] + for file in files: + utils.create_test_file(self.local_base_dir + file, 1024) + res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r') + self._assert_rsync_success(res) + + files.append('file3.txt') + for file in files: + utils.create_test_file(self.local_base_dir + file, 2048) + res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r', + '--existing') + self._assert_rsync_success(res) + output = str(res.stdout) + self.assertEqual(2, output.count('D100%')) + self.assertNotIn('file3.txt', output) + + def test_json_per_file(self): + """Runs rsync with -v --json.""" + + local_path = self.local_base_dir + 'test.txt' + utils.create_test_file(local_path, 1024) + res = utils.run_rsync(local_path, self.remote_base_dir, '-v', '--json') + self._assert_rsync_success(res) + output = str(res.stdout) + + for val in self.parse_json(output): + self.assertEqual(val['file'], 'test.txt') + self.assertEqual(val['operation'], 'Copy') + self.assertEqual(val['size'], 1024) + + # Those are actually all floats, but sometimes they get rounded to ints. + self.assertTrue(self.is_float_or_int(val['bytes_per_second'])) + self.assertTrue(self.is_float_or_int(val['duration'])) + self.assertTrue(self.is_float_or_int(val['eta'])) + self.assertTrue(self.is_float_or_int(val['total_duration'])) + self.assertTrue(self.is_float_or_int(val['total_eta'])) + self.assertTrue(self.is_float_or_int(val['total_progress'])) + + def test_json_total(self): + """Runs rsync with --json.""" + + local_path = self.local_base_dir + 'test.txt' + utils.create_test_file(local_path, 1024) + res = utils.run_rsync(local_path, self.remote_base_dir, '--json') + self._assert_rsync_success(res) + output = str(res.stdout) + + for val in self.parse_json(output): + self.assertNotIn('file', val) + + # Those are actually all floats, but sometimes they get rounded to ints. + self.assertTrue(self.is_float_or_int(val['total_duration'])) + self.assertTrue(self.is_float_or_int(val['total_eta'])) + self.assertTrue(self.is_float_or_int(val['total_progress'])) + + def parse_json(self, output): + """Parses the JSON lines of output.""" + lines = output.split('\r\n') + json_values = [] + for line in lines: + if str.startswith(line, '{'): + json_values.append(json.loads(line.strip())) + return json_values + + def is_float_or_int(self, val): + """Returns true if val is a float or an int.""" + return isinstance(val, float) or isinstance(val, int) + + +if __name__ == '__main__': + test_base.test_base.main() diff --git a/integration_tests/cdc_rsync/test_base.py b/integration_tests/cdc_rsync/test_base.py new file mode 100644 index 0000000..02fe397 --- /dev/null +++ b/integration_tests/cdc_rsync/test_base.py @@ -0,0 +1,113 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""cdc_rsync base test class.""" + +import datetime +import logging +import tempfile +import re +import unittest + +from integration_tests.framework import utils +from integration_tests.framework import test_base + + +class CdcRsyncTest(unittest.TestCase): + """cdc_rsync base test class.""" + + tmp_dir = None + local_base_dir = None + remote_base_dir = None + local_data_path = None + remote_data_path = None + + def setUp(self): + """Cleans up the remote test data folder, logs a marker, and initializes random.""" + super(CdcRsyncTest, self).setUp() + logging.debug('CdcRsyncTest -> setUp') + + utils.initialize(test_base.Flags.binary_path, test_base.Flags.user_host) + + now_str = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') + self.tmp_dir = tempfile.TemporaryDirectory( + prefix=f'_cdc_rsync_test_{now_str}') + self.local_base_dir = self.tmp_dir.name + '\\' + self.remote_base_dir = f'/tmp/_cdc_rsync_test_{now_str}/' + self.local_data_path = self.local_base_dir + 'testdata.dat' + self.remote_data_path = self.remote_base_dir + 'testdata.dat' + + logging.info('Local base dir: "%s"', self.local_base_dir) + logging.info('Remote base dir: "%s"', self.remote_base_dir) + utils.initialize_random() + + def tearDown(self): + """Cleans up the local and remote temp directories.""" + super(CdcRsyncTest, self).tearDown() + logging.debug('CdcRsyncTest -> tearDown') + self.tmp_dir.cleanup() + utils.get_ssh_command_output(f'rm -rf {self.remote_base_dir}') + + def _assert_rsync_success(self, res): + """Asserts if the return code is 0 and outputs return message with args.""" + self.assertEqual(res.returncode, 0, 'Return value is ' + str(res)) + + def _assert_regex(self, regex, value): + """Asserts that the regex string matches the given value.""" + self.assertIsNotNone( + re.search(regex, value), f'"Regex {regex}" does not match "{value}"') + + def _assert_not_regex(self, regex, value): + """Asserts that the regex string does not match the given value.""" + self.assertIsNone( + re.search(regex, value), + f'"Regex {regex}" unexpectedly matches "{value}"') + + def _assert_remote_dir_contains(self, + file_list, + remote_dir=None, + pattern='"*.[t|d]*"'): + """Asserts that the remote base dir contains exactly the list of files. + + Args: + file_list (list of strings): List of relative file paths to check + remote_dir (string, optional): Remote directory. Defaults to + remote_base_dir + pattern (string, optional): Pattern for matching file names. + """ + find_res = utils.get_ssh_command_output( + 'cd %s && find -name %s -print' % + (remote_dir or self.remote_base_dir, pattern)) + + # Note that assertCountEqual compares items independently of order + # (not just the size of the list). + found = sorted( + filter(lambda item: item and item != '.', find_res.split('\r\n'))) + expected = sorted(['./' + f for f in file_list]) + self.assertListEqual(found, expected) + + def _assert_remote_dir_does_not_contain(self, file_list): + """Asserts that the remote base dir contains none of the listed files. + + Args: + file_list (list of strings): List of relative file paths to check + """ + find_res = utils.get_ssh_command_output( + 'cd %s && find -name "*.[t|d]*" -print' % self.remote_base_dir) + + found = set(file_name for file_name in filter(None, find_res.split('\n'))) + + for file in file_list: + self.assertNotIn('./' + file, found) diff --git a/integration_tests/cdc_rsync/upload_test.py b/integration_tests/cdc_rsync/upload_test.py new file mode 100644 index 0000000..61c58be --- /dev/null +++ b/integration_tests/cdc_rsync/upload_test.py @@ -0,0 +1,894 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""cdc_rsync upload test.""" + +import json +import logging +import os +import subprocess +import time + +from integration_tests.framework import utils +from integration_tests.cdc_rsync import test_base + + +class UploadTest(test_base.CdcRsyncTest): + """cdc_rsync upload test class.""" + + def test_single_uncompressed(self): + """Uploads and syncs a file uncompressed.""" + + self._do_test_single(compressed=False) + + def test_upload_compressed(self): + """Uploads and syncs a file compressed.""" + + self._do_test_single(compressed=True) + + def _do_test_single(self, compressed): + """Runs rsync 3 times and validates results. + + 1) Uploads a file, checks sha1 hashes. + 2) Uploads the same file again, checks nothing changed. + 3) Modifies the file and uploads again. Checks sha1 hashes. + + Args: + compressed (bool): Whether to append '--compress' or not. + """ + compressed_arg = '--compress' if compressed else None + + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, + compressed_arg) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=1)) + self.assertTrue( + utils.sha1_matches(self.local_data_path, self.remote_data_path)) + + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, + compressed_arg) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, matching=1)) + + utils.create_test_file(self.local_data_path, 2534) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, + compressed_arg) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, changed=1)) + self.assertTrue( + utils.sha1_matches(self.local_data_path, self.remote_data_path)) + + def test_backslash_in_dest_folder(self): + r"""Verifies uploading to \mnt\developer.""" + + filepath = os.path.join(self.local_base_dir, 'file1.txt') + utils.create_test_file(filepath, 1) + res = utils.run_rsync(filepath, self.remote_base_dir.replace('/', '\\')) + self.assertTrue(utils.files_count_is(res, missing=1)) + self._assert_remote_dir_contains(['file1.txt']) + + def test_backslash_in_source_folder(self): + r"""Verifies uploading from /source/folder.""" + + filepath = os.path.join(self.local_base_dir, 'file1.txt') + utils.create_test_file(filepath, 1) + filepath = filepath.replace('\\', '/') + res = utils.run_rsync(filepath, self.remote_base_dir) + self.assertTrue(utils.files_count_is(res, missing=1)) + self._assert_remote_dir_contains(['file1.txt']) + + def test_single_unicode(self): + """Uploads a file with a non-ascii unicode path and checks sha1 signatures.""" + + nonascii_local_data_path = self.local_base_dir + '⛽⛽⛽⛽⛽⛽⛽⛽.dat' + nonascii_remote_data_path = self.remote_base_dir + '⛽⛽⛽⛽⛽⛽⛽⛽.dat' + utils.create_test_file(nonascii_local_data_path, 1024) + # In order to check that non-ascii characters are not considered as + # wildcard + # ? characters, create a second file. Only 1 file should be uploaded. + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(nonascii_local_data_path, self.remote_base_dir, None) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=1)) + self.assertTrue( + utils.sha1_matches(nonascii_local_data_path, nonascii_remote_data_path)) + + def test_uncompressed_no_empty_folders(self): + """Uploads and syncs multiple files uncompressed in different folders.""" + + self._do_test_no_empty_folders(compressed=False) + + def test_compressed_no_empty_folders(self): + """Uploads and syncs multiple files compressed in different folders.""" + + self._do_test_no_empty_folders(compressed=True) + + def _do_test_no_empty_folders(self, compressed): + """Runs rsync with(out) -r for a non-trivial directory and validates results. + + 1) Uploads a source directory with -r, checks sha1 hashes. + |-- rootdir + | |-- dir1 + | |-- file1_1.txt + | |-- file1_2.txt + | |-- dir2 + | |-- file2_1.txt + | |-- file0.txt + 2) Uploads the same source directory again without -r, + checks nothing has changed. The directory should be just skipped. + 3) Uploads the same source directory with --delete option and with -r. + Nothing should change. + 4) Removes dir1 and dir2 locally. + Uploads the same source directory with --delete option and with -r. + dir1 and dir2 should be removed from the remote instance. + + Args: + compressed (bool): Whether to append '--compress' or not. + """ + compressed_arg = '--compress' if compressed else None + local_root_path = self.local_base_dir + 'rootdir' + remote_root_path = self.remote_base_dir + 'rootdir/' + utils.create_test_file(local_root_path + '\\dir1\\file1_1.txt', 1024) + utils.create_test_file(local_root_path + '\\dir1\\file1_2.txt', 1024) + utils.create_test_file(local_root_path + '\\dir2\\file2_1.txt', 1024) + utils.create_test_file(local_root_path + '\\file0.txt', 1024) + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg, + '-r') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=3)) + self.assertTrue( + utils.sha1_matches(local_root_path + '\\dir1\\file1_1.txt', + remote_root_path + 'dir1/file1_1.txt')) + self.assertTrue( + utils.sha1_matches(local_root_path + '\\dir1\\file1_2.txt', + remote_root_path + 'dir1/file1_2.txt')) + self.assertTrue( + utils.sha1_matches(local_root_path + '\\dir2\\file2_1.txt', + remote_root_path + 'dir2/file2_1.txt')) + self.assertTrue( + utils.sha1_matches(local_root_path + '\\file0.txt', + remote_root_path + 'file0.txt')) + + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, extraneous_dir=1)) + + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg, + '-r', '--delete') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=3)) + + utils.remove_test_directory(local_root_path + '\\dir1\\') + utils.remove_test_directory(local_root_path + '\\dir2\\') + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg, + '-r', '--delete') + self._assert_rsync_success(res) + self.assertTrue( + utils.files_count_is( + res, matching=1, extraneous=3, matching_dir=1, extraneous_dir=2)) + self.assertFalse( + utils.does_directory_exist_remotely(remote_root_path + 'dir1')) + self.assertFalse( + utils.does_directory_exist_remotely(remote_root_path + 'dir2')) + + def _do_test_no_empty_folders_with_backslash(self, compressed): + """Runs rsync with(out) -r for a non-trivial directory with a trailing backslash. + + 1) Uploads a source directory with -r, checks sha1 hashes. + Everything from rootdir should be copied except rootdir itself. + |-- rootdir + | |-- dir1 + | |-- file1_1.txt + | |-- file1_2.txt + | |-- dir2 + | |-- file2_1.txt + | |-- file0.txt + 2) Uploads the same source directory again without -r, + checks nothing has changed. The directory should be just skipped. + 3) Uploads the same source directory with --delete option and with -r. + Nothing should change. + 4) Removes dir1 and dir2 locally. + Uploads the same source directory with --delete option and with -r. + dir1 and dir2 should be removed from the remote instance. + + Args: + compressed (bool): Whether to append '--compress' or not. + """ + compressed_arg = '--compress' if compressed else None + local_root_path = self.local_base_dir + 'rootdir\\' + utils.create_test_file(local_root_path + 'dir1\\file1_1.txt', 1024) + utils.create_test_file(local_root_path + 'dir1\\file1_2.txt', 1024) + utils.create_test_file(local_root_path + 'dir2\\file2_1.txt', 1024) + utils.create_test_file(local_root_path + 'file0.txt', 1024) + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg, + '-r') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=2)) + self.assertTrue( + utils.sha1_matches(local_root_path + 'dir1\\file1_1.txt', + self.remote_base_dir + 'dir1/file1_1.txt')) + self.assertTrue( + utils.sha1_matches(local_root_path + 'dir1\\file1_2.txt', + self.remote_base_dir + 'dir1/file1_2.txt')) + self.assertTrue( + utils.sha1_matches(local_root_path + 'dir2\\file2_1.txt', + self.remote_base_dir + 'dir2/file2_1.txt')) + self.assertTrue( + utils.sha1_matches(local_root_path + 'file0.txt', + self.remote_base_dir + 'file0.txt')) + + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is( + res, extraneous=1, extraneous_dir=2)) # file0.txt, dir1, dir2 + + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg, + '-r', '--delete') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=2)) + + utils.remove_test_directory(local_root_path + '\\dir1\\') + utils.remove_test_directory(local_root_path + '\\dir2\\') + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg, + '-r', '--delete') + self._assert_rsync_success(res) + self.assertTrue( + utils.files_count_is(res, matching=1, extraneous=3, extraneous_dir=2)) + self.assertFalse( + utils.does_directory_exist_remotely(self.remote_base_dir + 'dir1')) + self.assertFalse( + utils.does_directory_exist_remotely(self.remote_base_dir + 'dir2')) + + def test_uncompressed_no_empty_folders_with_backslash(self): + """Uploads multiple files uncompressed from a folder with a trailing backslash.""" + + self._do_test_no_empty_folders_with_backslash(compressed=False) + + def test_compressed_no_empty_folders_with_backslash(self): + """Uploads multiple files compressed from a folder with a trailing backslash.""" + + self._do_test_no_empty_folders_with_backslash(compressed=True) + + def test_uncompressed_with_empty_folders(self): + """Uploads and syncs multiple files uncompressed and empty folders.""" + + self._do_test_with_empty_folders(compressed=False) + + def test_compressed_with_empty_folders(self): + """Uploads and syncs multiple files compress and empty folders.""" + + self._do_test_with_empty_folders(compressed=True) + + def _do_test_with_empty_folders(self, compressed): + """Runs rsync with(out) -r for a non-trivial directory with empty folders. + + 1) Uploads a source directory with -r, checks sha1 hashes. + |-- rootdir + | |-- dir1 + | |-- emptydir2 + | |-- file1_1.txt + | |-- file1_2.txt + | |-- dir2 + | |-- file2_1.txt + | |-- emptydir1 + | |-- file0.txt + 2) Uploads the same source directory again without -r, + checks nothing has changed. The directory should be just skipped. + 3) Uploads the same source directory with --delete option and with -r. + Nothing should change. + 4) Removes dir1 and dir2 locally. + Uploads the same source directory with --delete option and with -r. + dir1 and dir2 should be removed from the remote instance. + + Args: + compressed (bool): Whether to append '--compress' or not. + """ + compressed_arg = '--compress' if compressed else None + local_root_path = self.local_base_dir + 'rootdir' + remote_root_path = self.remote_base_dir + 'rootdir/' + utils.create_test_file(local_root_path + '\\dir1\\file1_1.txt', 1024) + utils.create_test_file(local_root_path + '\\dir1\\file1_2.txt', 1024) + utils.create_test_directory(local_root_path + '\\dir1\\emptydir2\\') + utils.create_test_file(local_root_path + '\\dir2\\file2_1.txt', 1024) + utils.create_test_file(local_root_path + '\\file0.txt', 1024) + utils.create_test_directory(local_root_path + '\\emptydir1\\') + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg, + '-r') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=5)) + self.assertTrue( + utils.sha1_matches(local_root_path + '\\dir1\\file1_1.txt', + remote_root_path + 'dir1/file1_1.txt')) + self.assertTrue( + utils.sha1_matches(local_root_path + '\\dir1\\file1_2.txt', + remote_root_path + 'dir1/file1_2.txt')) + self.assertTrue( + utils.sha1_matches(local_root_path + '\\dir2\\file2_1.txt', + remote_root_path + 'dir2/file2_1.txt')) + self.assertTrue( + utils.sha1_matches(local_root_path + '\\file0.txt', + remote_root_path + 'file0.txt')) + + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, extraneous_dir=1)) + + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg, + '-r', '--delete') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=5)) + + utils.remove_test_directory(local_root_path + '\\dir1\\') + utils.remove_test_directory(local_root_path + '\\dir2\\') + res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg, + '-r', '--delete') + self._assert_rsync_success(res) + self.assertTrue( + utils.files_count_is( + res, matching=1, extraneous=3, matching_dir=2, extraneous_dir=3)) + self.assertIn('3/3 file(s) and 3/3 folder(s) deleted', res.stdout) + self.assertFalse( + utils.does_directory_exist_remotely(remote_root_path + 'dir1')) + self.assertFalse( + utils.does_directory_exist_remotely(remote_root_path + 'dir2')) + + def test_upload_empty_file(self): + """Uploads an empty file and checks sha1 signatures.""" + + empty_local_data_path = self.local_base_dir + 'emptyfile.dat' + empty_remote_data_path = self.remote_base_dir + 'emptyfile.dat' + utils.create_test_file(empty_local_data_path, 0) + res = utils.run_rsync(empty_local_data_path, self.remote_base_dir, None) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=1)) + self.assertTrue( + utils.sha1_matches(empty_local_data_path, empty_remote_data_path)) + + def test_upload_empty_folder_with_backslash(self): + """Uploads an empty folder with a trailing backslash.""" + + self._do_test_upload_empty_folder(with_backslash=True) + + def test_upload_empty_folder_no_backslash(self): + """Uploads an empty folder without a trailing backslash.""" + + self._do_test_upload_empty_folder(with_backslash=False) + + def _do_test_upload_empty_folder(self, with_backslash=False): + """Uploads an empty folder.""" + + local_data_dir = ( + self.local_base_dir + + 'empty_folder\\' if with_backslash else self.local_base_dir + + 'empty_folder') + res = utils.run_rsync(local_data_dir, self.remote_base_dir, None) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=0)) + + def test_whole_file_uncompressed(self): + """Uploads and syncs a file uncompressed with --whole-file.""" + + self._do_test_whole_file(compressed=False) + + def test_whole_file_compressed(self): + """Uploads and syncs a file compressed with --whole-file.""" + + self._do_test_whole_file(compressed=True) + + def _do_test_whole_file(self, compressed): + """Runs rsync 3 times with --whole-file -v options and validates results. + + 1) Uploads a file. + 2) Modifies the file and uploads it with --whole-file and -v options. + Checks the output contains C100%, not D100%. + 3) Modifies the file and uploads it with -W and -v options. + Checks the output contains C100%, not D100%. + + Args: + compressed (bool): Whether to append '--compress' or not. + """ + compressed_arg = '--compress' if compressed else None + + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, + compressed_arg) + + utils.create_test_file(self.local_data_path, 2534) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, + compressed_arg, '--whole-file', '-v') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, changed=1)) + self.assertIn('will be copied due to -W/--whole-file', str(res.stdout)) + self.assertIn('C100%', str(res.stdout)) + self.assertTrue( + utils.sha1_matches(self.local_data_path, self.remote_data_path)) + + utils.create_test_file(self.local_data_path, 3456) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, + compressed_arg, '-W', '-v') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, changed=1)) + self.assertIn('C100%', str(res.stdout)) + self.assertTrue( + utils.sha1_matches(self.local_data_path, self.remote_data_path)) + + def test_keep_file_permissions(self): + """Verifies that file permissions are kept for changed files.""" + + # Upload a file and check permissions. + utils.create_test_file(self.local_data_path, 1024) + utils.run_rsync(self.local_data_path, self.remote_base_dir) + ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path) + self.assertIn('-rw-r--r--', ls_res) + + # Add executable bit. + utils.get_ssh_command_output('chmod a+x %s*' % self.remote_data_path) + ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path) + self.assertIn('-rwxr-xr-x', ls_res) + + # Sync file again and verify permissions don't change. + utils.create_test_file(self.local_data_path, 1337) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, changed=1)) + ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path) + self.assertIn('-rwxr-xr-x', ls_res) + + def test_include_exclude(self): + """Verifies the --include and --exclude options.""" + + files = [ + 'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat', + 'folder1\\folder2\\file4.txt', 'folder3\\file5.txt' + ] + + for file in files: + utils.create_test_file(self.local_base_dir + file, 987) + + # Upload file2.txt and file3.dat. + res = utils.run_rsync(self.local_base_dir + '*', self.remote_base_dir, '-r', + '--include=*\\file2.txt', '--exclude=*.txt') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=3)) + self._assert_remote_dir_contains(['folder1/file2.txt', 'folder1/file3.dat']) + + # Upload all except *.dat with --delete, make sure file3.dat is kept. + utils.remove_test_file(self.local_base_dir + 'folder1\\file3.dat') + res = utils.run_rsync(self.local_base_dir + '*', self.remote_base_dir, '-r', + '--delete', '--exclude=*.dat') + self._assert_rsync_success(res) + self.assertTrue( + utils.files_count_is(res, missing=3, matching=1, matching_dir=3)) + self._assert_remote_dir_contains([ + 'file1.txt', 'folder1/file2.txt', 'folder1/file3.dat', + 'folder1/folder2/file4.txt', 'folder3/file5.txt' + ]) + + def test_exclude_include_from(self): + """Verifies the --include-from and --exclude-from options.""" + + files = [ + 'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat', + 'folder1\\folder2\\file4.txt', 'folder3\\file5.txt' + ] + + for file in files: + utils.create_test_file(self.local_base_dir + file, 987) + + include_file = self.local_base_dir + 'include.txt' + with open(include_file, 'wt') as f: + f.writelines(['file1.txt\n', 'folder3\\file5.txt']) + + exclude_file = self.local_base_dir + 'exclude.txt' + with open(exclude_file, 'wt') as f: + f.writelines(['*.txt']) + + res = utils.run_rsync('-r', '--include-from', include_file, + '--exclude-from', exclude_file, + self.local_base_dir + '*', self.remote_base_dir) + self.assertTrue(utils.files_count_is(res, missing=3, missing_dir=3)) + self._assert_remote_dir_contains( + ['file1.txt', 'folder1/file3.dat', 'folder3/file5.txt']) + + def test_files_from(self): + """Verifies the --files-from option.""" + + files = [ + 'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat', + 'folder1\\folder2\\file4.txt', 'folder3\\file5.txt' + ] + + for file in files: + utils.create_test_file(self.local_base_dir + file, 987) + + sources_file = self.local_base_dir + 'sources.txt' + with open(sources_file, 'wt') as f: + f.writelines([ + 'file1.txt\n', + '\n', + ' folder1\\file3.dat \n', + 'folder1\\.\\folder2\\file4.txt\n', # .\\ = rel path marker + ' folder3\\file5.txt\n', + '\n' + ]) + + res = utils.run_rsync('--files-from', sources_file, self.local_base_dir, + self.remote_base_dir) + self.assertTrue(utils.files_count_is(res, missing=4)) + self._assert_remote_dir_contains([ + 'file1.txt', 'folder1/file3.dat', 'folder2/file4.txt', + 'folder3/file5.txt' + ]) + + # Upload again to check that nothing changes. + res = utils.run_rsync('--files-from', sources_file, self.local_base_dir, + self.remote_base_dir) + self.assertTrue(utils.files_count_is(res, matching=4, extraneous_dir=3)) + + def test_checksum_file(self): + """Uploads and syncs a file with --checksum. + + 1) Uploads a file. + 2) Uploads a file with --checksum option. As the file was not changed, it + is recognized as matched. The output should contain D100%. + 3) Uploads the same file with --whole-file --checksum -v. + Checks the output contains C100%, not D100%. + 4) Modifies the file without changing its content. The file is + synchronized, the output should contain D100%. + + """ + + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir) + + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, + '--checksum', '-v') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, matching=1)) + self.assertIn('D100%', str(res.stdout)) + self.assertIn('will be synced due to -c/--checksum', str(res.stdout)) + + utils.create_test_file(self.local_data_path, 2534) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, + '--checksum', '-v', '--whole-file') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, changed=1)) + self.assertIn('C100%', str(res.stdout)) + self.assertIn('will be copied due to -c/--checksum and -W/--whole-file', + str(res.stdout)) + self.assertTrue( + utils.sha1_matches(self.local_data_path, self.remote_data_path)) + + utils.change_modified_time(self.local_data_path) + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-c', + '-v') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, changed=1)) + self.assertIn('D100%', str(res.stdout)) + + def test_sync_folder_when_remote_file_non_recursive(self): + """Non-recursively uploads a folder while there is a remote file with the same name.""" + + local_folder = self.local_base_dir + 'foldertocopy\\' + utils.create_test_directory(local_folder) + utils.get_ssh_command_output( + 'mkdir -p %s && touch %s' % + (self.remote_base_dir, self.remote_base_dir + 'foldertocopy')) + + res = utils.run_rsync(self.local_base_dir + 'foldertocopy', + self.remote_base_dir) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, extraneous=1)) + self.assertFalse( + utils.does_directory_exist_remotely(self.remote_base_dir + + 'foldertocopy')) + self.assertTrue( + utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy')) + + def test_sync_folder_when_remote_file_recursive_with_delete(self): + """Recursively uploads a folder while removing a remote file with the same name with --delete.""" + + local_folder = self.local_base_dir + 'foldertocopy\\' + utils.create_test_directory(local_folder) + utils.get_ssh_command_output( + 'mkdir -p %s && touch %s' % + (self.remote_base_dir, self.remote_base_dir + 'foldertocopy')) + + res = utils.run_rsync(self.local_base_dir + 'foldertocopy', + self.remote_base_dir, '-r', '--delete') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, extraneous=1, missing_dir=1)) + self.assertTrue( + utils.does_directory_exist_remotely(self.remote_base_dir + + 'foldertocopy')) + self.assertFalse( + utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy')) + self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout)) + + def test_sync_file_when_remote_folder_recursive_with_delete(self): + """Recursively uploads a file while removing a remote folder with the same name with --delete.""" + utils.create_test_file(self.local_data_path, 1024) + utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path) + + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, + '--delete', '-r') + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1)) + self.assertTrue( + utils.sha1_matches(self.local_data_path, self.remote_data_path)) + self.assertFalse(utils.does_directory_exist_remotely(self.remote_data_path)) + self.assertIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout)) + + def test_sync_file_when_remote_folder_empty_non_recursive(self): + """Non-recursively uploads a file while there is an empty remote folder with the same name.""" + self._do_test_sync_file_when_remote_folder_empty(recursive=False) + + def test_sync_file_when_remote_folder_empty_recursive(self): + """Recursively uploads a file while there is an empty remote folder with the same name.""" + self._do_test_sync_file_when_remote_folder_empty(recursive=True) + + def _do_test_sync_file_when_remote_folder_empty(self, recursive): + """Uploads a file while there is an empty remote folder with the same name. + + Args: + recursive (bool): Whether to append '-r' or not. + """ + flag = '-r' if recursive else None + utils.create_test_file(self.local_data_path, 1024) + utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path) + + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, flag) + self._assert_rsync_success(res) + self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1)) + self.assertTrue( + utils.sha1_matches(self.local_data_path, self.remote_data_path)) + self.assertFalse(utils.does_directory_exist_remotely(self.remote_data_path)) + self.assertNotIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout)) + + def test_sync_file_when_remote_folder_non_empty_non_recursive(self): + """Non-recursively uploads a file while there is a non-empty remote folder with the same name.""" + self._do_test_sync_file_when_remote_folder_non_empty(recursive=False) + + def test_sync_file_when_remote_folder_non_empty_recursive(self): + """Recursively uploads a file while there is a non-empty remote folder with the same name.""" + self._do_test_sync_file_when_remote_folder_non_empty(recursive=True) + + def _do_test_sync_file_when_remote_folder_non_empty(self, recursive): + """Uploads a file while there is a non-empty remote folder with the same name. + + Args: + recursive (bool): Whether to append '-r' or not. + """ + flag = '-r' if recursive else None + utils.create_test_file(self.local_data_path, 1024) + utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path) + utils.get_ssh_command_output( + 'mkdir -p %s && touch %s' % + (self.remote_base_dir, self.remote_data_path + '/file1.txt')) + + res = utils.run_rsync(self.local_data_path, self.remote_base_dir, flag) + self.assertIn('remove() failed: Directory not empty.', str(res.stderr)) + if recursive: + self.assertTrue( + utils.files_count_is(res, missing=1, extraneous=1, extraneous_dir=1)) + else: + self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1)) + self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path)) + self.assertTrue( + utils.does_file_exist_remotely(self.remote_data_path + '/file1.txt')) + self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path)) + + def test_upload_from_dot(self): + """Uploads files from the current directory ('.').""" + utils.create_test_file(self.local_base_dir + 'file1.txt', 1024) + utils.create_test_file(self.local_base_dir + 'dir\\file2.txt', 1024) + + prev_cwd = os.getcwd() + os.chdir(self.local_base_dir) + try: + # Uploading recursivly should pick up all files and dirs. + res = utils.run_rsync('.', self.remote_base_dir, '-r') + self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=1)) + self._assert_remote_dir_contains(['file1.txt', 'dir/file2.txt']) + + # Uploading again should not change anything. + res = utils.run_rsync('.', self.remote_base_dir, '-r') + self.assertTrue(utils.files_count_is(res, matching=2, matching_dir=1)) + + # Verify that non-recursive uploads do nothing. + res = utils.run_rsync('.', self.remote_base_dir) + self.assertTrue(utils.files_count_is(res, extraneous=1, extraneous_dir=1)) + finally: + os.chdir(prev_cwd) + + def test_upload_from_dotdot(self): + """Uploads files from the parent directory ('..').""" + utils.create_test_file(self.local_base_dir + 'file1.txt', 1024) + utils.create_test_file(self.local_base_dir + 'dir\\file2.txt', 1024) + + prev_cwd = os.getcwd() + os.chdir(self.local_base_dir + 'dir') + try: + # Uploading recursivly should pick up all files and dirs. + res = utils.run_rsync('..', self.remote_base_dir, '-r') + self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=1)) + self._assert_remote_dir_contains(['file1.txt', 'dir/file2.txt']) + + # Uploading again should not change anything. + res = utils.run_rsync('..', self.remote_base_dir, '-r') + self.assertTrue(utils.files_count_is(res, matching=2, matching_dir=1)) + + # Verify that non-recursive uploads do nothing. + res = utils.run_rsync('..', self.remote_base_dir) + self.assertTrue(utils.files_count_is(res, extraneous=1, extraneous_dir=1)) + finally: + os.chdir(prev_cwd) + + def test_existing(self): + """Runs rsync with --existing for a non-trivial directory. + + 1) Uploads a source directory with -r. + |-- rootdir + | |-- dir1 + | |-- emptydir2 + | |-- file1_1.txt + | |-- file1_2.txt -> rename to file1_3.txt (step 2) + | |-- (step2) emptydir3 + | |-- dir2 + | |-- file2_1.txt + | |-- emptydir1 -> rename emptydir4 (step 2) + | |-- file0.txt -> change (step 2) + 2) Add new files/folders, remove and change some files/folders. + 3) Uploads the same source directory with --existing option and with -r. + Only files existing on the server are changed, nothing is removed. + 4) Uploads the same source directory with --existing --delete -r. + Files non-existing on the server are deleted. + """ + local_root_path = self.local_base_dir + 'rootdir' + remote_root_path = self.remote_base_dir + 'rootdir/' + + files = [ + '\\dir1\\file1_1.txt', '\\dir1\\file1_2.txt', '\\dir2\\file2_1.txt', + '\\file0.txt' + ] + for file in files: + utils.create_test_file(local_root_path + file, 1024) + dirs = ['\\dir1\\emptydir2\\', '\\emptydir1\\'] + for directory in dirs: + utils.create_test_directory(local_root_path + directory) + res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r') + self._assert_rsync_success(res) + + utils.remove_test_file(local_root_path + '\\dir1\\file1_2.txt') + utils.create_test_file(local_root_path + '\\dir1\\file1_3.txt', 1024) + utils.create_test_directory(local_root_path + '\\dir1\\emptydir3\\') + utils.remove_test_directory(local_root_path + '\\emptydir1\\') + utils.create_test_directory(local_root_path + '\\emptydir4\\') + utils.create_test_file(local_root_path + '\\file0.txt', 2034) + + res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r', + '--existing') + self._assert_rsync_success(res) + self.assertTrue( + utils.files_count_is( + res, + missing=1, + missing_dir=2, + matching=2, + matching_dir=4, + changed=1, + extraneous=1, + extraneous_dir=1)) + self.assertTrue( + utils.does_directory_exist_remotely(remote_root_path + 'emptydir1')) + self.assertFalse( + utils.does_directory_exist_remotely(remote_root_path + 'emptydir4')) + self.assertFalse( + utils.does_directory_exist_remotely(remote_root_path + + 'dir1/emptydir3')) + self.assertTrue( + utils.does_file_exist_remotely(remote_root_path + 'dir1/file1_2.txt')) + self.assertFalse( + utils.does_file_exist_remotely(remote_root_path + 'dir1/file1_3.txt')) + + res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r', + '--existing', '--delete') + self._assert_rsync_success(res) + self.assertTrue( + utils.files_count_is( + res, + missing=1, + missing_dir=2, + matching=3, + matching_dir=4, + extraneous=1, + extraneous_dir=1)) + self.assertIn('1/1 file(s) and 1/1 folder(s) deleted', res.stdout) + self.assertFalse( + utils.does_directory_exist_remotely(remote_root_path + 'emptydir1')) + self.assertFalse( + utils.does_file_exist_remotely(remote_root_path + 'dir2/file1_2.txt')) + + def test_copy_dest(self): + r"""Runs rsync with --copy-dest option. + + Copies testdata.dat to + Copies the "cdc_rsync_e2e_test" package locally and syncs it with + --copy-dest. Verifies that the files are actually sync'ed (D), not + copied (C). + + Raises: + Exception: On timeout waiting for mount to appear (after 20 seconds) + """ + + copy_dest_dir = self.remote_base_dir + 'copy_dest_dir' + + utils.create_test_file(self.local_data_path, 1024) + res = utils.run_rsync(self.local_data_path, copy_dest_dir) + self._assert_rsync_success(res) + + # Upload package using --package. + res = utils.run_rsync('--copy-dest', copy_dest_dir, self.local_data_path, + self.remote_base_dir, '-v') + self._assert_rsync_success(res) + self.assertIn('D100%', res.stdout) + self.assertNotIn('C100%', res.stdout) + + def test_upload_executables(self): + """Uploads executable files and checks that they have the x bit set.""" + + # Use the cdc rsync binaries as test executables. + local_exe_path = utils.CDC_RSYNC_PATH + local_elf_path = os.path.join( + os.path.dirname(local_exe_path), 'cdc_rsync_server') + + remote_exe_path = self.remote_base_dir + os.path.basename(local_exe_path) + remote_elf_path = self.remote_base_dir + os.path.basename(local_elf_path) + + # Copy the files to the gamelet. + res = utils.run_rsync(local_exe_path, local_elf_path, self.remote_base_dir) + self._assert_rsync_success(res) + + # Check that both files have the executable bit set. + stats = utils.get_ssh_command_output('stat -c "%%a" %s %s' % + (remote_exe_path, remote_elf_path)) + self.assertEqual(stats.count('755'), 2, stats) + + # Remove executable bits. + utils.get_ssh_command_output('chmod -x %s %s' % + (remote_exe_path, remote_elf_path)) + + # Sync again, using -c to force a sync. + res = utils.run_rsync('-c', local_exe_path, local_elf_path, + self.remote_base_dir) + self._assert_rsync_success(res) + + # Validate that the executable bits were restored. + stats = utils.get_ssh_command_output('stat -c "%%a" %s %s' % + (remote_exe_path, remote_elf_path)) + self.assertEqual(stats.count('755'), 2, stats) + + def _run(self, args): + logging.debug('Running %s', ' '.join(args)) + res = subprocess.run(args, capture_output=True) + self.assertEqual(res.returncode, 0, 'Command failed: ' + str(res)) + res.stdout = res.stdout.decode('ascii') + logging.debug('\r\n%s', res.stdout) + return res + + +if __name__ == '__main__': + test_base.test_base.main() diff --git a/integration_tests/framework/__init__.py b/integration_tests/framework/__init__.py new file mode 100644 index 0000000..6d6d126 --- /dev/null +++ b/integration_tests/framework/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/integration_tests/framework/test_base.py b/integration_tests/framework/test_base.py new file mode 100644 index 0000000..cec88cd --- /dev/null +++ b/integration_tests/framework/test_base.py @@ -0,0 +1,63 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Test main and flags.""" + +import argparse +import contextlib +import logging +import sys +import unittest + +from integration_tests.framework import test_runner + + +class Flags(object): + binary_path = None + user_host = None + ssh_port = 22 + + +def main(): + parser = argparse.ArgumentParser(description='End-to-end integration test.') + parser.add_argument('--binary_path', help='Target [user@]host', required=True) + parser.add_argument('--user_host', help='Target [user@]host', required=True) + parser.add_argument( + '--ssh_port', + type=int, + help='SSH port for connecting to the host', + default=22) + parser.add_argument('--log_file', help='Log file path') + + # Capture all remaining arguments to pass to unittest.main(). + args, unittest_args = parser.parse_known_args() + Flags.binary_path = args.binary_path + Flags.user_host = args.user_host + Flags.ssh_port = args.ssh_port + + # Log to STDERR + log_format = ('%(levelname)-8s%(asctime)s ' + '%(filename)s:%(lineno)-3d %(message)s') + log_stream = sys.stderr + + if args.log_file: + log_stream = open(args.log_file, 'w') + + with log_stream: + logging.basicConfig( + format=log_format, level=logging.DEBUG, stream=log_stream) + + unittest.main( + argv=sys.argv[:1] + unittest_args, testRunner=test_runner.TestRunner()) diff --git a/integration_tests/framework/test_runner.py b/integration_tests/framework/test_runner.py new file mode 100644 index 0000000..b82d213 --- /dev/null +++ b/integration_tests/framework/test_runner.py @@ -0,0 +1,66 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Test runner, adds some sugar around logs to make them easier to read.""" + +import logging +import traceback +import unittest + + +class TestRunner(object): + """Runner producing test xml output.""" + + def run(self, test): # pylint: disable=invalid-name + result = TestResult() + logging.info('Running tests...') + test(result) + logging.info('\n\n******************* TESTS FINISHED *******************\n') + logging.info('Ran %d tests with %d errors and %d failures', result.testsRun, + len(result.errors), len(result.failures)) + for test_and_stack in result.failures: + logging.info('\n\n[ TEST FAILED ] %s\n', test_and_stack[0]) + logging.info( + '%s', test_and_stack[1].replace('\\\\r', + '\r').replace('\\\\n', '\n').replace( + '\\r', '\r').replace('\\n', '\n')) + + return result + + +class TestResult(unittest.TestResult): + + def startTest(self, test): + """Called when the given test is about to be run.""" + logging.info('\n\n===== BEGIN TEST CASE: %s =====\n', test) + unittest.TestResult.startTest(self, test) + + def stopTest(self, test): + """Called when the given test has been run.""" + unittest.TestResult.stopTest(self, test) + logging.info('\n\n===== END TEST CASE: %s =====\n', test) + + def addError(self, test, err): + unittest.TestResult.addError(self, test, err) + self._LogFailureInfo(err) + + def addFailure(self, test, err): + unittest.TestResult.addFailure(self, test, err) + self._LogFailureInfo(err) + + def _LogFailureInfo(self, err): + exctype, exc, tb = err + detail = ''.join(traceback.format_exception(exctype, exc, tb)) + logging.error('FAILURE: %s', detail) diff --git a/integration_tests/framework/utils.py b/integration_tests/framework/utils.py new file mode 100644 index 0000000..92d208c --- /dev/null +++ b/integration_tests/framework/utils.py @@ -0,0 +1,331 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Utils for file transfer tests.""" + +import hashlib +import logging +import os +import pathlib +import random +import shutil +import string +import subprocess +import time + +CDC_RSYNC_PATH = None +USER_HOST = None + +SHA1_LEN = 40 +SHA1_BUF_SIZE = 65536 +RANDOM = random.Random() + + +def initialize(cdc_rsync_path, user_host): + """Sets global variables.""" + global CDC_RSYNC_PATH, USER_HOST + + CDC_RSYNC_PATH = cdc_rsync_path + USER_HOST = user_host + + +def initialize_random(): + """Sets random seed.""" + global RANDOM + seed = int(time.time()) + logging.debug('Use random seed %i', seed) + RANDOM.seed(seed) + + +def _remove_carriage_return_lines(text): + r"""Removes *\r, keeps only *\r\n lines. + + Args: + text (string): Text to remove lines from (usually cdc_rsync output). + + Returns: + string: Text with lines removed. + """ + + # Some lines have \r\r\n, treat them properly. + ret = '' + for line in text.replace('\r\r', '\r').split('\r\n'): + ret += line.split('\r')[-1] + '\r\n' + return ret + + +def run_rsync(*args): + """Runs cdc_rsync with given args. + + The last positional argument is assumed to be the destination. The user/host + prefix [user@]host: is optional. If it does not have one, then it is prefixed + by |USER_HOST|:. + + Args: + *args (string): cdc_rsync arguments. + + Returns: + CompletedProcess: cdc_rsync process info with exit code and stdout/stderr. + """ + + # Prefix last positional argument with [user@]host: if it doesn't have such + # a prefix yet. Note that this won't work in all cases, e.g. if + # '--exclude', 'file' is passed. Use '--exclude=file' instead. + args_list = list(filter(None, args)) + for n in range(len(args_list) - 1, 0, -1): + if args_list[n][0] != '-' and not ':' in args_list[n]: + args_list[n] = USER_HOST + ":" + args_list[n] + break + + command = [CDC_RSYNC_PATH, *args_list] + + # Workaround issue with unicode logging. + logging.debug( + 'Executing %s ', + ' '.join(command).encode('utf-8').decode('ascii', 'backslashreplace')) + res = subprocess.run(command, capture_output=True) + # Remove lines ending with \r since those are temp display lines. + res.stdout = _remove_carriage_return_lines(res.stdout.decode('ascii')) + if res.stdout.strip(): + logging.debug('\r\n%s', res.stdout) + return res + + +def files_count_is(cdc_rsync_res, + missing=0, + missing_dir=0, + changed=0, + matching=0, + matching_dir=0, + extraneous=0, + extraneous_dir=0): + r"""Verifies that the output of cdc_rsync indicates the given file counts. + + Args: + cdc_rsync_res (CompletedProcess): Completed cdc_rsync process + missing (int, optional): Number of missing files. Defaults to 0. + missing_dir (int, optional): Number of missing folders. Defaults to 0. + changed (int, optional): Number of changed files. Defaults to 0. + matching (int, optional): Number of matching files. Defaults to 0. + matching_dir (int, optional): Number of matching folders. Defaults to 0. + extraneous (int, optional): Number of extraneous files. Defaults to 0. + extraneous_dir (int, optional): Number of extraneous folders. \ Defaults + to 0. + + Returns: + bool: True if all file counts match. + """ + missing_ok = '%i file(s) and %i folder(s) are not present' % ( + missing, missing_dir) in cdc_rsync_res.stdout + changed_ok = '%i file(s) changed' % (changed) in cdc_rsync_res.stdout + matching_ok = '%i file(s) and %i folder(s) match' % ( + matching, matching_dir) in cdc_rsync_res.stdout or """%i file(s) and %i \ +folder(s) have matching modified time and size""" % ( + matching, matching_dir) in cdc_rsync_res.stdout + extraneous_ok = """%i file(s) and %i folder(s) on the instance do not exist \ +on this machine""" % (extraneous, extraneous_dir) in cdc_rsync_res.stdout + return missing_ok and changed_ok and matching_ok and extraneous_ok + + +def sha1sum_local(filepath): + """Computes the sha1 hash of a local file. + + Args: + filepath (string): Path of the local (Windows) file + + Returns: + string: sha1 hash + """ + sha1 = hashlib.sha1() + with open(filepath, 'rb') as f: + while True: + data = f.read(SHA1_BUF_SIZE) + if not data: + break + sha1.update(data) + return sha1.hexdigest() + + +def sha1sum_remote(filepath): + """Computes the sha1 hash of a remote file. + + Args: + filepath (string): Path of the remote (Linux) file + + Returns: + string: sha1 hash + """ + return get_ssh_command_output('sha1sum %s' % filepath)[0:SHA1_LEN] + + +def sha1_matches(local_path, remote_path): + """Compares the sha1 hashes of a local and a remote file. + + Args: + local_path (string): Path of the local (Windows) file + remote_path (string): Path of the remote (Linux) file + + Returns: + bool: True if the sha1 hashes match + """ + + sha1_local = sha1sum_local(local_path) + sha1_remote = sha1sum_remote(remote_path) + return sha1_local == sha1_remote + + +def create_test_file(local_path, size, printable_data=True, append=False): + """Creates a test file with random text of given size. + + Args: + local_path (string): Local path of the file to create. + size (integer): Size of the file to create (bytes). + printable_data (bool, optional): If the data should be printable. Writing + a file with printable data is slower, for 1GB of data this takes ~5 + minutes, in comparison to ~2 seconds for non printable data. Defaults + to True. + append (bool, optional): If append mode should be used. Defaults to False. + """ + pathlib.Path(os.path.dirname(local_path)).mkdir(parents=True, exist_ok=True) + + mode = None + random_bytes = None + if printable_data: + mode = 'at' if append else 'wt' + random_bytes = ''.join( + RANDOM.choices(string.ascii_uppercase + string.digits, k=size)) + else: + mode = 'ab' if append else 'wb' + random_bytes = os.urandom(size) + + with open(local_path, mode) as f: + if size > 0: + f.write(random_bytes) + + +def remove_test_file(local_path): + """Deletes a test file. + + Args: + local_path (string): Local path of the file to delete. + """ + os.remove(local_path) + + +def create_test_directory(local_path): + """Creates a directory. + + Args: + local_path (string): Local path of the directory to create. + """ + pathlib.Path(os.path.dirname(local_path)).mkdir(parents=True, exist_ok=True) + + +def remove_test_directory(local_path): + """Removes a directory with its content. + + Args: + local_path (string): Local path of the directory to remove. + """ + shutil.rmtree(pathlib.Path(os.path.dirname(local_path)), ignore_errors=True) + + +def does_directory_exist_remotely(path): + """Checks if a directory exists on the remote instance. + + Args: + path (string): Path of the remote (Linux) directory + + Returns: + bool: True if a directory exists. + """ + return 'yes' in get_ssh_command_output('test -d %s && echo "yes"' % path) + + +def does_file_exist_remotely(path): + """Checks if a file exists on the remote instance. + + Args: + path (string): Path of the remote (Linux) file + + Returns: + bool: True if a file exists. + """ + return 'yes' in get_ssh_command_output('test -f %s && echo "yes"' % path) + + +def change_modified_time(path): + """Changes the modified time of the given file. + + Args: + path (string): Path of the local file + """ + stats = os.stat(path) + os.utime(path, (stats.st_atime, stats.st_mtime + 1)) + + +def get_ssh_command_output(cmd): + """Runs an SSH command using the command from the CDC_SSH_COMMAND env var. + + Args: + cmd (string): Command that is being run remotely + + Returns: + string: The output of the ssh command. + """ + ssh_command = os.environ.get('CDC_SSH_COMMAND') or "ssh" + full_ssh_cmd = '%s -tt "%s" -- %s' % (ssh_command, USER_HOST, + quote_argument(cmd)) + res = subprocess.run(full_ssh_cmd, capture_output=True) + if res.returncode != 0: + logging.warning('SSH command %s failed with code %i, stderr: %s', cmd, + res.returncode, res.stderr) + return res.stdout.decode('ascii', errors='replace') + + +def quote_argument(argument): + # This isn't fully generic, but does the job... It doesn't handle when the + # argument already escapes quotes, for instance. + return '"' + argument.replace('"', '\\"') + '"' + + +def get_sorted_files(remote_dir, pattern='"*.[t|d]*"'): + """Returns a sorted list of files in the remote_dir. + + Args: + remote_dir (string): Remote directory. + pattern (string, optional): Pattern for matching file names. + + Returns: + string: Sorted list of files found in the remote directory. + """ + find_res = get_ssh_command_output('cd %s && find -name %s -print' % + (remote_dir, pattern)) + + found = sorted( + filter(lambda item: item and item != '.', find_res.split('\r\n'))) + return found + + +def write_file(path, content): + """Writes a file and creates the parent directory if it does not exist yet. + + Args: + path (string): File path to create. + content (string): File content. + """ + pathlib.Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True) + with open(path, 'wt') as file: + file.write(content)