github-mirrors/netris-cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/nestriness/cdc-file-transfer.git synced 2026-05-01 18:23:07 +03:00
Code Issues Packages Projects Releases Wiki Activity

[cdc_rsync] Add integration tests (#42)

Browse Source 
[cdc_rsync] Add integration tests

This CL adds Python integration tests for cdc_rsync. To run the tests,
you need to supply a Linux host and proper configuration for cdc_rsync
to work:

  set CDC_SSH_COMMAND=C:\path\to\ssh.exe <args>
  set CDC_SCP_COMMAND=C:\path\to\scp.exe <args>
  C:\python38\python.exe -m integration_tests.cdc_rsync.all_tests --binary_path=C:\full\path\to\cdc_rsync.exe --user_host=user@host

Ran the tests and made sure they worked.
This commit is contained in:
Lutz Justen
2022-12-08 08:39:43 +01:00
committed by GitHub
parent d2b594a41d
commit 668c2ca8df
15 changed files with 2151 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
integration_tests/cdc_rsync/__init__.py Normal file
View File

@@ -0,0 +1,13 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

44
integration_tests/cdc_rsync/all_tests.py Normal file
View File

@@ -0,0 +1,44 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
import unittest
from integration_tests.cdc_rsync import connection_test
from integration_tests.cdc_rsync import deployment_test
from integration_tests.cdc_rsync import dry_run_test
from integration_tests.cdc_rsync import output_test
from integration_tests.cdc_rsync import upload_test
from integration_tests.framework import test_base
# pylint: disable=g-doc-args,g-doc-return-or-yield
def load_tests(loader, unused_tests, unused_pattern):
"""Customizes the list of test cases to run.
See the Python documentation for details:
https://docs.python.org/3/library/unittest.html#load-tests-protocol
"""
suite = unittest.TestSuite()
suite.addTests(loader.loadTestsFromModule(connection_test))
suite.addTests(loader.loadTestsFromModule(deployment_test))
suite.addTests(loader.loadTestsFromModule(dry_run_test))
suite.addTests(loader.loadTestsFromModule(output_test))
suite.addTests(loader.loadTestsFromModule(upload_test))
return suite
if __name__ == '__main__':
test_base.main()

131
integration_tests/cdc_rsync/connection_test.py Normal file
View File

@@ -0,0 +1,131 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""cdc_rsync connection test."""
from concurrent import futures
import socket
import time
from integration_tests.framework import utils
from integration_tests.cdc_rsync import test_base
RETURN_CODE_SUCCESS = 0
RETURN_CODE_GENERIC_ERROR = 1
RETURN_CODE_CONNECTION_TIMEOUT = 2
RETURN_CODE_ADDRESS_IN_USE = 4
FIRST_PORT = 44450
LAST_PORT = 44459
class ConnectionTest(test_base.CdcRsyncTest):
"""cdc_rsync connection test class."""
def test_valid_instance(self):
"""Runs rsync with --instance option for a valid id.
1) Uploads a file with --instance option instead of --ip --port.
2) Checks the file exists on the used instance.
"""
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
self._assert_rsync_success(res)
self.assertTrue(utils.does_file_exist_remotely(self.remote_data_path))
def test_invalid_instance(self):
"""Runs rsync with --instance option for an invalid id.
1) Uploads a file with --instance option for a non-existing id.
2) Checks the error message.
"""
bad_host = 'bad_host'
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path,
bad_host + ":" + self.remote_base_dir)
self.assertEqual(res.returncode, RETURN_CODE_GENERIC_ERROR)
self.assertIn('lost connection', str(res.stderr))
def test_contimeout(self):
"""Runs rsync with --contimeout option for an invalid ip.
1) Uploads a file with bad IP address.
2) Checks the error message and that it timed out after ~5 seconds.
3) Uploads a file with bad IP address and --contimeout 1.
4) Checks the error message and that it timed out after ~1 second.
"""
utils.create_test_file(self.local_data_path, 1024)
bad_host = '192.0.2.1'
start = time.time()
res = utils.run_rsync(self.local_data_path,
bad_host + ":" + self.remote_base_dir)
elapsed_time = time.time() - start
self.assertGreater(elapsed_time, 4.5)
self.assertEqual(res.returncode, RETURN_CODE_CONNECTION_TIMEOUT)
self.assertIn('Error: Server connection timed out', str(res.stderr))
start = time.time()
res = utils.run_rsync(self.local_data_path,
bad_host + ":" + self.remote_base_dir,
'--contimeout=1')
elapsed_time = time.time() - start
self.assertLess(elapsed_time, 3)
self.assertEqual(res.returncode, RETURN_CODE_CONNECTION_TIMEOUT)
self.assertIn('Error: Server connection timed out', str(res.stderr))
def test_multiple_instances(self):
"""Runs multiple instances of rsync at the same time."""
num_instances = LAST_PORT - FIRST_PORT + 1
local_data_paths = []
for n in range(num_instances):
path = self.local_base_dir + ('testdata_%i.dat' % n)
utils.create_test_file(path, 1024)
local_data_paths.append(path)
with futures.ThreadPoolExecutor(max_workers=num_instances) as executor:
res = []
for n in range(num_instances):
res.append(
executor.submit(utils.run_rsync, local_data_paths[n],
self.remote_base_dir))
for r in res:
self._assert_rsync_success(r.result())
def test_address_in_use(self):
"""Blocks all ports and checks that rsync fails with the expected error."""
sockets = []
try:
# Occupy all ports.
for port in range(FIRST_PORT, LAST_PORT + 1):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sockets.append(s)
s.bind(('127.0.0.1', port))
s.listen()
# rsync shouldn't be able to find an available port now.
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
self.assertIn('All ports are already in use', str(res.stderr))
finally:
for s in sockets:
s.close()
if __name__ == '__main__':
test_base.test_base.main()

110
integration_tests/cdc_rsync/deployment_test.py Normal file
View File

@@ -0,0 +1,110 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""cdc_rsync deployment test."""
from integration_tests.framework import utils
from integration_tests.cdc_rsync import test_base
REMOTE_FOLDER = '~/.cache/cdc-file-transfer/bin/'
class DeploymentTest(test_base.CdcRsyncTest):
"""cdc_rsync deployment test class."""
def _assert_deployment(self, initial_ts, file, msg):
"""Checks rsync and library are uploaded and the given file's timestamp matches initial_ts."""
res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
self._assert_rsync_success(res)
self.assertIn(msg, str(res.stdout))
changed_ts = utils.get_ssh_command_output('stat --format=%%y %s' %
REMOTE_FOLDER + file)
self.assertEqual(initial_ts, changed_ts)
def _change_file_preserve_timestamp(self, file):
"""Changes a file preserving it timestamp."""
utils.get_ssh_command_output(
'touch -r %s %s' %
(REMOTE_FOLDER + file, REMOTE_FOLDER + file + '.tmp'))
utils.get_ssh_command_output('truncate -s +100 %s' % REMOTE_FOLDER + file)
utils.get_ssh_command_output(
'touch -r %s %s' %
(REMOTE_FOLDER + file + '.tmp', REMOTE_FOLDER + file))
utils.get_ssh_command_output('rm %s' % (REMOTE_FOLDER + file + '.tmp'))
def test_no_server(self):
"""Checks that cdc_rsync_server is uploaded if not present on the gamelet.
1) Wipes ‘/opt/developer/tools/bin/’ on the gamelet.
2) Uploads a file.
3) Verifies that cdc_rsync_server exists in that folder.
"""
utils.get_ssh_command_output('rm -rf %s*' % REMOTE_FOLDER)
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
self._assert_rsync_success(res)
self.assertIn('Server not deployed. Deploying...', str(res.stdout))
self._assert_remote_dir_contains(['cdc_rsync_server'],
remote_dir=REMOTE_FOLDER,
pattern='"*"')
def test_modified_server(self):
"""Checks that cdc_rsync_server is re-uploaded.
1) Touches cdc_rsync_server in ‘REMOTE_FOLDER’.
2) Uploads a file.
3) Verifies that cdc_rsync_server is re-uploaded.
4) Appends a few bytes to cdc_rsync_server while keeping its timestamp.
6) Uploads a file.
7) Verifies that cdc_rsync_server is re-uploaded.
"""
# To be sure that cdc_rsync_server exist on the remote system
# do an "empty" copy.
utils.run_rsync(self.local_base_dir, self.remote_base_dir)
remote_server_path = REMOTE_FOLDER + 'cdc_rsync_server'
initial_ts = utils.get_ssh_command_output('stat --format=%%y %s' %
remote_server_path)
utils.get_ssh_command_output('touch -d \'1 November 2020 00:00\' %s' %
remote_server_path)
changed_ts = utils.get_ssh_command_output('stat --format=%%y %s' %
remote_server_path)
self.assertNotEqual(initial_ts, changed_ts)
utils.create_test_file(self.local_data_path, 1024)
self._assert_deployment(initial_ts, 'cdc_rsync_server',
'Server outdated. Redeploying...')
self._change_file_preserve_timestamp('cdc_rsync_server')
self._assert_deployment(initial_ts, 'cdc_rsync_server',
'Server outdated. Redeploying...')
def test_read_only_server(self):
"""Checks that cdc_rsync_server is overwritten if it is read-only."""
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
self._assert_rsync_success(res)
# Modify cdc_rsync_server and wipe permissions.
remote_server_path = REMOTE_FOLDER + 'cdc_rsync_server'
utils.get_ssh_command_output('echo "xxx" > %s && chmod 0 %s' %
(remote_server_path, remote_server_path))
res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
self._assert_rsync_success(res)
self.assertIn('Server failed to start. Redeploying...', str(res.stdout))
if __name__ == '__main__':
test_base.test_base.main()

116
integration_tests/cdc_rsync/dry_run_test.py Normal file
View File

@@ -0,0 +1,116 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""cdc_rsync dry-run test."""
from integration_tests.framework import utils
from integration_tests.cdc_rsync import test_base
class DryRunTest(test_base.CdcRsyncTest):
"""cdc_rsync dry-run test class."""
def test_dry_run(self):
"""Verifies --dry-run option.
1) Uploads file1.txt and file2.txt.
2) Modifies file2.txt.
3) Dry-runs file2.txt and file3.txt with --dry-run -r --delete.
Result: a missing (file3.txt), a changed (file2.txt) and an extraneous
(file1.txt) file. No files should be changed on the server.
"""
files = ['file1.txt', 'file2.txt', 'file3.txt']
for file in files:
utils.create_test_file(self.local_base_dir + file, 987)
res = utils.run_rsync(self.local_base_dir + 'file1.txt',
self.local_base_dir + 'file2.txt',
self.remote_base_dir, '-v')
self._assert_rsync_success(res)
self._assert_remote_dir_contains(['file1.txt', 'file2.txt'])
# Dry-run of uploading changed/new/to delete files.
utils.create_test_file(self.local_base_dir + 'file2.txt', 2534)
res = utils.run_rsync(self.local_base_dir + 'file2.txt',
self.local_base_dir + 'file3.txt',
self.remote_base_dir, '-v', '--dry-run', '--delete',
'-r')
self._assert_rsync_success(res)
self.assertTrue(
utils.files_count_is(res, missing=1, changed=1, extraneous=1))
self._assert_remote_dir_does_not_contain(['file3.txt'])
self._assert_remote_dir_contains(['file1.txt', 'file2.txt'])
self.assertIn('file1.txt', str(res.stdout))
self.assertIn('deleted 1 / 1', str(res.stdout))
self.assertIn('file2.txt', str(res.stdout))
self.assertIn('D100%', str(res.stdout))
self.assertIn('file3.txt', str(res.stdout))
self.assertIn('C100%', str(res.stdout))
self.assertFalse(
utils.sha1_matches(self.local_base_dir + 'file2.txt',
self.remote_base_dir + 'file2.txt'))
def test_dry_run_sync_folder_when_remote_file_recursive_with_delete(self):
"""Dry-runs a recursive upload of a folder while removing a remote file with the same name with --delete."""
local_folder = self.local_base_dir + 'foldertocopy\\'
utils.create_test_directory(local_folder)
utils.get_ssh_command_output(
'mkdir -p %s && touch %s' %
(self.remote_base_dir, self.remote_base_dir + 'foldertocopy'))
res = utils.run_rsync(self.local_base_dir + 'foldertocopy',
self.remote_base_dir, '-r', '--dry-run', '--delete')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, extraneous=1, missing_dir=1))
self.assertFalse(
utils.does_directory_exist_remotely(self.remote_base_dir +
'foldertocopy'))
self.assertTrue(
utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy'))
self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout))
def test_dry_run_sync_file_when_remote_folder_recursive_with_delete(self):
"""Dry-runs a recursive upload of a file while removing an empty remote folder with the same name with --delete."""
utils.create_test_file(self.local_data_path, 1024)
utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
'--dry-run', '-r', '--delete')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path))
self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path))
self.assertIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
def test_dry_run_sync_file_when_remote_folder_empty(self):
"""Dry-runs a non-recursive upload of a file while there is an empty remote folder with the same name."""
utils.create_test_file(self.local_data_path, 1024)
utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
'--dry-run')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path))
self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path))
self.assertNotIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
if __name__ == '__main__':
test_base.test_base.main()

243
integration_tests/cdc_rsync/output_test.py Normal file
View File

@@ -0,0 +1,243 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""cdc_rsync output test."""
import json
from integration_tests.framework import utils
from integration_tests.cdc_rsync import test_base
class OutputTest(test_base.CdcRsyncTest):
"""cdc_rsync output test class."""
def test_plain(self):
"""Runs rsync and verifies the total progress.
1) Uploads a file, verifies that the total progress is shown.
2) Uploads an empty folder with -r --delete options.
Verifies that the total delete messages are shown.
"""
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
self._assert_rsync_success(res)
self.assertIn('100% TOT', str(res.stdout))
utils.remove_test_file(self.local_data_path)
res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r',
'--delete')
self._assert_rsync_success(res)
self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout))
def test_verbose_1(self):
"""Runs rsync with -v option for multiple files.
1) Uploads 3 files with ‘-v’.
Verifies that each file is listed in the output as ‘C100%’.
2) Modifies 3 files, uploads them again with ‘--v’.
Verifies that each file is listed in the output as ‘D100%’.
3) Uploads an empty folder with -r --delete options.
Verifies that the delete messages are shown.
"""
files = ['file1. txt', 'file2.txt', 'file3.txt']
for file in files:
utils.create_test_file(self.local_base_dir + file, 1024)
res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r')
self._assert_rsync_success(res)
self.assertEqual(3, str(res.stdout).count('C100%'))
for file in files:
utils.create_test_file(self.local_base_dir + file, 2048)
res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r')
self._assert_rsync_success(res)
self.assertEqual(3, str(res.stdout).count('D100%'))
for file in files:
utils.remove_test_file(self.local_base_dir + file)
res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r',
'--delete')
self._assert_rsync_success(res)
self.assertIn('will be deleted due to --delete', str(res.stdout))
self.assertIn('3/3 file(s) and 0/0 folder(s) deleted', str(res.stdout))
def test_verbose_2(self):
"""Runs rsync with -vv option.
1) Uploads a file with ‘-vv’.
2) Verifies that additional logs show up.
"""
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vv')
self._assert_rsync_success(res)
output = str(res.stdout)
# client-side output
self._assert_regex('Starting process', output)
self._assert_not_regex(
r'process\.cc\([0-9]+\): Start\(\): Starting process', output)
# server-side output
self._assert_regex(
'INFO Finding all files in destination folder '
f"'{self.remote_base_dir}'", output)
self.assertNotIn('DEBUG', output)
def test_verbose_3(self):
"""Runs rsync with -vvv option.
1) Uploads a file with ‘-vvv’.
Verifies that additional logs show up (LOG_DEBUG logs).
2) Uploads a file to ‘/invalid’ with ‘-vvv’.
Verifies that error messages including filenames are shown.
"""
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vvv')
self._assert_rsync_success(res)
output = str(res.stdout)
# client-side output
self._assert_regex(
r'cdc_rsync_client\.cc\([0-9]+\): SendOptions\(\): Sending options',
output)
# server-side output
self._assert_regex(
r'DEBUG server_socket\.cc\([0-9]+\): Receive\(\): EOF\(\) detected',
output)
# TODO: Add a check here, as currently the output is misleading
# res = utils.run_rsync(self.local_data_path, '/invalid', '-vvv')
def test_verbose_4(self):
"""Runs rsync with -vvv option.
1) Uploads a file with ‘-vvvv’.
2) Verifies that additional logs show up (LOG_VERBOSE logs).
"""
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vvvv')
self._assert_rsync_success(res)
output = str(res.stdout)
# client-side output
self._assert_regex(
r'message_pump\.cc\([0-9]+\): ThreadDoSendPacket\(\): Sent packet of size',
output)
# server-side output
self._assert_regex(
r'VERBOSE message_pump\.cc\([0-9]+\): ThreadDoReceivePacket\(\): Received packet of size',
output)
def test_quiet(self):
"""Runs rsync with -q option.
1) Uploads a file with ‘-q’.
2) Verifies that no output is shown.
"""
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-q')
self._assert_rsync_success(res)
self.assertEqual('\r\n', res.stdout)
def test_quiet_error(self):
"""Runs rsync with -q option still showing errors.
1) Uploads a file with ‘-q’ and bad options.
2) Verifies that an error message is shown.
"""
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-q',
'-t')
self.assertEqual(res.returncode, 1)
self.assertEqual('\r\n', str(res.stdout))
self.assertIn('Unknown option: \'t\'', str(res.stderr))
# TODO: Add a test case for the non-existing destination.
def test_existing_verbose_1(self):
"""Runs rsync with -v --existing."""
files = ['file1.txt', 'file2.txt']
for file in files:
utils.create_test_file(self.local_base_dir + file, 1024)
res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r')
self._assert_rsync_success(res)
files.append('file3.txt')
for file in files:
utils.create_test_file(self.local_base_dir + file, 2048)
res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r',
'--existing')
self._assert_rsync_success(res)
output = str(res.stdout)
self.assertEqual(2, output.count('D100%'))
self.assertNotIn('file3.txt', output)
def test_json_per_file(self):
"""Runs rsync with -v --json."""
local_path = self.local_base_dir + 'test.txt'
utils.create_test_file(local_path, 1024)
res = utils.run_rsync(local_path, self.remote_base_dir, '-v', '--json')
self._assert_rsync_success(res)
output = str(res.stdout)
for val in self.parse_json(output):
self.assertEqual(val['file'], 'test.txt')
self.assertEqual(val['operation'], 'Copy')
self.assertEqual(val['size'], 1024)
# Those are actually all floats, but sometimes they get rounded to ints.
self.assertTrue(self.is_float_or_int(val['bytes_per_second']))
self.assertTrue(self.is_float_or_int(val['duration']))
self.assertTrue(self.is_float_or_int(val['eta']))
self.assertTrue(self.is_float_or_int(val['total_duration']))
self.assertTrue(self.is_float_or_int(val['total_eta']))
self.assertTrue(self.is_float_or_int(val['total_progress']))
def test_json_total(self):
"""Runs rsync with --json."""
local_path = self.local_base_dir + 'test.txt'
utils.create_test_file(local_path, 1024)
res = utils.run_rsync(local_path, self.remote_base_dir, '--json')
self._assert_rsync_success(res)
output = str(res.stdout)
for val in self.parse_json(output):
self.assertNotIn('file', val)
# Those are actually all floats, but sometimes they get rounded to ints.
self.assertTrue(self.is_float_or_int(val['total_duration']))
self.assertTrue(self.is_float_or_int(val['total_eta']))
self.assertTrue(self.is_float_or_int(val['total_progress']))
def parse_json(self, output):
"""Parses the JSON lines of output."""
lines = output.split('\r\n')
json_values = []
for line in lines:
if str.startswith(line, '{'):
json_values.append(json.loads(line.strip()))
return json_values
def is_float_or_int(self, val):
"""Returns true if val is a float or an int."""
return isinstance(val, float) or isinstance(val, int)
if __name__ == '__main__':
test_base.test_base.main()

113
integration_tests/cdc_rsync/test_base.py Normal file
View File

@@ -0,0 +1,113 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""cdc_rsync base test class."""
import datetime
import logging
import tempfile
import re
import unittest
from integration_tests.framework import utils
from integration_tests.framework import test_base
class CdcRsyncTest(unittest.TestCase):
"""cdc_rsync base test class."""
tmp_dir = None
local_base_dir = None
remote_base_dir = None
local_data_path = None
remote_data_path = None
def setUp(self):
"""Cleans up the remote test data folder, logs a marker, and initializes random."""
super(CdcRsyncTest, self).setUp()
logging.debug('CdcRsyncTest -> setUp')
utils.initialize(test_base.Flags.binary_path, test_base.Flags.user_host)
now_str = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
self.tmp_dir = tempfile.TemporaryDirectory(
prefix=f'_cdc_rsync_test_{now_str}')
self.local_base_dir = self.tmp_dir.name + '\\'
self.remote_base_dir = f'/tmp/_cdc_rsync_test_{now_str}/'
self.local_data_path = self.local_base_dir + 'testdata.dat'
self.remote_data_path = self.remote_base_dir + 'testdata.dat'
logging.info('Local base dir: "%s"', self.local_base_dir)
logging.info('Remote base dir: "%s"', self.remote_base_dir)
utils.initialize_random()
def tearDown(self):
"""Cleans up the local and remote temp directories."""
super(CdcRsyncTest, self).tearDown()
logging.debug('CdcRsyncTest -> tearDown')
self.tmp_dir.cleanup()
utils.get_ssh_command_output(f'rm -rf {self.remote_base_dir}')
def _assert_rsync_success(self, res):
"""Asserts if the return code is 0 and outputs return message with args."""
self.assertEqual(res.returncode, 0, 'Return value is ' + str(res))
def _assert_regex(self, regex, value):
"""Asserts that the regex string matches the given value."""
self.assertIsNotNone(
re.search(regex, value), f'"Regex {regex}" does not match "{value}"')
def _assert_not_regex(self, regex, value):
"""Asserts that the regex string does not match the given value."""
self.assertIsNone(
re.search(regex, value),
f'"Regex {regex}" unexpectedly matches "{value}"')
def _assert_remote_dir_contains(self,
file_list,
remote_dir=None,
pattern='"*.[t|d]*"'):
"""Asserts that the remote base dir contains exactly the list of files.
Args:
file_list (list of strings): List of relative file paths to check
remote_dir (string, optional): Remote directory. Defaults to
remote_base_dir
pattern (string, optional): Pattern for matching file names.
"""
find_res = utils.get_ssh_command_output(
'cd %s && find -name %s -print' %
(remote_dir or self.remote_base_dir, pattern))
# Note that assertCountEqual compares items independently of order
# (not just the size of the list).
found = sorted(
filter(lambda item: item and item != '.', find_res.split('\r\n')))
expected = sorted(['./' + f for f in file_list])
self.assertListEqual(found, expected)
def _assert_remote_dir_does_not_contain(self, file_list):
"""Asserts that the remote base dir contains none of the listed files.
Args:
file_list (list of strings): List of relative file paths to check
"""
find_res = utils.get_ssh_command_output(
'cd %s && find -name "*.[t|d]*" -print' % self.remote_base_dir)
found = set(file_name for file_name in filter(None, find_res.split('\n')))
for file in file_list:
self.assertNotIn('./' + file, found)

894
integration_tests/cdc_rsync/upload_test.py Normal file
View File

@@ -0,0 +1,894 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""cdc_rsync upload test."""
import json
import logging
import os
import subprocess
import time
from integration_tests.framework import utils
from integration_tests.cdc_rsync import test_base
class UploadTest(test_base.CdcRsyncTest):
"""cdc_rsync upload test class."""
def test_single_uncompressed(self):
"""Uploads and syncs a file uncompressed."""
self._do_test_single(compressed=False)
def test_upload_compressed(self):
"""Uploads and syncs a file compressed."""
self._do_test_single(compressed=True)
def _do_test_single(self, compressed):
"""Runs rsync 3 times and validates results.
1) Uploads a file, checks sha1 hashes.
2) Uploads the same file again, checks nothing changed.
3) Modifies the file and uploads again. Checks sha1 hashes.
Args:
compressed (bool): Whether to append '--compress' or not.
"""
compressed_arg = '--compress' if compressed else None
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
compressed_arg)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=1))
self.assertTrue(
utils.sha1_matches(self.local_data_path, self.remote_data_path))
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
compressed_arg)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, matching=1))
utils.create_test_file(self.local_data_path, 2534)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
compressed_arg)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, changed=1))
self.assertTrue(
utils.sha1_matches(self.local_data_path, self.remote_data_path))
def test_backslash_in_dest_folder(self):
r"""Verifies uploading to \mnt\developer."""
filepath = os.path.join(self.local_base_dir, 'file1.txt')
utils.create_test_file(filepath, 1)
res = utils.run_rsync(filepath, self.remote_base_dir.replace('/', '\\'))
self.assertTrue(utils.files_count_is(res, missing=1))
self._assert_remote_dir_contains(['file1.txt'])
def test_backslash_in_source_folder(self):
r"""Verifies uploading from /source/folder."""
filepath = os.path.join(self.local_base_dir, 'file1.txt')
utils.create_test_file(filepath, 1)
filepath = filepath.replace('\\', '/')
res = utils.run_rsync(filepath, self.remote_base_dir)
self.assertTrue(utils.files_count_is(res, missing=1))
self._assert_remote_dir_contains(['file1.txt'])
def test_single_unicode(self):
"""Uploads a file with a non-ascii unicode path and checks sha1 signatures."""
nonascii_local_data_path = self.local_base_dir + '⛽⛽⛽⛽⛽⛽⛽⛽.dat'
nonascii_remote_data_path = self.remote_base_dir + '⛽⛽⛽⛽⛽⛽⛽⛽.dat'
utils.create_test_file(nonascii_local_data_path, 1024)
# In order to check that non-ascii characters are not considered as
# wildcard
# ? characters, create a second file. Only 1 file should be uploaded.
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(nonascii_local_data_path, self.remote_base_dir, None)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=1))
self.assertTrue(
utils.sha1_matches(nonascii_local_data_path, nonascii_remote_data_path))
def test_uncompressed_no_empty_folders(self):
"""Uploads and syncs multiple files uncompressed in different folders."""
self._do_test_no_empty_folders(compressed=False)
def test_compressed_no_empty_folders(self):
"""Uploads and syncs multiple files compressed in different folders."""
self._do_test_no_empty_folders(compressed=True)
def _do_test_no_empty_folders(self, compressed):
"""Runs rsync with(out) -r for a non-trivial directory and validates results.
1) Uploads a source directory with -r, checks sha1 hashes.
|-- rootdir
| |-- dir1
| |-- file1_1.txt
| |-- file1_2.txt
| |-- dir2
| |-- file2_1.txt
| |-- file0.txt
2) Uploads the same source directory again without -r,
checks nothing has changed. The directory should be just skipped.
3) Uploads the same source directory with --delete option and with -r.
Nothing should change.
4) Removes dir1 and dir2 locally.
Uploads the same source directory with --delete option and with -r.
dir1 and dir2 should be removed from the remote instance.
Args:
compressed (bool): Whether to append '--compress' or not.
"""
compressed_arg = '--compress' if compressed else None
local_root_path = self.local_base_dir + 'rootdir'
remote_root_path = self.remote_base_dir + 'rootdir/'
utils.create_test_file(local_root_path + '\\dir1\\file1_1.txt', 1024)
utils.create_test_file(local_root_path + '\\dir1\\file1_2.txt', 1024)
utils.create_test_file(local_root_path + '\\dir2\\file2_1.txt', 1024)
utils.create_test_file(local_root_path + '\\file0.txt', 1024)
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
'-r')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=3))
self.assertTrue(
utils.sha1_matches(local_root_path + '\\dir1\\file1_1.txt',
remote_root_path + 'dir1/file1_1.txt'))
self.assertTrue(
utils.sha1_matches(local_root_path + '\\dir1\\file1_2.txt',
remote_root_path + 'dir1/file1_2.txt'))
self.assertTrue(
utils.sha1_matches(local_root_path + '\\dir2\\file2_1.txt',
remote_root_path + 'dir2/file2_1.txt'))
self.assertTrue(
utils.sha1_matches(local_root_path + '\\file0.txt',
remote_root_path + 'file0.txt'))
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, extraneous_dir=1))
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
'-r', '--delete')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=3))
utils.remove_test_directory(local_root_path + '\\dir1\\')
utils.remove_test_directory(local_root_path + '\\dir2\\')
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
'-r', '--delete')
self._assert_rsync_success(res)
self.assertTrue(
utils.files_count_is(
res, matching=1, extraneous=3, matching_dir=1, extraneous_dir=2))
self.assertFalse(
utils.does_directory_exist_remotely(remote_root_path + 'dir1'))
self.assertFalse(
utils.does_directory_exist_remotely(remote_root_path + 'dir2'))
def _do_test_no_empty_folders_with_backslash(self, compressed):
"""Runs rsync with(out) -r for a non-trivial directory with a trailing backslash.
1) Uploads a source directory with -r, checks sha1 hashes.
Everything from rootdir should be copied except rootdir itself.
|-- rootdir
| |-- dir1
| |-- file1_1.txt
| |-- file1_2.txt
| |-- dir2
| |-- file2_1.txt
| |-- file0.txt
2) Uploads the same source directory again without -r,
checks nothing has changed. The directory should be just skipped.
3) Uploads the same source directory with --delete option and with -r.
Nothing should change.
4) Removes dir1 and dir2 locally.
Uploads the same source directory with --delete option and with -r.
dir1 and dir2 should be removed from the remote instance.
Args:
compressed (bool): Whether to append '--compress' or not.
"""
compressed_arg = '--compress' if compressed else None
local_root_path = self.local_base_dir + 'rootdir\\'
utils.create_test_file(local_root_path + 'dir1\\file1_1.txt', 1024)
utils.create_test_file(local_root_path + 'dir1\\file1_2.txt', 1024)
utils.create_test_file(local_root_path + 'dir2\\file2_1.txt', 1024)
utils.create_test_file(local_root_path + 'file0.txt', 1024)
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
'-r')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=2))
self.assertTrue(
utils.sha1_matches(local_root_path + 'dir1\\file1_1.txt',
self.remote_base_dir + 'dir1/file1_1.txt'))
self.assertTrue(
utils.sha1_matches(local_root_path + 'dir1\\file1_2.txt',
self.remote_base_dir + 'dir1/file1_2.txt'))
self.assertTrue(
utils.sha1_matches(local_root_path + 'dir2\\file2_1.txt',
self.remote_base_dir + 'dir2/file2_1.txt'))
self.assertTrue(
utils.sha1_matches(local_root_path + 'file0.txt',
self.remote_base_dir + 'file0.txt'))
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(
res, extraneous=1, extraneous_dir=2)) # file0.txt, dir1, dir2
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
'-r', '--delete')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=2))
utils.remove_test_directory(local_root_path + '\\dir1\\')
utils.remove_test_directory(local_root_path + '\\dir2\\')
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
'-r', '--delete')
self._assert_rsync_success(res)
self.assertTrue(
utils.files_count_is(res, matching=1, extraneous=3, extraneous_dir=2))
self.assertFalse(
utils.does_directory_exist_remotely(self.remote_base_dir + 'dir1'))
self.assertFalse(
utils.does_directory_exist_remotely(self.remote_base_dir + 'dir2'))
def test_uncompressed_no_empty_folders_with_backslash(self):
"""Uploads multiple files uncompressed from a folder with a trailing backslash."""
self._do_test_no_empty_folders_with_backslash(compressed=False)
def test_compressed_no_empty_folders_with_backslash(self):
"""Uploads multiple files compressed from a folder with a trailing backslash."""
self._do_test_no_empty_folders_with_backslash(compressed=True)
def test_uncompressed_with_empty_folders(self):
"""Uploads and syncs multiple files uncompressed and empty folders."""
self._do_test_with_empty_folders(compressed=False)
def test_compressed_with_empty_folders(self):
"""Uploads and syncs multiple files compress and empty folders."""
self._do_test_with_empty_folders(compressed=True)
def _do_test_with_empty_folders(self, compressed):
"""Runs rsync with(out) -r for a non-trivial directory with empty folders.
1) Uploads a source directory with -r, checks sha1 hashes.
|-- rootdir
| |-- dir1
| |-- emptydir2
| |-- file1_1.txt
| |-- file1_2.txt
| |-- dir2
| |-- file2_1.txt
| |-- emptydir1
| |-- file0.txt
2) Uploads the same source directory again without -r,
checks nothing has changed. The directory should be just skipped.
3) Uploads the same source directory with --delete option and with -r.
Nothing should change.
4) Removes dir1 and dir2 locally.
Uploads the same source directory with --delete option and with -r.
dir1 and dir2 should be removed from the remote instance.
Args:
compressed (bool): Whether to append '--compress' or not.
"""
compressed_arg = '--compress' if compressed else None
local_root_path = self.local_base_dir + 'rootdir'
remote_root_path = self.remote_base_dir + 'rootdir/'
utils.create_test_file(local_root_path + '\\dir1\\file1_1.txt', 1024)
utils.create_test_file(local_root_path + '\\dir1\\file1_2.txt', 1024)
utils.create_test_directory(local_root_path + '\\dir1\\emptydir2\\')
utils.create_test_file(local_root_path + '\\dir2\\file2_1.txt', 1024)
utils.create_test_file(local_root_path + '\\file0.txt', 1024)
utils.create_test_directory(local_root_path + '\\emptydir1\\')
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
'-r')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=5))
self.assertTrue(
utils.sha1_matches(local_root_path + '\\dir1\\file1_1.txt',
remote_root_path + 'dir1/file1_1.txt'))
self.assertTrue(
utils.sha1_matches(local_root_path + '\\dir1\\file1_2.txt',
remote_root_path + 'dir1/file1_2.txt'))
self.assertTrue(
utils.sha1_matches(local_root_path + '\\dir2\\file2_1.txt',
remote_root_path + 'dir2/file2_1.txt'))
self.assertTrue(
utils.sha1_matches(local_root_path + '\\file0.txt',
remote_root_path + 'file0.txt'))
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, extraneous_dir=1))
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
'-r', '--delete')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=5))
utils.remove_test_directory(local_root_path + '\\dir1\\')
utils.remove_test_directory(local_root_path + '\\dir2\\')
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
'-r', '--delete')
self._assert_rsync_success(res)
self.assertTrue(
utils.files_count_is(
res, matching=1, extraneous=3, matching_dir=2, extraneous_dir=3))
self.assertIn('3/3 file(s) and 3/3 folder(s) deleted', res.stdout)
self.assertFalse(
utils.does_directory_exist_remotely(remote_root_path + 'dir1'))
self.assertFalse(
utils.does_directory_exist_remotely(remote_root_path + 'dir2'))
def test_upload_empty_file(self):
"""Uploads an empty file and checks sha1 signatures."""
empty_local_data_path = self.local_base_dir + 'emptyfile.dat'
empty_remote_data_path = self.remote_base_dir + 'emptyfile.dat'
utils.create_test_file(empty_local_data_path, 0)
res = utils.run_rsync(empty_local_data_path, self.remote_base_dir, None)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=1))
self.assertTrue(
utils.sha1_matches(empty_local_data_path, empty_remote_data_path))
def test_upload_empty_folder_with_backslash(self):
"""Uploads an empty folder with a trailing backslash."""
self._do_test_upload_empty_folder(with_backslash=True)
def test_upload_empty_folder_no_backslash(self):
"""Uploads an empty folder without a trailing backslash."""
self._do_test_upload_empty_folder(with_backslash=False)
def _do_test_upload_empty_folder(self, with_backslash=False):
"""Uploads an empty folder."""
local_data_dir = (
self.local_base_dir +
'empty_folder\\' if with_backslash else self.local_base_dir +
'empty_folder')
res = utils.run_rsync(local_data_dir, self.remote_base_dir, None)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=0))
def test_whole_file_uncompressed(self):
"""Uploads and syncs a file uncompressed with --whole-file."""
self._do_test_whole_file(compressed=False)
def test_whole_file_compressed(self):
"""Uploads and syncs a file compressed with --whole-file."""
self._do_test_whole_file(compressed=True)
def _do_test_whole_file(self, compressed):
"""Runs rsync 3 times with --whole-file -v options and validates results.
1) Uploads a file.
2) Modifies the file and uploads it with --whole-file and -v options.
Checks the output contains C100%, not D100%.
3) Modifies the file and uploads it with -W and -v options.
Checks the output contains C100%, not D100%.
Args:
compressed (bool): Whether to append '--compress' or not.
"""
compressed_arg = '--compress' if compressed else None
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
compressed_arg)
utils.create_test_file(self.local_data_path, 2534)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
compressed_arg, '--whole-file', '-v')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, changed=1))
self.assertIn('will be copied due to -W/--whole-file', str(res.stdout))
self.assertIn('C100%', str(res.stdout))
self.assertTrue(
utils.sha1_matches(self.local_data_path, self.remote_data_path))
utils.create_test_file(self.local_data_path, 3456)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
compressed_arg, '-W', '-v')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, changed=1))
self.assertIn('C100%', str(res.stdout))
self.assertTrue(
utils.sha1_matches(self.local_data_path, self.remote_data_path))
def test_keep_file_permissions(self):
"""Verifies that file permissions are kept for changed files."""
# Upload a file and check permissions.
utils.create_test_file(self.local_data_path, 1024)
utils.run_rsync(self.local_data_path, self.remote_base_dir)
ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
self.assertIn('-rw-r--r--', ls_res)
# Add executable bit.
utils.get_ssh_command_output('chmod a+x %s*' % self.remote_data_path)
ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
self.assertIn('-rwxr-xr-x', ls_res)
# Sync file again and verify permissions don't change.
utils.create_test_file(self.local_data_path, 1337)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, changed=1))
ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
self.assertIn('-rwxr-xr-x', ls_res)
def test_include_exclude(self):
"""Verifies the --include and --exclude options."""
files = [
'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
]
for file in files:
utils.create_test_file(self.local_base_dir + file, 987)
# Upload file2.txt and file3.dat.
res = utils.run_rsync(self.local_base_dir + '*', self.remote_base_dir, '-r',
'--include=*\\file2.txt', '--exclude=*.txt')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=3))
self._assert_remote_dir_contains(['folder1/file2.txt', 'folder1/file3.dat'])
# Upload all except *.dat with --delete, make sure file3.dat is kept.
utils.remove_test_file(self.local_base_dir + 'folder1\\file3.dat')
res = utils.run_rsync(self.local_base_dir + '*', self.remote_base_dir, '-r',
'--delete', '--exclude=*.dat')
self._assert_rsync_success(res)
self.assertTrue(
utils.files_count_is(res, missing=3, matching=1, matching_dir=3))
self._assert_remote_dir_contains([
'file1.txt', 'folder1/file2.txt', 'folder1/file3.dat',
'folder1/folder2/file4.txt', 'folder3/file5.txt'
])
def test_exclude_include_from(self):
"""Verifies the --include-from and --exclude-from options."""
files = [
'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
]
for file in files:
utils.create_test_file(self.local_base_dir + file, 987)
include_file = self.local_base_dir + 'include.txt'
with open(include_file, 'wt') as f:
f.writelines(['file1.txt\n', 'folder3\\file5.txt'])
exclude_file = self.local_base_dir + 'exclude.txt'
with open(exclude_file, 'wt') as f:
f.writelines(['*.txt'])
res = utils.run_rsync('-r', '--include-from', include_file,
'--exclude-from', exclude_file,
self.local_base_dir + '*', self.remote_base_dir)
self.assertTrue(utils.files_count_is(res, missing=3, missing_dir=3))
self._assert_remote_dir_contains(
['file1.txt', 'folder1/file3.dat', 'folder3/file5.txt'])
def test_files_from(self):
"""Verifies the --files-from option."""
files = [
'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
]
for file in files:
utils.create_test_file(self.local_base_dir + file, 987)
sources_file = self.local_base_dir + 'sources.txt'
with open(sources_file, 'wt') as f:
f.writelines([
'file1.txt\n',
'\n',
' folder1\\file3.dat \n',
'folder1\\.\\folder2\\file4.txt\n', # .\\ = rel path marker
' folder3\\file5.txt\n',
'\n'
])
res = utils.run_rsync('--files-from', sources_file, self.local_base_dir,
self.remote_base_dir)
self.assertTrue(utils.files_count_is(res, missing=4))
self._assert_remote_dir_contains([
'file1.txt', 'folder1/file3.dat', 'folder2/file4.txt',
'folder3/file5.txt'
])
# Upload again to check that nothing changes.
res = utils.run_rsync('--files-from', sources_file, self.local_base_dir,
self.remote_base_dir)
self.assertTrue(utils.files_count_is(res, matching=4, extraneous_dir=3))
def test_checksum_file(self):
"""Uploads and syncs a file with --checksum.
1) Uploads a file.
2) Uploads a file with --checksum option. As the file was not changed, it
is recognized as matched. The output should contain D100%.
3) Uploads the same file with --whole-file --checksum -v.
Checks the output contains C100%, not D100%.
4) Modifies the file without changing its content. The file is
synchronized, the output should contain D100%.
"""
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
'--checksum', '-v')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, matching=1))
self.assertIn('D100%', str(res.stdout))
self.assertIn('will be synced due to -c/--checksum', str(res.stdout))
utils.create_test_file(self.local_data_path, 2534)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
'--checksum', '-v', '--whole-file')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, changed=1))
self.assertIn('C100%', str(res.stdout))
self.assertIn('will be copied due to -c/--checksum and -W/--whole-file',
str(res.stdout))
self.assertTrue(
utils.sha1_matches(self.local_data_path, self.remote_data_path))
utils.change_modified_time(self.local_data_path)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-c',
'-v')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, changed=1))
self.assertIn('D100%', str(res.stdout))
def test_sync_folder_when_remote_file_non_recursive(self):
"""Non-recursively uploads a folder while there is a remote file with the same name."""
local_folder = self.local_base_dir + 'foldertocopy\\'
utils.create_test_directory(local_folder)
utils.get_ssh_command_output(
'mkdir -p %s && touch %s' %
(self.remote_base_dir, self.remote_base_dir + 'foldertocopy'))
res = utils.run_rsync(self.local_base_dir + 'foldertocopy',
self.remote_base_dir)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, extraneous=1))
self.assertFalse(
utils.does_directory_exist_remotely(self.remote_base_dir +
'foldertocopy'))
self.assertTrue(
utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy'))
def test_sync_folder_when_remote_file_recursive_with_delete(self):
"""Recursively uploads a folder while removing a remote file with the same name with --delete."""
local_folder = self.local_base_dir + 'foldertocopy\\'
utils.create_test_directory(local_folder)
utils.get_ssh_command_output(
'mkdir -p %s && touch %s' %
(self.remote_base_dir, self.remote_base_dir + 'foldertocopy'))
res = utils.run_rsync(self.local_base_dir + 'foldertocopy',
self.remote_base_dir, '-r', '--delete')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, extraneous=1, missing_dir=1))
self.assertTrue(
utils.does_directory_exist_remotely(self.remote_base_dir +
'foldertocopy'))
self.assertFalse(
utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy'))
self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout))
def test_sync_file_when_remote_folder_recursive_with_delete(self):
"""Recursively uploads a file while removing a remote folder with the same name with --delete."""
utils.create_test_file(self.local_data_path, 1024)
utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
'--delete', '-r')
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
self.assertTrue(
utils.sha1_matches(self.local_data_path, self.remote_data_path))
self.assertFalse(utils.does_directory_exist_remotely(self.remote_data_path))
self.assertIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
def test_sync_file_when_remote_folder_empty_non_recursive(self):
"""Non-recursively uploads a file while there is an empty remote folder with the same name."""
self._do_test_sync_file_when_remote_folder_empty(recursive=False)
def test_sync_file_when_remote_folder_empty_recursive(self):
"""Recursively uploads a file while there is an empty remote folder with the same name."""
self._do_test_sync_file_when_remote_folder_empty(recursive=True)
def _do_test_sync_file_when_remote_folder_empty(self, recursive):
"""Uploads a file while there is an empty remote folder with the same name.
Args:
recursive (bool): Whether to append '-r' or not.
"""
flag = '-r' if recursive else None
utils.create_test_file(self.local_data_path, 1024)
utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
res = utils.run_rsync(self.local_data_path, self.remote_base_dir, flag)
self._assert_rsync_success(res)
self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
self.assertTrue(
utils.sha1_matches(self.local_data_path, self.remote_data_path))
self.assertFalse(utils.does_directory_exist_remotely(self.remote_data_path))
self.assertNotIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
def test_sync_file_when_remote_folder_non_empty_non_recursive(self):
"""Non-recursively uploads a file while there is a non-empty remote folder with the same name."""
self._do_test_sync_file_when_remote_folder_non_empty(recursive=False)
def test_sync_file_when_remote_folder_non_empty_recursive(self):
"""Recursively uploads a file while there is a non-empty remote folder with the same name."""
self._do_test_sync_file_when_remote_folder_non_empty(recursive=True)
def _do_test_sync_file_when_remote_folder_non_empty(self, recursive):
"""Uploads a file while there is a non-empty remote folder with the same name.
Args:
recursive (bool): Whether to append '-r' or not.
"""
flag = '-r' if recursive else None
utils.create_test_file(self.local_data_path, 1024)
utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
utils.get_ssh_command_output(
'mkdir -p %s && touch %s' %
(self.remote_base_dir, self.remote_data_path + '/file1.txt'))
res = utils.run_rsync(self.local_data_path, self.remote_base_dir, flag)
self.assertIn('remove() failed: Directory not empty.', str(res.stderr))
if recursive:
self.assertTrue(
utils.files_count_is(res, missing=1, extraneous=1, extraneous_dir=1))
else:
self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path))
self.assertTrue(
utils.does_file_exist_remotely(self.remote_data_path + '/file1.txt'))
self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path))
def test_upload_from_dot(self):
"""Uploads files from the current directory ('.')."""
utils.create_test_file(self.local_base_dir + 'file1.txt', 1024)
utils.create_test_file(self.local_base_dir + 'dir\\file2.txt', 1024)
prev_cwd = os.getcwd()
os.chdir(self.local_base_dir)
try:
# Uploading recursivly should pick up all files and dirs.
res = utils.run_rsync('.', self.remote_base_dir, '-r')
self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=1))
self._assert_remote_dir_contains(['file1.txt', 'dir/file2.txt'])
# Uploading again should not change anything.
res = utils.run_rsync('.', self.remote_base_dir, '-r')
self.assertTrue(utils.files_count_is(res, matching=2, matching_dir=1))
# Verify that non-recursive uploads do nothing.
res = utils.run_rsync('.', self.remote_base_dir)
self.assertTrue(utils.files_count_is(res, extraneous=1, extraneous_dir=1))
finally:
os.chdir(prev_cwd)
def test_upload_from_dotdot(self):
"""Uploads files from the parent directory ('..')."""
utils.create_test_file(self.local_base_dir + 'file1.txt', 1024)
utils.create_test_file(self.local_base_dir + 'dir\\file2.txt', 1024)
prev_cwd = os.getcwd()
os.chdir(self.local_base_dir + 'dir')
try:
# Uploading recursivly should pick up all files and dirs.
res = utils.run_rsync('..', self.remote_base_dir, '-r')
self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=1))
self._assert_remote_dir_contains(['file1.txt', 'dir/file2.txt'])
# Uploading again should not change anything.
res = utils.run_rsync('..', self.remote_base_dir, '-r')
self.assertTrue(utils.files_count_is(res, matching=2, matching_dir=1))
# Verify that non-recursive uploads do nothing.
res = utils.run_rsync('..', self.remote_base_dir)
self.assertTrue(utils.files_count_is(res, extraneous=1, extraneous_dir=1))
finally:
os.chdir(prev_cwd)
def test_existing(self):
"""Runs rsync with --existing for a non-trivial directory.
1) Uploads a source directory with -r.
|-- rootdir
| |-- dir1
| |-- emptydir2
| |-- file1_1.txt
| |-- file1_2.txt -> rename to file1_3.txt (step 2)
| |-- (step2) emptydir3
| |-- dir2
| |-- file2_1.txt
| |-- emptydir1 -> rename emptydir4 (step 2)
| |-- file0.txt -> change (step 2)
2) Add new files/folders, remove and change some files/folders.
3) Uploads the same source directory with --existing option and with -r.
Only files existing on the server are changed, nothing is removed.
4) Uploads the same source directory with --existing --delete -r.
Files non-existing on the server are deleted.
"""
local_root_path = self.local_base_dir + 'rootdir'
remote_root_path = self.remote_base_dir + 'rootdir/'
files = [
'\\dir1\\file1_1.txt', '\\dir1\\file1_2.txt', '\\dir2\\file2_1.txt',
'\\file0.txt'
]
for file in files:
utils.create_test_file(local_root_path + file, 1024)
dirs = ['\\dir1\\emptydir2\\', '\\emptydir1\\']
for directory in dirs:
utils.create_test_directory(local_root_path + directory)
res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r')
self._assert_rsync_success(res)
utils.remove_test_file(local_root_path + '\\dir1\\file1_2.txt')
utils.create_test_file(local_root_path + '\\dir1\\file1_3.txt', 1024)
utils.create_test_directory(local_root_path + '\\dir1\\emptydir3\\')
utils.remove_test_directory(local_root_path + '\\emptydir1\\')
utils.create_test_directory(local_root_path + '\\emptydir4\\')
utils.create_test_file(local_root_path + '\\file0.txt', 2034)
res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r',
'--existing')
self._assert_rsync_success(res)
self.assertTrue(
utils.files_count_is(
res,
missing=1,
missing_dir=2,
matching=2,
matching_dir=4,
changed=1,
extraneous=1,
extraneous_dir=1))
self.assertTrue(
utils.does_directory_exist_remotely(remote_root_path + 'emptydir1'))
self.assertFalse(
utils.does_directory_exist_remotely(remote_root_path + 'emptydir4'))
self.assertFalse(
utils.does_directory_exist_remotely(remote_root_path +
'dir1/emptydir3'))
self.assertTrue(
utils.does_file_exist_remotely(remote_root_path + 'dir1/file1_2.txt'))
self.assertFalse(
utils.does_file_exist_remotely(remote_root_path + 'dir1/file1_3.txt'))
res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r',
'--existing', '--delete')
self._assert_rsync_success(res)
self.assertTrue(
utils.files_count_is(
res,
missing=1,
missing_dir=2,
matching=3,
matching_dir=4,
extraneous=1,
extraneous_dir=1))
self.assertIn('1/1 file(s) and 1/1 folder(s) deleted', res.stdout)
self.assertFalse(
utils.does_directory_exist_remotely(remote_root_path + 'emptydir1'))
self.assertFalse(
utils.does_file_exist_remotely(remote_root_path + 'dir2/file1_2.txt'))
def test_copy_dest(self):
r"""Runs rsync with --copy-dest option.
Copies testdata.dat to
Copies the "cdc_rsync_e2e_test" package locally and syncs it with
--copy-dest. Verifies that the files are actually sync'ed (D), not
copied (C).
Raises:
Exception: On timeout waiting for mount to appear (after 20 seconds)
"""
copy_dest_dir = self.remote_base_dir + 'copy_dest_dir'
utils.create_test_file(self.local_data_path, 1024)
res = utils.run_rsync(self.local_data_path, copy_dest_dir)
self._assert_rsync_success(res)
# Upload package using --package.
res = utils.run_rsync('--copy-dest', copy_dest_dir, self.local_data_path,
self.remote_base_dir, '-v')
self._assert_rsync_success(res)
self.assertIn('D100%', res.stdout)
self.assertNotIn('C100%', res.stdout)
def test_upload_executables(self):
"""Uploads executable files and checks that they have the x bit set."""
# Use the cdc rsync binaries as test executables.
local_exe_path = utils.CDC_RSYNC_PATH
local_elf_path = os.path.join(
os.path.dirname(local_exe_path), 'cdc_rsync_server')
remote_exe_path = self.remote_base_dir + os.path.basename(local_exe_path)
remote_elf_path = self.remote_base_dir + os.path.basename(local_elf_path)
# Copy the files to the gamelet.
res = utils.run_rsync(local_exe_path, local_elf_path, self.remote_base_dir)
self._assert_rsync_success(res)
# Check that both files have the executable bit set.
stats = utils.get_ssh_command_output('stat -c "%%a" %s %s' %
(remote_exe_path, remote_elf_path))
self.assertEqual(stats.count('755'), 2, stats)
# Remove executable bits.
utils.get_ssh_command_output('chmod -x %s %s' %
(remote_exe_path, remote_elf_path))
# Sync again, using -c to force a sync.
res = utils.run_rsync('-c', local_exe_path, local_elf_path,
self.remote_base_dir)
self._assert_rsync_success(res)
# Validate that the executable bits were restored.
stats = utils.get_ssh_command_output('stat -c "%%a" %s %s' %
(remote_exe_path, remote_elf_path))
self.assertEqual(stats.count('755'), 2, stats)
def _run(self, args):
logging.debug('Running %s', ' '.join(args))
res = subprocess.run(args, capture_output=True)
self.assertEqual(res.returncode, 0, 'Command failed: ' + str(res))
res.stdout = res.stdout.decode('ascii')
logging.debug('\r\n%s', res.stdout)
return res
if __name__ == '__main__':
test_base.test_base.main()