mirror of
https://github.com/nestriness/cdc-file-transfer.git
synced 2026-01-30 14:35:37 +02:00
[cdc_rsync] Add integration tests (#42)
[cdc_rsync] Add integration tests This CL adds Python integration tests for cdc_rsync. To run the tests, you need to supply a Linux host and proper configuration for cdc_rsync to work: set CDC_SSH_COMMAND=C:\path\to\ssh.exe <args> set CDC_SCP_COMMAND=C:\path\to\scp.exe <args> C:\python38\python.exe -m integration_tests.cdc_rsync.all_tests --binary_path=C:\full\path\to\cdc_rsync.exe --user_host=user@host Ran the tests and made sure they worked.
This commit is contained in:
Lutz Justen
GitHub
894
integration_tests/cdc_rsync/upload_test.py
Normal file
894
integration_tests/cdc_rsync/upload_test.py
Normal file
@@ -0,0 +1,894 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Lint as: python3
|
||||
"""cdc_rsync upload test."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from integration_tests.framework import utils
|
||||
from integration_tests.cdc_rsync import test_base
|
||||
|
||||
|
||||
class UploadTest(test_base.CdcRsyncTest):
|
||||
"""cdc_rsync upload test class."""
|
||||
|
||||
def test_single_uncompressed(self):
|
||||
"""Uploads and syncs a file uncompressed."""
|
||||
|
||||
self._do_test_single(compressed=False)
|
||||
|
||||
def test_upload_compressed(self):
|
||||
"""Uploads and syncs a file compressed."""
|
||||
|
||||
self._do_test_single(compressed=True)
|
||||
|
||||
def _do_test_single(self, compressed):
|
||||
"""Runs rsync 3 times and validates results.
|
||||
|
||||
1) Uploads a file, checks sha1 hashes.
|
||||
2) Uploads the same file again, checks nothing changed.
|
||||
3) Modifies the file and uploads again. Checks sha1 hashes.
|
||||
|
||||
Args:
|
||||
compressed (bool): Whether to append '--compress' or not.
|
||||
"""
|
||||
compressed_arg = '--compress' if compressed else None
|
||||
|
||||
utils.create_test_file(self.local_data_path, 1024)
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
|
||||
compressed_arg)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, missing=1))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(self.local_data_path, self.remote_data_path))
|
||||
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
|
||||
compressed_arg)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, matching=1))
|
||||
|
||||
utils.create_test_file(self.local_data_path, 2534)
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
|
||||
compressed_arg)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, changed=1))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(self.local_data_path, self.remote_data_path))
|
||||
|
||||
def test_backslash_in_dest_folder(self):
|
||||
r"""Verifies uploading to \mnt\developer."""
|
||||
|
||||
filepath = os.path.join(self.local_base_dir, 'file1.txt')
|
||||
utils.create_test_file(filepath, 1)
|
||||
res = utils.run_rsync(filepath, self.remote_base_dir.replace('/', '\\'))
|
||||
self.assertTrue(utils.files_count_is(res, missing=1))
|
||||
self._assert_remote_dir_contains(['file1.txt'])
|
||||
|
||||
def test_backslash_in_source_folder(self):
|
||||
r"""Verifies uploading from /source/folder."""
|
||||
|
||||
filepath = os.path.join(self.local_base_dir, 'file1.txt')
|
||||
utils.create_test_file(filepath, 1)
|
||||
filepath = filepath.replace('\\', '/')
|
||||
res = utils.run_rsync(filepath, self.remote_base_dir)
|
||||
self.assertTrue(utils.files_count_is(res, missing=1))
|
||||
self._assert_remote_dir_contains(['file1.txt'])
|
||||
|
||||
def test_single_unicode(self):
|
||||
"""Uploads a file with a non-ascii unicode path and checks sha1 signatures."""
|
||||
|
||||
nonascii_local_data_path = self.local_base_dir + '⛽⛽⛽⛽⛽⛽⛽⛽.dat'
|
||||
nonascii_remote_data_path = self.remote_base_dir + '⛽⛽⛽⛽⛽⛽⛽⛽.dat'
|
||||
utils.create_test_file(nonascii_local_data_path, 1024)
|
||||
# In order to check that non-ascii characters are not considered as
|
||||
# wildcard
|
||||
# ? characters, create a second file. Only 1 file should be uploaded.
|
||||
utils.create_test_file(self.local_data_path, 1024)
|
||||
res = utils.run_rsync(nonascii_local_data_path, self.remote_base_dir, None)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, missing=1))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(nonascii_local_data_path, nonascii_remote_data_path))
|
||||
|
||||
def test_uncompressed_no_empty_folders(self):
|
||||
"""Uploads and syncs multiple files uncompressed in different folders."""
|
||||
|
||||
self._do_test_no_empty_folders(compressed=False)
|
||||
|
||||
def test_compressed_no_empty_folders(self):
|
||||
"""Uploads and syncs multiple files compressed in different folders."""
|
||||
|
||||
self._do_test_no_empty_folders(compressed=True)
|
||||
|
||||
def _do_test_no_empty_folders(self, compressed):
|
||||
"""Runs rsync with(out) -r for a non-trivial directory and validates results.
|
||||
|
||||
1) Uploads a source directory with -r, checks sha1 hashes.
|
||||
|-- rootdir
|
||||
| |-- dir1
|
||||
| |-- file1_1.txt
|
||||
| |-- file1_2.txt
|
||||
| |-- dir2
|
||||
| |-- file2_1.txt
|
||||
| |-- file0.txt
|
||||
2) Uploads the same source directory again without -r,
|
||||
checks nothing has changed. The directory should be just skipped.
|
||||
3) Uploads the same source directory with --delete option and with -r.
|
||||
Nothing should change.
|
||||
4) Removes dir1 and dir2 locally.
|
||||
Uploads the same source directory with --delete option and with -r.
|
||||
dir1 and dir2 should be removed from the remote instance.
|
||||
|
||||
Args:
|
||||
compressed (bool): Whether to append '--compress' or not.
|
||||
"""
|
||||
compressed_arg = '--compress' if compressed else None
|
||||
local_root_path = self.local_base_dir + 'rootdir'
|
||||
remote_root_path = self.remote_base_dir + 'rootdir/'
|
||||
utils.create_test_file(local_root_path + '\\dir1\\file1_1.txt', 1024)
|
||||
utils.create_test_file(local_root_path + '\\dir1\\file1_2.txt', 1024)
|
||||
utils.create_test_file(local_root_path + '\\dir2\\file2_1.txt', 1024)
|
||||
utils.create_test_file(local_root_path + '\\file0.txt', 1024)
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
|
||||
'-r')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=3))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + '\\dir1\\file1_1.txt',
|
||||
remote_root_path + 'dir1/file1_1.txt'))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + '\\dir1\\file1_2.txt',
|
||||
remote_root_path + 'dir1/file1_2.txt'))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + '\\dir2\\file2_1.txt',
|
||||
remote_root_path + 'dir2/file2_1.txt'))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + '\\file0.txt',
|
||||
remote_root_path + 'file0.txt'))
|
||||
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, extraneous_dir=1))
|
||||
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
|
||||
'-r', '--delete')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=3))
|
||||
|
||||
utils.remove_test_directory(local_root_path + '\\dir1\\')
|
||||
utils.remove_test_directory(local_root_path + '\\dir2\\')
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
|
||||
'-r', '--delete')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(
|
||||
utils.files_count_is(
|
||||
res, matching=1, extraneous=3, matching_dir=1, extraneous_dir=2))
|
||||
self.assertFalse(
|
||||
utils.does_directory_exist_remotely(remote_root_path + 'dir1'))
|
||||
self.assertFalse(
|
||||
utils.does_directory_exist_remotely(remote_root_path + 'dir2'))
|
||||
|
||||
def _do_test_no_empty_folders_with_backslash(self, compressed):
|
||||
"""Runs rsync with(out) -r for a non-trivial directory with a trailing backslash.
|
||||
|
||||
1) Uploads a source directory with -r, checks sha1 hashes.
|
||||
Everything from rootdir should be copied except rootdir itself.
|
||||
|-- rootdir
|
||||
| |-- dir1
|
||||
| |-- file1_1.txt
|
||||
| |-- file1_2.txt
|
||||
| |-- dir2
|
||||
| |-- file2_1.txt
|
||||
| |-- file0.txt
|
||||
2) Uploads the same source directory again without -r,
|
||||
checks nothing has changed. The directory should be just skipped.
|
||||
3) Uploads the same source directory with --delete option and with -r.
|
||||
Nothing should change.
|
||||
4) Removes dir1 and dir2 locally.
|
||||
Uploads the same source directory with --delete option and with -r.
|
||||
dir1 and dir2 should be removed from the remote instance.
|
||||
|
||||
Args:
|
||||
compressed (bool): Whether to append '--compress' or not.
|
||||
"""
|
||||
compressed_arg = '--compress' if compressed else None
|
||||
local_root_path = self.local_base_dir + 'rootdir\\'
|
||||
utils.create_test_file(local_root_path + 'dir1\\file1_1.txt', 1024)
|
||||
utils.create_test_file(local_root_path + 'dir1\\file1_2.txt', 1024)
|
||||
utils.create_test_file(local_root_path + 'dir2\\file2_1.txt', 1024)
|
||||
utils.create_test_file(local_root_path + 'file0.txt', 1024)
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
|
||||
'-r')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=2))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + 'dir1\\file1_1.txt',
|
||||
self.remote_base_dir + 'dir1/file1_1.txt'))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + 'dir1\\file1_2.txt',
|
||||
self.remote_base_dir + 'dir1/file1_2.txt'))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + 'dir2\\file2_1.txt',
|
||||
self.remote_base_dir + 'dir2/file2_1.txt'))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + 'file0.txt',
|
||||
self.remote_base_dir + 'file0.txt'))
|
||||
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(
|
||||
res, extraneous=1, extraneous_dir=2)) # file0.txt, dir1, dir2
|
||||
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
|
||||
'-r', '--delete')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=2))
|
||||
|
||||
utils.remove_test_directory(local_root_path + '\\dir1\\')
|
||||
utils.remove_test_directory(local_root_path + '\\dir2\\')
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
|
||||
'-r', '--delete')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(
|
||||
utils.files_count_is(res, matching=1, extraneous=3, extraneous_dir=2))
|
||||
self.assertFalse(
|
||||
utils.does_directory_exist_remotely(self.remote_base_dir + 'dir1'))
|
||||
self.assertFalse(
|
||||
utils.does_directory_exist_remotely(self.remote_base_dir + 'dir2'))
|
||||
|
||||
def test_uncompressed_no_empty_folders_with_backslash(self):
|
||||
"""Uploads multiple files uncompressed from a folder with a trailing backslash."""
|
||||
|
||||
self._do_test_no_empty_folders_with_backslash(compressed=False)
|
||||
|
||||
def test_compressed_no_empty_folders_with_backslash(self):
|
||||
"""Uploads multiple files compressed from a folder with a trailing backslash."""
|
||||
|
||||
self._do_test_no_empty_folders_with_backslash(compressed=True)
|
||||
|
||||
def test_uncompressed_with_empty_folders(self):
|
||||
"""Uploads and syncs multiple files uncompressed and empty folders."""
|
||||
|
||||
self._do_test_with_empty_folders(compressed=False)
|
||||
|
||||
def test_compressed_with_empty_folders(self):
|
||||
"""Uploads and syncs multiple files compress and empty folders."""
|
||||
|
||||
self._do_test_with_empty_folders(compressed=True)
|
||||
|
||||
def _do_test_with_empty_folders(self, compressed):
|
||||
"""Runs rsync with(out) -r for a non-trivial directory with empty folders.
|
||||
|
||||
1) Uploads a source directory with -r, checks sha1 hashes.
|
||||
|-- rootdir
|
||||
| |-- dir1
|
||||
| |-- emptydir2
|
||||
| |-- file1_1.txt
|
||||
| |-- file1_2.txt
|
||||
| |-- dir2
|
||||
| |-- file2_1.txt
|
||||
| |-- emptydir1
|
||||
| |-- file0.txt
|
||||
2) Uploads the same source directory again without -r,
|
||||
checks nothing has changed. The directory should be just skipped.
|
||||
3) Uploads the same source directory with --delete option and with -r.
|
||||
Nothing should change.
|
||||
4) Removes dir1 and dir2 locally.
|
||||
Uploads the same source directory with --delete option and with -r.
|
||||
dir1 and dir2 should be removed from the remote instance.
|
||||
|
||||
Args:
|
||||
compressed (bool): Whether to append '--compress' or not.
|
||||
"""
|
||||
compressed_arg = '--compress' if compressed else None
|
||||
local_root_path = self.local_base_dir + 'rootdir'
|
||||
remote_root_path = self.remote_base_dir + 'rootdir/'
|
||||
utils.create_test_file(local_root_path + '\\dir1\\file1_1.txt', 1024)
|
||||
utils.create_test_file(local_root_path + '\\dir1\\file1_2.txt', 1024)
|
||||
utils.create_test_directory(local_root_path + '\\dir1\\emptydir2\\')
|
||||
utils.create_test_file(local_root_path + '\\dir2\\file2_1.txt', 1024)
|
||||
utils.create_test_file(local_root_path + '\\file0.txt', 1024)
|
||||
utils.create_test_directory(local_root_path + '\\emptydir1\\')
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
|
||||
'-r')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=5))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + '\\dir1\\file1_1.txt',
|
||||
remote_root_path + 'dir1/file1_1.txt'))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + '\\dir1\\file1_2.txt',
|
||||
remote_root_path + 'dir1/file1_2.txt'))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + '\\dir2\\file2_1.txt',
|
||||
remote_root_path + 'dir2/file2_1.txt'))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(local_root_path + '\\file0.txt',
|
||||
remote_root_path + 'file0.txt'))
|
||||
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, extraneous_dir=1))
|
||||
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
|
||||
'-r', '--delete')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=5))
|
||||
|
||||
utils.remove_test_directory(local_root_path + '\\dir1\\')
|
||||
utils.remove_test_directory(local_root_path + '\\dir2\\')
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
|
||||
'-r', '--delete')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(
|
||||
utils.files_count_is(
|
||||
res, matching=1, extraneous=3, matching_dir=2, extraneous_dir=3))
|
||||
self.assertIn('3/3 file(s) and 3/3 folder(s) deleted', res.stdout)
|
||||
self.assertFalse(
|
||||
utils.does_directory_exist_remotely(remote_root_path + 'dir1'))
|
||||
self.assertFalse(
|
||||
utils.does_directory_exist_remotely(remote_root_path + 'dir2'))
|
||||
|
||||
def test_upload_empty_file(self):
|
||||
"""Uploads an empty file and checks sha1 signatures."""
|
||||
|
||||
empty_local_data_path = self.local_base_dir + 'emptyfile.dat'
|
||||
empty_remote_data_path = self.remote_base_dir + 'emptyfile.dat'
|
||||
utils.create_test_file(empty_local_data_path, 0)
|
||||
res = utils.run_rsync(empty_local_data_path, self.remote_base_dir, None)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, missing=1))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(empty_local_data_path, empty_remote_data_path))
|
||||
|
||||
def test_upload_empty_folder_with_backslash(self):
|
||||
"""Uploads an empty folder with a trailing backslash."""
|
||||
|
||||
self._do_test_upload_empty_folder(with_backslash=True)
|
||||
|
||||
def test_upload_empty_folder_no_backslash(self):
|
||||
"""Uploads an empty folder without a trailing backslash."""
|
||||
|
||||
self._do_test_upload_empty_folder(with_backslash=False)
|
||||
|
||||
def _do_test_upload_empty_folder(self, with_backslash=False):
|
||||
"""Uploads an empty folder."""
|
||||
|
||||
local_data_dir = (
|
||||
self.local_base_dir +
|
||||
'empty_folder\\' if with_backslash else self.local_base_dir +
|
||||
'empty_folder')
|
||||
res = utils.run_rsync(local_data_dir, self.remote_base_dir, None)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, missing=0))
|
||||
|
||||
def test_whole_file_uncompressed(self):
|
||||
"""Uploads and syncs a file uncompressed with --whole-file."""
|
||||
|
||||
self._do_test_whole_file(compressed=False)
|
||||
|
||||
def test_whole_file_compressed(self):
|
||||
"""Uploads and syncs a file compressed with --whole-file."""
|
||||
|
||||
self._do_test_whole_file(compressed=True)
|
||||
|
||||
def _do_test_whole_file(self, compressed):
|
||||
"""Runs rsync 3 times with --whole-file -v options and validates results.
|
||||
|
||||
1) Uploads a file.
|
||||
2) Modifies the file and uploads it with --whole-file and -v options.
|
||||
Checks the output contains C100%, not D100%.
|
||||
3) Modifies the file and uploads it with -W and -v options.
|
||||
Checks the output contains C100%, not D100%.
|
||||
|
||||
Args:
|
||||
compressed (bool): Whether to append '--compress' or not.
|
||||
"""
|
||||
compressed_arg = '--compress' if compressed else None
|
||||
|
||||
utils.create_test_file(self.local_data_path, 1024)
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
|
||||
compressed_arg)
|
||||
|
||||
utils.create_test_file(self.local_data_path, 2534)
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
|
||||
compressed_arg, '--whole-file', '-v')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, changed=1))
|
||||
self.assertIn('will be copied due to -W/--whole-file', str(res.stdout))
|
||||
self.assertIn('C100%', str(res.stdout))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(self.local_data_path, self.remote_data_path))
|
||||
|
||||
utils.create_test_file(self.local_data_path, 3456)
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
|
||||
compressed_arg, '-W', '-v')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, changed=1))
|
||||
self.assertIn('C100%', str(res.stdout))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(self.local_data_path, self.remote_data_path))
|
||||
|
||||
def test_keep_file_permissions(self):
|
||||
"""Verifies that file permissions are kept for changed files."""
|
||||
|
||||
# Upload a file and check permissions.
|
||||
utils.create_test_file(self.local_data_path, 1024)
|
||||
utils.run_rsync(self.local_data_path, self.remote_base_dir)
|
||||
ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
|
||||
self.assertIn('-rw-r--r--', ls_res)
|
||||
|
||||
# Add executable bit.
|
||||
utils.get_ssh_command_output('chmod a+x %s*' % self.remote_data_path)
|
||||
ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
|
||||
self.assertIn('-rwxr-xr-x', ls_res)
|
||||
|
||||
# Sync file again and verify permissions don't change.
|
||||
utils.create_test_file(self.local_data_path, 1337)
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, changed=1))
|
||||
ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
|
||||
self.assertIn('-rwxr-xr-x', ls_res)
|
||||
|
||||
def test_include_exclude(self):
|
||||
"""Verifies the --include and --exclude options."""
|
||||
|
||||
files = [
|
||||
'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
|
||||
'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
|
||||
]
|
||||
|
||||
for file in files:
|
||||
utils.create_test_file(self.local_base_dir + file, 987)
|
||||
|
||||
# Upload file2.txt and file3.dat.
|
||||
res = utils.run_rsync(self.local_base_dir + '*', self.remote_base_dir, '-r',
|
||||
'--include=*\\file2.txt', '--exclude=*.txt')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=3))
|
||||
self._assert_remote_dir_contains(['folder1/file2.txt', 'folder1/file3.dat'])
|
||||
|
||||
# Upload all except *.dat with --delete, make sure file3.dat is kept.
|
||||
utils.remove_test_file(self.local_base_dir + 'folder1\\file3.dat')
|
||||
res = utils.run_rsync(self.local_base_dir + '*', self.remote_base_dir, '-r',
|
||||
'--delete', '--exclude=*.dat')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(
|
||||
utils.files_count_is(res, missing=3, matching=1, matching_dir=3))
|
||||
self._assert_remote_dir_contains([
|
||||
'file1.txt', 'folder1/file2.txt', 'folder1/file3.dat',
|
||||
'folder1/folder2/file4.txt', 'folder3/file5.txt'
|
||||
])
|
||||
|
||||
def test_exclude_include_from(self):
|
||||
"""Verifies the --include-from and --exclude-from options."""
|
||||
|
||||
files = [
|
||||
'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
|
||||
'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
|
||||
]
|
||||
|
||||
for file in files:
|
||||
utils.create_test_file(self.local_base_dir + file, 987)
|
||||
|
||||
include_file = self.local_base_dir + 'include.txt'
|
||||
with open(include_file, 'wt') as f:
|
||||
f.writelines(['file1.txt\n', 'folder3\\file5.txt'])
|
||||
|
||||
exclude_file = self.local_base_dir + 'exclude.txt'
|
||||
with open(exclude_file, 'wt') as f:
|
||||
f.writelines(['*.txt'])
|
||||
|
||||
res = utils.run_rsync('-r', '--include-from', include_file,
|
||||
'--exclude-from', exclude_file,
|
||||
self.local_base_dir + '*', self.remote_base_dir)
|
||||
self.assertTrue(utils.files_count_is(res, missing=3, missing_dir=3))
|
||||
self._assert_remote_dir_contains(
|
||||
['file1.txt', 'folder1/file3.dat', 'folder3/file5.txt'])
|
||||
|
||||
def test_files_from(self):
|
||||
"""Verifies the --files-from option."""
|
||||
|
||||
files = [
|
||||
'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
|
||||
'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
|
||||
]
|
||||
|
||||
for file in files:
|
||||
utils.create_test_file(self.local_base_dir + file, 987)
|
||||
|
||||
sources_file = self.local_base_dir + 'sources.txt'
|
||||
with open(sources_file, 'wt') as f:
|
||||
f.writelines([
|
||||
'file1.txt\n',
|
||||
'\n',
|
||||
' folder1\\file3.dat \n',
|
||||
'folder1\\.\\folder2\\file4.txt\n', # .\\ = rel path marker
|
||||
' folder3\\file5.txt\n',
|
||||
'\n'
|
||||
])
|
||||
|
||||
res = utils.run_rsync('--files-from', sources_file, self.local_base_dir,
|
||||
self.remote_base_dir)
|
||||
self.assertTrue(utils.files_count_is(res, missing=4))
|
||||
self._assert_remote_dir_contains([
|
||||
'file1.txt', 'folder1/file3.dat', 'folder2/file4.txt',
|
||||
'folder3/file5.txt'
|
||||
])
|
||||
|
||||
# Upload again to check that nothing changes.
|
||||
res = utils.run_rsync('--files-from', sources_file, self.local_base_dir,
|
||||
self.remote_base_dir)
|
||||
self.assertTrue(utils.files_count_is(res, matching=4, extraneous_dir=3))
|
||||
|
||||
def test_checksum_file(self):
|
||||
"""Uploads and syncs a file with --checksum.
|
||||
|
||||
1) Uploads a file.
|
||||
2) Uploads a file with --checksum option. As the file was not changed, it
|
||||
is recognized as matched. The output should contain D100%.
|
||||
3) Uploads the same file with --whole-file --checksum -v.
|
||||
Checks the output contains C100%, not D100%.
|
||||
4) Modifies the file without changing its content. The file is
|
||||
synchronized, the output should contain D100%.
|
||||
|
||||
"""
|
||||
|
||||
utils.create_test_file(self.local_data_path, 1024)
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
|
||||
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
|
||||
'--checksum', '-v')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, matching=1))
|
||||
self.assertIn('D100%', str(res.stdout))
|
||||
self.assertIn('will be synced due to -c/--checksum', str(res.stdout))
|
||||
|
||||
utils.create_test_file(self.local_data_path, 2534)
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
|
||||
'--checksum', '-v', '--whole-file')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, changed=1))
|
||||
self.assertIn('C100%', str(res.stdout))
|
||||
self.assertIn('will be copied due to -c/--checksum and -W/--whole-file',
|
||||
str(res.stdout))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(self.local_data_path, self.remote_data_path))
|
||||
|
||||
utils.change_modified_time(self.local_data_path)
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-c',
|
||||
'-v')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, changed=1))
|
||||
self.assertIn('D100%', str(res.stdout))
|
||||
|
||||
def test_sync_folder_when_remote_file_non_recursive(self):
|
||||
"""Non-recursively uploads a folder while there is a remote file with the same name."""
|
||||
|
||||
local_folder = self.local_base_dir + 'foldertocopy\\'
|
||||
utils.create_test_directory(local_folder)
|
||||
utils.get_ssh_command_output(
|
||||
'mkdir -p %s && touch %s' %
|
||||
(self.remote_base_dir, self.remote_base_dir + 'foldertocopy'))
|
||||
|
||||
res = utils.run_rsync(self.local_base_dir + 'foldertocopy',
|
||||
self.remote_base_dir)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, extraneous=1))
|
||||
self.assertFalse(
|
||||
utils.does_directory_exist_remotely(self.remote_base_dir +
|
||||
'foldertocopy'))
|
||||
self.assertTrue(
|
||||
utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy'))
|
||||
|
||||
def test_sync_folder_when_remote_file_recursive_with_delete(self):
|
||||
"""Recursively uploads a folder while removing a remote file with the same name with --delete."""
|
||||
|
||||
local_folder = self.local_base_dir + 'foldertocopy\\'
|
||||
utils.create_test_directory(local_folder)
|
||||
utils.get_ssh_command_output(
|
||||
'mkdir -p %s && touch %s' %
|
||||
(self.remote_base_dir, self.remote_base_dir + 'foldertocopy'))
|
||||
|
||||
res = utils.run_rsync(self.local_base_dir + 'foldertocopy',
|
||||
self.remote_base_dir, '-r', '--delete')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, extraneous=1, missing_dir=1))
|
||||
self.assertTrue(
|
||||
utils.does_directory_exist_remotely(self.remote_base_dir +
|
||||
'foldertocopy'))
|
||||
self.assertFalse(
|
||||
utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy'))
|
||||
self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout))
|
||||
|
||||
def test_sync_file_when_remote_folder_recursive_with_delete(self):
|
||||
"""Recursively uploads a file while removing a remote folder with the same name with --delete."""
|
||||
utils.create_test_file(self.local_data_path, 1024)
|
||||
utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
|
||||
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
|
||||
'--delete', '-r')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(self.local_data_path, self.remote_data_path))
|
||||
self.assertFalse(utils.does_directory_exist_remotely(self.remote_data_path))
|
||||
self.assertIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
|
||||
|
||||
def test_sync_file_when_remote_folder_empty_non_recursive(self):
|
||||
"""Non-recursively uploads a file while there is an empty remote folder with the same name."""
|
||||
self._do_test_sync_file_when_remote_folder_empty(recursive=False)
|
||||
|
||||
def test_sync_file_when_remote_folder_empty_recursive(self):
|
||||
"""Recursively uploads a file while there is an empty remote folder with the same name."""
|
||||
self._do_test_sync_file_when_remote_folder_empty(recursive=True)
|
||||
|
||||
def _do_test_sync_file_when_remote_folder_empty(self, recursive):
|
||||
"""Uploads a file while there is an empty remote folder with the same name.
|
||||
|
||||
Args:
|
||||
recursive (bool): Whether to append '-r' or not.
|
||||
"""
|
||||
flag = '-r' if recursive else None
|
||||
utils.create_test_file(self.local_data_path, 1024)
|
||||
utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
|
||||
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir, flag)
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
|
||||
self.assertTrue(
|
||||
utils.sha1_matches(self.local_data_path, self.remote_data_path))
|
||||
self.assertFalse(utils.does_directory_exist_remotely(self.remote_data_path))
|
||||
self.assertNotIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
|
||||
|
||||
def test_sync_file_when_remote_folder_non_empty_non_recursive(self):
|
||||
"""Non-recursively uploads a file while there is a non-empty remote folder with the same name."""
|
||||
self._do_test_sync_file_when_remote_folder_non_empty(recursive=False)
|
||||
|
||||
def test_sync_file_when_remote_folder_non_empty_recursive(self):
|
||||
"""Recursively uploads a file while there is a non-empty remote folder with the same name."""
|
||||
self._do_test_sync_file_when_remote_folder_non_empty(recursive=True)
|
||||
|
||||
def _do_test_sync_file_when_remote_folder_non_empty(self, recursive):
|
||||
"""Uploads a file while there is a non-empty remote folder with the same name.
|
||||
|
||||
Args:
|
||||
recursive (bool): Whether to append '-r' or not.
|
||||
"""
|
||||
flag = '-r' if recursive else None
|
||||
utils.create_test_file(self.local_data_path, 1024)
|
||||
utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
|
||||
utils.get_ssh_command_output(
|
||||
'mkdir -p %s && touch %s' %
|
||||
(self.remote_base_dir, self.remote_data_path + '/file1.txt'))
|
||||
|
||||
res = utils.run_rsync(self.local_data_path, self.remote_base_dir, flag)
|
||||
self.assertIn('remove() failed: Directory not empty.', str(res.stderr))
|
||||
if recursive:
|
||||
self.assertTrue(
|
||||
utils.files_count_is(res, missing=1, extraneous=1, extraneous_dir=1))
|
||||
else:
|
||||
self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
|
||||
self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path))
|
||||
self.assertTrue(
|
||||
utils.does_file_exist_remotely(self.remote_data_path + '/file1.txt'))
|
||||
self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path))
|
||||
|
||||
def test_upload_from_dot(self):
|
||||
"""Uploads files from the current directory ('.')."""
|
||||
utils.create_test_file(self.local_base_dir + 'file1.txt', 1024)
|
||||
utils.create_test_file(self.local_base_dir + 'dir\\file2.txt', 1024)
|
||||
|
||||
prev_cwd = os.getcwd()
|
||||
os.chdir(self.local_base_dir)
|
||||
try:
|
||||
# Uploading recursivly should pick up all files and dirs.
|
||||
res = utils.run_rsync('.', self.remote_base_dir, '-r')
|
||||
self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=1))
|
||||
self._assert_remote_dir_contains(['file1.txt', 'dir/file2.txt'])
|
||||
|
||||
# Uploading again should not change anything.
|
||||
res = utils.run_rsync('.', self.remote_base_dir, '-r')
|
||||
self.assertTrue(utils.files_count_is(res, matching=2, matching_dir=1))
|
||||
|
||||
# Verify that non-recursive uploads do nothing.
|
||||
res = utils.run_rsync('.', self.remote_base_dir)
|
||||
self.assertTrue(utils.files_count_is(res, extraneous=1, extraneous_dir=1))
|
||||
finally:
|
||||
os.chdir(prev_cwd)
|
||||
|
||||
def test_upload_from_dotdot(self):
|
||||
"""Uploads files from the parent directory ('..')."""
|
||||
utils.create_test_file(self.local_base_dir + 'file1.txt', 1024)
|
||||
utils.create_test_file(self.local_base_dir + 'dir\\file2.txt', 1024)
|
||||
|
||||
prev_cwd = os.getcwd()
|
||||
os.chdir(self.local_base_dir + 'dir')
|
||||
try:
|
||||
# Uploading recursivly should pick up all files and dirs.
|
||||
res = utils.run_rsync('..', self.remote_base_dir, '-r')
|
||||
self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=1))
|
||||
self._assert_remote_dir_contains(['file1.txt', 'dir/file2.txt'])
|
||||
|
||||
# Uploading again should not change anything.
|
||||
res = utils.run_rsync('..', self.remote_base_dir, '-r')
|
||||
self.assertTrue(utils.files_count_is(res, matching=2, matching_dir=1))
|
||||
|
||||
# Verify that non-recursive uploads do nothing.
|
||||
res = utils.run_rsync('..', self.remote_base_dir)
|
||||
self.assertTrue(utils.files_count_is(res, extraneous=1, extraneous_dir=1))
|
||||
finally:
|
||||
os.chdir(prev_cwd)
|
||||
|
||||
def test_existing(self):
|
||||
"""Runs rsync with --existing for a non-trivial directory.
|
||||
|
||||
1) Uploads a source directory with -r.
|
||||
|-- rootdir
|
||||
| |-- dir1
|
||||
| |-- emptydir2
|
||||
| |-- file1_1.txt
|
||||
| |-- file1_2.txt -> rename to file1_3.txt (step 2)
|
||||
| |-- (step2) emptydir3
|
||||
| |-- dir2
|
||||
| |-- file2_1.txt
|
||||
| |-- emptydir1 -> rename emptydir4 (step 2)
|
||||
| |-- file0.txt -> change (step 2)
|
||||
2) Add new files/folders, remove and change some files/folders.
|
||||
3) Uploads the same source directory with --existing option and with -r.
|
||||
Only files existing on the server are changed, nothing is removed.
|
||||
4) Uploads the same source directory with --existing --delete -r.
|
||||
Files non-existing on the server are deleted.
|
||||
"""
|
||||
local_root_path = self.local_base_dir + 'rootdir'
|
||||
remote_root_path = self.remote_base_dir + 'rootdir/'
|
||||
|
||||
files = [
|
||||
'\\dir1\\file1_1.txt', '\\dir1\\file1_2.txt', '\\dir2\\file2_1.txt',
|
||||
'\\file0.txt'
|
||||
]
|
||||
for file in files:
|
||||
utils.create_test_file(local_root_path + file, 1024)
|
||||
dirs = ['\\dir1\\emptydir2\\', '\\emptydir1\\']
|
||||
for directory in dirs:
|
||||
utils.create_test_directory(local_root_path + directory)
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r')
|
||||
self._assert_rsync_success(res)
|
||||
|
||||
utils.remove_test_file(local_root_path + '\\dir1\\file1_2.txt')
|
||||
utils.create_test_file(local_root_path + '\\dir1\\file1_3.txt', 1024)
|
||||
utils.create_test_directory(local_root_path + '\\dir1\\emptydir3\\')
|
||||
utils.remove_test_directory(local_root_path + '\\emptydir1\\')
|
||||
utils.create_test_directory(local_root_path + '\\emptydir4\\')
|
||||
utils.create_test_file(local_root_path + '\\file0.txt', 2034)
|
||||
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r',
|
||||
'--existing')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(
|
||||
utils.files_count_is(
|
||||
res,
|
||||
missing=1,
|
||||
missing_dir=2,
|
||||
matching=2,
|
||||
matching_dir=4,
|
||||
changed=1,
|
||||
extraneous=1,
|
||||
extraneous_dir=1))
|
||||
self.assertTrue(
|
||||
utils.does_directory_exist_remotely(remote_root_path + 'emptydir1'))
|
||||
self.assertFalse(
|
||||
utils.does_directory_exist_remotely(remote_root_path + 'emptydir4'))
|
||||
self.assertFalse(
|
||||
utils.does_directory_exist_remotely(remote_root_path +
|
||||
'dir1/emptydir3'))
|
||||
self.assertTrue(
|
||||
utils.does_file_exist_remotely(remote_root_path + 'dir1/file1_2.txt'))
|
||||
self.assertFalse(
|
||||
utils.does_file_exist_remotely(remote_root_path + 'dir1/file1_3.txt'))
|
||||
|
||||
res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r',
|
||||
'--existing', '--delete')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertTrue(
|
||||
utils.files_count_is(
|
||||
res,
|
||||
missing=1,
|
||||
missing_dir=2,
|
||||
matching=3,
|
||||
matching_dir=4,
|
||||
extraneous=1,
|
||||
extraneous_dir=1))
|
||||
self.assertIn('1/1 file(s) and 1/1 folder(s) deleted', res.stdout)
|
||||
self.assertFalse(
|
||||
utils.does_directory_exist_remotely(remote_root_path + 'emptydir1'))
|
||||
self.assertFalse(
|
||||
utils.does_file_exist_remotely(remote_root_path + 'dir2/file1_2.txt'))
|
||||
|
||||
def test_copy_dest(self):
|
||||
r"""Runs rsync with --copy-dest option.
|
||||
|
||||
Copies testdata.dat to
|
||||
Copies the "cdc_rsync_e2e_test" package locally and syncs it with
|
||||
--copy-dest. Verifies that the files are actually sync'ed (D), not
|
||||
copied (C).
|
||||
|
||||
Raises:
|
||||
Exception: On timeout waiting for mount to appear (after 20 seconds)
|
||||
"""
|
||||
|
||||
copy_dest_dir = self.remote_base_dir + 'copy_dest_dir'
|
||||
|
||||
utils.create_test_file(self.local_data_path, 1024)
|
||||
res = utils.run_rsync(self.local_data_path, copy_dest_dir)
|
||||
self._assert_rsync_success(res)
|
||||
|
||||
# Upload package using --package.
|
||||
res = utils.run_rsync('--copy-dest', copy_dest_dir, self.local_data_path,
|
||||
self.remote_base_dir, '-v')
|
||||
self._assert_rsync_success(res)
|
||||
self.assertIn('D100%', res.stdout)
|
||||
self.assertNotIn('C100%', res.stdout)
|
||||
|
||||
def test_upload_executables(self):
|
||||
"""Uploads executable files and checks that they have the x bit set."""
|
||||
|
||||
# Use the cdc rsync binaries as test executables.
|
||||
local_exe_path = utils.CDC_RSYNC_PATH
|
||||
local_elf_path = os.path.join(
|
||||
os.path.dirname(local_exe_path), 'cdc_rsync_server')
|
||||
|
||||
remote_exe_path = self.remote_base_dir + os.path.basename(local_exe_path)
|
||||
remote_elf_path = self.remote_base_dir + os.path.basename(local_elf_path)
|
||||
|
||||
# Copy the files to the gamelet.
|
||||
res = utils.run_rsync(local_exe_path, local_elf_path, self.remote_base_dir)
|
||||
self._assert_rsync_success(res)
|
||||
|
||||
# Check that both files have the executable bit set.
|
||||
stats = utils.get_ssh_command_output('stat -c "%%a" %s %s' %
|
||||
(remote_exe_path, remote_elf_path))
|
||||
self.assertEqual(stats.count('755'), 2, stats)
|
||||
|
||||
# Remove executable bits.
|
||||
utils.get_ssh_command_output('chmod -x %s %s' %
|
||||
(remote_exe_path, remote_elf_path))
|
||||
|
||||
# Sync again, using -c to force a sync.
|
||||
res = utils.run_rsync('-c', local_exe_path, local_elf_path,
|
||||
self.remote_base_dir)
|
||||
self._assert_rsync_success(res)
|
||||
|
||||
# Validate that the executable bits were restored.
|
||||
stats = utils.get_ssh_command_output('stat -c "%%a" %s %s' %
|
||||
(remote_exe_path, remote_elf_path))
|
||||
self.assertEqual(stats.count('755'), 2, stats)
|
||||
|
||||
def _run(self, args):
|
||||
logging.debug('Running %s', ' '.join(args))
|
||||
res = subprocess.run(args, capture_output=True)
|
||||
self.assertEqual(res.returncode, 0, 'Command failed: ' + str(res))
|
||||
res.stdout = res.stdout.decode('ascii')
|
||||
logging.debug('\r\n%s', res.stdout)
|
||||
return res
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_base.test_base.main()
|
||||
Reference in New Issue
Block a user