[cdc_rsync] Add integration tests (#42)

[cdc_rsync] Add integration tests

This CL adds Python integration tests for cdc_rsync. To run the tests,
you need to supply a Linux host and proper configuration for cdc_rsync
to work:

  set CDC_SSH_COMMAND=C:\path\to\ssh.exe <args>
  set CDC_SCP_COMMAND=C:\path\to\scp.exe <args>
  C:\python38\python.exe -m integration_tests.cdc_rsync.all_tests --binary_path=C:\full\path\to\cdc_rsync.exe --user_host=user@host

Ran the tests and made sure they worked.

.gitignore

@@ -12,3 +12,4 @@ dependencies
 .qtc_clangd
 bazel-*
 user.bazelrc
+*.pyc

integration_tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

integration_tests/cdc_rsync/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

integration_tests/cdc_rsync/all_tests.py

@@ -0,0 +1,44 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import unittest
+
+from integration_tests.cdc_rsync import connection_test
+from integration_tests.cdc_rsync import deployment_test
+from integration_tests.cdc_rsync import dry_run_test
+from integration_tests.cdc_rsync import output_test
+from integration_tests.cdc_rsync import upload_test
+from integration_tests.framework import test_base
+
+
+# pylint: disable=g-doc-args,g-doc-return-or-yield
+def load_tests(loader, unused_tests, unused_pattern):
+  """Customizes the list of test cases to run.
+
+  See the Python documentation for details:
+  https://docs.python.org/3/library/unittest.html#load-tests-protocol
+  """
+  suite = unittest.TestSuite()
+  suite.addTests(loader.loadTestsFromModule(connection_test))
+  suite.addTests(loader.loadTestsFromModule(deployment_test))
+  suite.addTests(loader.loadTestsFromModule(dry_run_test))
+  suite.addTests(loader.loadTestsFromModule(output_test))
+  suite.addTests(loader.loadTestsFromModule(upload_test))
+
+  return suite
+
+
+if __name__ == '__main__':
+  test_base.main()

integration_tests/cdc_rsync/connection_test.py

@@ -0,0 +1,131 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync connection test."""
+
+from concurrent import futures
+import socket
+import time
+
+from integration_tests.framework import utils
+from integration_tests.cdc_rsync import test_base
+
+RETURN_CODE_SUCCESS = 0
+RETURN_CODE_GENERIC_ERROR = 1
+RETURN_CODE_CONNECTION_TIMEOUT = 2
+RETURN_CODE_ADDRESS_IN_USE = 4
+
+FIRST_PORT = 44450
+LAST_PORT = 44459
+
+
+class ConnectionTest(test_base.CdcRsyncTest):
+  """cdc_rsync connection test class."""
+
+  def test_valid_instance(self):
+    """Runs rsync with --instance option for a valid id.
+
+       1) Uploads a file with --instance option instead of --ip --port.
+       2) Checks the file exists on the used instance.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.does_file_exist_remotely(self.remote_data_path))
+
+  def test_invalid_instance(self):
+    """Runs rsync with --instance option for an invalid id.
+
+       1) Uploads a file with --instance option for a non-existing id.
+       2) Checks the error message.
+    """
+    bad_host = 'bad_host'
+
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path,
+                          bad_host + ":" + self.remote_base_dir)
+    self.assertEqual(res.returncode, RETURN_CODE_GENERIC_ERROR)
+    self.assertIn('lost connection', str(res.stderr))
+
+  def test_contimeout(self):
+    """Runs rsync with --contimeout option for an invalid ip.
+
+       1) Uploads a file with bad IP address.
+       2) Checks the error message and that it timed out after ~5 seconds.
+       3) Uploads a file with bad IP address and --contimeout 1.
+       4) Checks the error message and that it timed out after ~1 second.
+
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    bad_host = '192.0.2.1'
+    start = time.time()
+    res = utils.run_rsync(self.local_data_path,
+                          bad_host + ":" + self.remote_base_dir)
+    elapsed_time = time.time() - start
+    self.assertGreater(elapsed_time, 4.5)
+    self.assertEqual(res.returncode, RETURN_CODE_CONNECTION_TIMEOUT)
+    self.assertIn('Error: Server connection timed out', str(res.stderr))
+
+    start = time.time()
+    res = utils.run_rsync(self.local_data_path,
+                          bad_host + ":" + self.remote_base_dir,
+                          '--contimeout=1')
+    elapsed_time = time.time() - start
+    self.assertLess(elapsed_time, 3)
+    self.assertEqual(res.returncode, RETURN_CODE_CONNECTION_TIMEOUT)
+    self.assertIn('Error: Server connection timed out', str(res.stderr))
+
+  def test_multiple_instances(self):
+    """Runs multiple instances of rsync at the same time."""
+    num_instances = LAST_PORT - FIRST_PORT + 1
+
+    local_data_paths = []
+    for n in range(num_instances):
+      path = self.local_base_dir + ('testdata_%i.dat' % n)
+      utils.create_test_file(path, 1024)
+      local_data_paths.append(path)
+
+    with futures.ThreadPoolExecutor(max_workers=num_instances) as executor:
+      res = []
+      for n in range(num_instances):
+        res.append(
+            executor.submit(utils.run_rsync, local_data_paths[n],
+                            self.remote_base_dir))
+      for r in res:
+        self._assert_rsync_success(r.result())
+
+  def test_address_in_use(self):
+    """Blocks all ports and checks that rsync fails with the expected error."""
+    sockets = []
+    try:
+      # Occupy all ports.
+      for port in range(FIRST_PORT, LAST_PORT + 1):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sockets.append(s)
+        s.bind(('127.0.0.1', port))
+        s.listen()
+
+      # rsync shouldn't be able to find an available port now.
+      utils.create_test_file(self.local_data_path, 1024)
+      res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+      self.assertIn('All ports are already in use', str(res.stderr))
+
+    finally:
+      for s in sockets:
+        s.close()
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_rsync/deployment_test.py

@@ -0,0 +1,110 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync deployment test."""
+
+from integration_tests.framework import utils
+from integration_tests.cdc_rsync import test_base
+
+REMOTE_FOLDER = '~/.cache/cdc-file-transfer/bin/'
+
+
+class DeploymentTest(test_base.CdcRsyncTest):
+  """cdc_rsync deployment test class."""
+
+  def _assert_deployment(self, initial_ts, file, msg):
+    """Checks rsync and library are uploaded and the given file's timestamp matches initial_ts."""
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertIn(msg, str(res.stdout))
+    changed_ts = utils.get_ssh_command_output('stat --format=%%y %s' %
+                                              REMOTE_FOLDER + file)
+    self.assertEqual(initial_ts, changed_ts)
+
+  def _change_file_preserve_timestamp(self, file):
+    """Changes a file preserving it timestamp."""
+    utils.get_ssh_command_output(
+        'touch -r %s %s' %
+        (REMOTE_FOLDER + file, REMOTE_FOLDER + file + '.tmp'))
+    utils.get_ssh_command_output('truncate -s +100 %s' % REMOTE_FOLDER + file)
+    utils.get_ssh_command_output(
+        'touch -r %s %s' %
+        (REMOTE_FOLDER + file + '.tmp', REMOTE_FOLDER + file))
+    utils.get_ssh_command_output('rm  %s' % (REMOTE_FOLDER + file + '.tmp'))
+
+  def test_no_server(self):
+    """Checks that cdc_rsync_server is uploaded if not present on the gamelet.
+
+      1) Wipes ‘/opt/developer/tools/bin/’ on the gamelet.
+      2) Uploads a file.
+      3) Verifies that cdc_rsync_server exists in that folder.
+    """
+    utils.get_ssh_command_output('rm -rf %s*' % REMOTE_FOLDER)
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertIn('Server not deployed. Deploying...', str(res.stdout))
+    self._assert_remote_dir_contains(['cdc_rsync_server'],
+                                     remote_dir=REMOTE_FOLDER,
+                                     pattern='"*"')
+
+  def test_modified_server(self):
+    """Checks that cdc_rsync_server is re-uploaded.
+
+      1) Touches cdc_rsync_server in ‘REMOTE_FOLDER’.
+      2) Uploads a file.
+      3) Verifies that cdc_rsync_server is re-uploaded.
+      4) Appends a few bytes to cdc_rsync_server while keeping its timestamp.
+      6) Uploads a file.
+      7) Verifies that cdc_rsync_server is re-uploaded.
+    """
+    # To be sure that cdc_rsync_server exist on the remote system
+    # do an "empty" copy.
+    utils.run_rsync(self.local_base_dir, self.remote_base_dir)
+
+    remote_server_path = REMOTE_FOLDER + 'cdc_rsync_server'
+    initial_ts = utils.get_ssh_command_output('stat --format=%%y %s' %
+                                              remote_server_path)
+    utils.get_ssh_command_output('touch -d \'1 November 2020 00:00\' %s' %
+                                 remote_server_path)
+    changed_ts = utils.get_ssh_command_output('stat --format=%%y %s' %
+                                              remote_server_path)
+    self.assertNotEqual(initial_ts, changed_ts)
+    utils.create_test_file(self.local_data_path, 1024)
+    self._assert_deployment(initial_ts, 'cdc_rsync_server',
+                            'Server outdated. Redeploying...')
+
+    self._change_file_preserve_timestamp('cdc_rsync_server')
+    self._assert_deployment(initial_ts, 'cdc_rsync_server',
+                            'Server outdated. Redeploying...')
+
+  def test_read_only_server(self):
+    """Checks that cdc_rsync_server is overwritten if it is read-only."""
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+
+    # Modify cdc_rsync_server and wipe permissions.
+    remote_server_path = REMOTE_FOLDER + 'cdc_rsync_server'
+    utils.get_ssh_command_output('echo "xxx" > %s && chmod 0 %s' %
+                                 (remote_server_path, remote_server_path))
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertIn('Server failed to start. Redeploying...', str(res.stdout))
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_rsync/dry_run_test.py

@@ -0,0 +1,116 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync dry-run test."""
+
+from integration_tests.framework import utils
+from integration_tests.cdc_rsync import test_base
+
+
+class DryRunTest(test_base.CdcRsyncTest):
+  """cdc_rsync dry-run test class."""
+
+  def test_dry_run(self):
+    """Verifies --dry-run option.
+
+      1) Uploads file1.txt and file2.txt.
+      2) Modifies file2.txt.
+      3) Dry-runs file2.txt and file3.txt with --dry-run -r --delete.
+      Result: a missing (file3.txt), a changed (file2.txt) and an extraneous
+      (file1.txt) file. No files should be changed on the server.
+
+    """
+
+    files = ['file1.txt', 'file2.txt', 'file3.txt']
+
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 987)
+
+    res = utils.run_rsync(self.local_base_dir + 'file1.txt',
+                          self.local_base_dir + 'file2.txt',
+                          self.remote_base_dir, '-v')
+    self._assert_rsync_success(res)
+    self._assert_remote_dir_contains(['file1.txt', 'file2.txt'])
+
+    # Dry-run of uploading changed/new/to delete files.
+    utils.create_test_file(self.local_base_dir + 'file2.txt', 2534)
+    res = utils.run_rsync(self.local_base_dir + 'file2.txt',
+                          self.local_base_dir + 'file3.txt',
+                          self.remote_base_dir, '-v', '--dry-run', '--delete',
+                          '-r')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(res, missing=1, changed=1, extraneous=1))
+    self._assert_remote_dir_does_not_contain(['file3.txt'])
+    self._assert_remote_dir_contains(['file1.txt', 'file2.txt'])
+    self.assertIn('file1.txt', str(res.stdout))
+    self.assertIn('deleted 1 / 1', str(res.stdout))
+    self.assertIn('file2.txt', str(res.stdout))
+    self.assertIn('D100%', str(res.stdout))
+    self.assertIn('file3.txt', str(res.stdout))
+    self.assertIn('C100%', str(res.stdout))
+    self.assertFalse(
+        utils.sha1_matches(self.local_base_dir + 'file2.txt',
+                           self.remote_base_dir + 'file2.txt'))
+
+  def test_dry_run_sync_folder_when_remote_file_recursive_with_delete(self):
+    """Dry-runs a recursive upload of a folder while removing a remote file with the same name with --delete."""
+
+    local_folder = self.local_base_dir + 'foldertocopy\\'
+    utils.create_test_directory(local_folder)
+    utils.get_ssh_command_output(
+        'mkdir -p %s && touch %s' %
+        (self.remote_base_dir, self.remote_base_dir + 'foldertocopy'))
+
+    res = utils.run_rsync(self.local_base_dir + 'foldertocopy',
+                          self.remote_base_dir, '-r', '--dry-run', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, extraneous=1, missing_dir=1))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(self.remote_base_dir +
+                                            'foldertocopy'))
+    self.assertTrue(
+        utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy'))
+    self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout))
+
+  def test_dry_run_sync_file_when_remote_folder_recursive_with_delete(self):
+    """Dry-runs a recursive upload of a file while removing an empty remote folder with the same name with --delete."""
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          '--dry-run', '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
+    self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path))
+    self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path))
+    self.assertIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
+
+  def test_dry_run_sync_file_when_remote_folder_empty(self):
+    """Dry-runs a non-recursive upload of a file while there is an empty remote folder with the same name."""
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          '--dry-run')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
+    self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path))
+    self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path))
+    self.assertNotIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_rsync/output_test.py

@@ -0,0 +1,243 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync output test."""
+
+import json
+
+from integration_tests.framework import utils
+from integration_tests.cdc_rsync import test_base
+
+
+class OutputTest(test_base.CdcRsyncTest):
+  """cdc_rsync output test class."""
+
+  def test_plain(self):
+    """Runs rsync and verifies the total progress.
+
+      1) Uploads a file, verifies that the total progress is shown.
+      2) Uploads an empty folder with -r --delete options.
+      Verifies that the total delete messages are shown.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertIn('100% TOT', str(res.stdout))
+
+    utils.remove_test_file(self.local_data_path)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r',
+                          '--delete')
+    self._assert_rsync_success(res)
+    self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout))
+
+  def test_verbose_1(self):
+    """Runs rsync with -v option for multiple files.
+
+      1) Uploads 3 files with ‘-v’.
+      Verifies that each file is listed in the output as ‘C100%’.
+      2) Modifies 3 files, uploads them again with ‘--v’.
+      Verifies that each file is listed in the output as ‘D100%’.
+      3) Uploads an empty folder with -r --delete options.
+      Verifies that the delete messages are shown.
+    """
+    files = ['file1. txt', 'file2.txt', 'file3.txt']
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 1024)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r')
+    self._assert_rsync_success(res)
+    self.assertEqual(3, str(res.stdout).count('C100%'))
+
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 2048)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r')
+    self._assert_rsync_success(res)
+    self.assertEqual(3, str(res.stdout).count('D100%'))
+
+    for file in files:
+      utils.remove_test_file(self.local_base_dir + file)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r',
+                          '--delete')
+    self._assert_rsync_success(res)
+    self.assertIn('will be deleted due to --delete', str(res.stdout))
+    self.assertIn('3/3 file(s) and 0/0 folder(s) deleted', str(res.stdout))
+
+  def test_verbose_2(self):
+    """Runs rsync with -vv option.
+
+      1) Uploads a file with ‘-vv’.
+      2) Verifies that additional logs show up.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vv')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+
+    # client-side output
+    self._assert_regex('Starting process', output)
+    self._assert_not_regex(
+        r'process\.cc\([0-9]+\): Start\(\): Starting process', output)
+
+    # server-side output
+    self._assert_regex(
+        'INFO    Finding all files in destination folder '
+        f"'{self.remote_base_dir}'", output)
+    self.assertNotIn('DEBUG', output)
+
+  def test_verbose_3(self):
+    """Runs rsync with -vvv option.
+
+      1) Uploads a file with ‘-vvv’.
+      Verifies that additional logs show up (LOG_DEBUG logs).
+      2) Uploads a file to  ‘/invalid’ with ‘-vvv’.
+      Verifies that error messages including filenames are shown.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vvv')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+
+    # client-side output
+    self._assert_regex(
+        r'cdc_rsync_client\.cc\([0-9]+\): SendOptions\(\): Sending options',
+        output)
+
+    # server-side output
+    self._assert_regex(
+        r'DEBUG   server_socket\.cc\([0-9]+\): Receive\(\): EOF\(\) detected',
+        output)
+
+    # TODO: Add a check here, as currently the output is misleading
+    # res = utils.run_rsync(self.local_data_path, '/invalid', '-vvv')
+
+  def test_verbose_4(self):
+    """Runs rsync with -vvv option.
+
+      1) Uploads a file with ‘-vvvv’.
+      2) Verifies that additional logs show up (LOG_VERBOSE logs).
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vvvv')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+
+    # client-side output
+    self._assert_regex(
+        r'message_pump\.cc\([0-9]+\): ThreadDoSendPacket\(\): Sent packet of size',
+        output)
+
+    # server-side output
+    self._assert_regex(
+        r'VERBOSE message_pump\.cc\([0-9]+\): ThreadDoReceivePacket\(\): Received packet of size',
+        output)
+
+  def test_quiet(self):
+    """Runs rsync with -q option.
+
+      1) Uploads a file with ‘-q’.
+      2) Verifies that no output is shown.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-q')
+    self._assert_rsync_success(res)
+    self.assertEqual('\r\n', res.stdout)
+
+  def test_quiet_error(self):
+    """Runs rsync with -q option still showing errors.
+
+      1) Uploads a file with ‘-q’ and bad options.
+      2) Verifies that an error message is shown.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-q',
+                          '-t')
+    self.assertEqual(res.returncode, 1)
+    self.assertEqual('\r\n', str(res.stdout))
+    self.assertIn('Unknown option: \'t\'', str(res.stderr))
+    # TODO: Add a test case for the non-existing destination.
+
+  def test_existing_verbose_1(self):
+    """Runs rsync with -v --existing."""
+
+    files = ['file1.txt', 'file2.txt']
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 1024)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r')
+    self._assert_rsync_success(res)
+
+    files.append('file3.txt')
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 2048)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r',
+                          '--existing')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+    self.assertEqual(2, output.count('D100%'))
+    self.assertNotIn('file3.txt', output)
+
+  def test_json_per_file(self):
+    """Runs rsync with -v --json."""
+
+    local_path = self.local_base_dir + 'test.txt'
+    utils.create_test_file(local_path, 1024)
+    res = utils.run_rsync(local_path, self.remote_base_dir, '-v', '--json')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+
+    for val in self.parse_json(output):
+      self.assertEqual(val['file'], 'test.txt')
+      self.assertEqual(val['operation'], 'Copy')
+      self.assertEqual(val['size'], 1024)
+
+      # Those are actually all floats, but sometimes they get rounded to ints.
+      self.assertTrue(self.is_float_or_int(val['bytes_per_second']))
+      self.assertTrue(self.is_float_or_int(val['duration']))
+      self.assertTrue(self.is_float_or_int(val['eta']))
+      self.assertTrue(self.is_float_or_int(val['total_duration']))
+      self.assertTrue(self.is_float_or_int(val['total_eta']))
+      self.assertTrue(self.is_float_or_int(val['total_progress']))
+
+  def test_json_total(self):
+    """Runs rsync with --json."""
+
+    local_path = self.local_base_dir + 'test.txt'
+    utils.create_test_file(local_path, 1024)
+    res = utils.run_rsync(local_path, self.remote_base_dir, '--json')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+
+    for val in self.parse_json(output):
+      self.assertNotIn('file', val)
+
+      # Those are actually all floats, but sometimes they get rounded to ints.
+      self.assertTrue(self.is_float_or_int(val['total_duration']))
+      self.assertTrue(self.is_float_or_int(val['total_eta']))
+      self.assertTrue(self.is_float_or_int(val['total_progress']))
+
+  def parse_json(self, output):
+    """Parses the JSON lines of output."""
+    lines = output.split('\r\n')
+    json_values = []
+    for line in lines:
+      if str.startswith(line, '{'):
+        json_values.append(json.loads(line.strip()))
+    return json_values
+
+  def is_float_or_int(self, val):
+    """Returns true if val is a float or an int."""
+    return isinstance(val, float) or isinstance(val, int)
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_rsync/test_base.py

@@ -0,0 +1,113 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync base test class."""
+
+import datetime
+import logging
+import tempfile
+import re
+import unittest
+
+from integration_tests.framework import utils
+from integration_tests.framework import test_base
+
+
+class CdcRsyncTest(unittest.TestCase):
+  """cdc_rsync base test class."""
+
+  tmp_dir = None
+  local_base_dir = None
+  remote_base_dir = None
+  local_data_path = None
+  remote_data_path = None
+
+  def setUp(self):
+    """Cleans up the remote test data folder, logs a marker, and initializes random."""
+    super(CdcRsyncTest, self).setUp()
+    logging.debug('CdcRsyncTest -> setUp')
+
+    utils.initialize(test_base.Flags.binary_path, test_base.Flags.user_host)
+
+    now_str = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
+    self.tmp_dir = tempfile.TemporaryDirectory(
+        prefix=f'_cdc_rsync_test_{now_str}')
+    self.local_base_dir = self.tmp_dir.name + '\\'
+    self.remote_base_dir = f'/tmp/_cdc_rsync_test_{now_str}/'
+    self.local_data_path = self.local_base_dir + 'testdata.dat'
+    self.remote_data_path = self.remote_base_dir + 'testdata.dat'
+
+    logging.info('Local base dir: "%s"', self.local_base_dir)
+    logging.info('Remote base dir: "%s"', self.remote_base_dir)
+    utils.initialize_random()
+
+  def tearDown(self):
+    """Cleans up the local and remote temp directories."""
+    super(CdcRsyncTest, self).tearDown()
+    logging.debug('CdcRsyncTest -> tearDown')
+    self.tmp_dir.cleanup()
+    utils.get_ssh_command_output(f'rm -rf {self.remote_base_dir}')
+
+  def _assert_rsync_success(self, res):
+    """Asserts if the return code is 0 and outputs return message with args."""
+    self.assertEqual(res.returncode, 0, 'Return value is ' + str(res))
+
+  def _assert_regex(self, regex, value):
+    """Asserts that the regex string matches the given value."""
+    self.assertIsNotNone(
+        re.search(regex, value), f'"Regex {regex}" does not match "{value}"')
+
+  def _assert_not_regex(self, regex, value):
+    """Asserts that the regex string does not match the given value."""
+    self.assertIsNone(
+        re.search(regex, value),
+        f'"Regex {regex}" unexpectedly matches "{value}"')
+
+  def _assert_remote_dir_contains(self,
+                                  file_list,
+                                  remote_dir=None,
+                                  pattern='"*.[t|d]*"'):
+    """Asserts that the remote base dir contains exactly the list of files.
+
+    Args:
+        file_list (list of strings): List of relative file paths to check
+        remote_dir (string, optional): Remote directory. Defaults to
+            remote_base_dir
+        pattern (string, optional): Pattern for matching file names.
+    """
+    find_res = utils.get_ssh_command_output(
+        'cd %s && find -name %s -print' %
+        (remote_dir or self.remote_base_dir, pattern))
+
+    # Note that assertCountEqual compares items independently of order
+    # (not just the size of the list).
+    found = sorted(
+        filter(lambda item: item and item != '.', find_res.split('\r\n')))
+    expected = sorted(['./' + f for f in file_list])
+    self.assertListEqual(found, expected)
+
+  def _assert_remote_dir_does_not_contain(self, file_list):
+    """Asserts that the remote base dir contains none of the listed files.
+
+    Args:
+        file_list (list of strings): List of relative file paths to check
+    """
+    find_res = utils.get_ssh_command_output(
+        'cd %s && find -name "*.[t|d]*" -print' % self.remote_base_dir)
+
+    found = set(file_name for file_name in filter(None, find_res.split('\n')))
+
+    for file in file_list:
+      self.assertNotIn('./' + file, found)

integration_tests/cdc_rsync/upload_test.py

@@ -0,0 +1,894 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync upload test."""
+
+import json
+import logging
+import os
+import subprocess
+import time
+
+from integration_tests.framework import utils
+from integration_tests.cdc_rsync import test_base
+
+
+class UploadTest(test_base.CdcRsyncTest):
+  """cdc_rsync upload test class."""
+
+  def test_single_uncompressed(self):
+    """Uploads and syncs a file uncompressed."""
+
+    self._do_test_single(compressed=False)
+
+  def test_upload_compressed(self):
+    """Uploads and syncs a file compressed."""
+
+    self._do_test_single(compressed=True)
+
+  def _do_test_single(self, compressed):
+    """Runs rsync 3 times and validates results.
+
+       1) Uploads a file, checks sha1 hashes.
+       2) Uploads the same file again, checks nothing changed.
+       3) Modifies the file and uploads again. Checks sha1 hashes.
+
+    Args:
+        compressed (bool): Whether to append '--compress' or not.
+    """
+    compressed_arg = '--compress' if compressed else None
+
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, matching=1))
+
+    utils.create_test_file(self.local_data_path, 2534)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+
+  def test_backslash_in_dest_folder(self):
+    r"""Verifies uploading to \mnt\developer."""
+
+    filepath = os.path.join(self.local_base_dir, 'file1.txt')
+    utils.create_test_file(filepath, 1)
+    res = utils.run_rsync(filepath, self.remote_base_dir.replace('/', '\\'))
+    self.assertTrue(utils.files_count_is(res, missing=1))
+    self._assert_remote_dir_contains(['file1.txt'])
+
+  def test_backslash_in_source_folder(self):
+    r"""Verifies uploading from /source/folder."""
+
+    filepath = os.path.join(self.local_base_dir, 'file1.txt')
+    utils.create_test_file(filepath, 1)
+    filepath = filepath.replace('\\', '/')
+    res = utils.run_rsync(filepath, self.remote_base_dir)
+    self.assertTrue(utils.files_count_is(res, missing=1))
+    self._assert_remote_dir_contains(['file1.txt'])
+
+  def test_single_unicode(self):
+    """Uploads a file with a non-ascii unicode path and checks sha1 signatures."""
+
+    nonascii_local_data_path = self.local_base_dir + '⛽⛽⛽⛽⛽⛽⛽⛽.dat'
+    nonascii_remote_data_path = self.remote_base_dir + '⛽⛽⛽⛽⛽⛽⛽⛽.dat'
+    utils.create_test_file(nonascii_local_data_path, 1024)
+    # In order to check that non-ascii characters are not considered as
+    # wildcard
+    # ?  characters, create a second file.  Only 1 file should be uploaded.
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(nonascii_local_data_path, self.remote_base_dir, None)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1))
+    self.assertTrue(
+        utils.sha1_matches(nonascii_local_data_path, nonascii_remote_data_path))
+
+  def test_uncompressed_no_empty_folders(self):
+    """Uploads and syncs multiple files uncompressed in different folders."""
+
+    self._do_test_no_empty_folders(compressed=False)
+
+  def test_compressed_no_empty_folders(self):
+    """Uploads and syncs multiple files compressed in different folders."""
+
+    self._do_test_no_empty_folders(compressed=True)
+
+  def _do_test_no_empty_folders(self, compressed):
+    """Runs rsync with(out) -r for a non-trivial directory and validates results.
+
+       1) Uploads a source directory with -r, checks sha1 hashes.
+       |-- rootdir
+       |   |-- dir1
+       |      |-- file1_1.txt
+       |      |-- file1_2.txt
+       |   |-- dir2
+       |      |-- file2_1.txt
+       |   |-- file0.txt
+       2) Uploads the same source directory again without -r,
+       checks nothing has changed. The directory should be just skipped.
+       3) Uploads the same source directory with --delete option and with -r.
+       Nothing should change.
+       4) Removes dir1 and dir2 locally.
+       Uploads the same source directory with --delete option and with -r.
+       dir1 and dir2 should be removed from the remote instance.
+
+    Args:
+        compressed (bool): Whether to append '--compress' or not.
+    """
+    compressed_arg = '--compress' if compressed else None
+    local_root_path = self.local_base_dir + 'rootdir'
+    remote_root_path = self.remote_base_dir + 'rootdir/'
+    utils.create_test_file(local_root_path + '\\dir1\\file1_1.txt', 1024)
+    utils.create_test_file(local_root_path + '\\dir1\\file1_2.txt', 1024)
+    utils.create_test_file(local_root_path + '\\dir2\\file2_1.txt', 1024)
+    utils.create_test_file(local_root_path + '\\file0.txt', 1024)
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=3))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir1\\file1_1.txt',
+                           remote_root_path + 'dir1/file1_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir1\\file1_2.txt',
+                           remote_root_path + 'dir1/file1_2.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir2\\file2_1.txt',
+                           remote_root_path + 'dir2/file2_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\file0.txt',
+                           remote_root_path + 'file0.txt'))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, extraneous_dir=1))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=3))
+
+    utils.remove_test_directory(local_root_path + '\\dir1\\')
+    utils.remove_test_directory(local_root_path + '\\dir2\\')
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(
+            res, matching=1, extraneous=3, matching_dir=1, extraneous_dir=2))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'dir1'))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'dir2'))
+
+  def _do_test_no_empty_folders_with_backslash(self, compressed):
+    """Runs rsync with(out) -r for a non-trivial directory with a trailing backslash.
+
+       1) Uploads a source directory with -r, checks sha1 hashes.
+       Everything from rootdir should be copied except rootdir itself.
+       |-- rootdir
+       |   |-- dir1
+       |      |-- file1_1.txt
+       |      |-- file1_2.txt
+       |   |-- dir2
+       |      |-- file2_1.txt
+       |   |-- file0.txt
+       2) Uploads the same source directory again without -r,
+       checks nothing has changed. The directory should be just skipped.
+       3) Uploads the same source directory with --delete option and with -r.
+       Nothing should change.
+       4) Removes dir1 and dir2 locally.
+       Uploads the same source directory with --delete option and with -r.
+       dir1 and dir2 should be removed from the remote instance.
+
+    Args:
+        compressed (bool): Whether to append '--compress' or not.
+    """
+    compressed_arg = '--compress' if compressed else None
+    local_root_path = self.local_base_dir + 'rootdir\\'
+    utils.create_test_file(local_root_path + 'dir1\\file1_1.txt', 1024)
+    utils.create_test_file(local_root_path + 'dir1\\file1_2.txt', 1024)
+    utils.create_test_file(local_root_path + 'dir2\\file2_1.txt', 1024)
+    utils.create_test_file(local_root_path + 'file0.txt', 1024)
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=2))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + 'dir1\\file1_1.txt',
+                           self.remote_base_dir + 'dir1/file1_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + 'dir1\\file1_2.txt',
+                           self.remote_base_dir + 'dir1/file1_2.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + 'dir2\\file2_1.txt',
+                           self.remote_base_dir + 'dir2/file2_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + 'file0.txt',
+                           self.remote_base_dir + 'file0.txt'))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(
+        res, extraneous=1, extraneous_dir=2))  # file0.txt, dir1, dir2
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=2))
+
+    utils.remove_test_directory(local_root_path + '\\dir1\\')
+    utils.remove_test_directory(local_root_path + '\\dir2\\')
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(res, matching=1, extraneous=3, extraneous_dir=2))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(self.remote_base_dir + 'dir1'))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(self.remote_base_dir + 'dir2'))
+
+  def test_uncompressed_no_empty_folders_with_backslash(self):
+    """Uploads multiple files uncompressed from a folder with a trailing backslash."""
+
+    self._do_test_no_empty_folders_with_backslash(compressed=False)
+
+  def test_compressed_no_empty_folders_with_backslash(self):
+    """Uploads multiple files compressed from a folder with a trailing backslash."""
+
+    self._do_test_no_empty_folders_with_backslash(compressed=True)
+
+  def test_uncompressed_with_empty_folders(self):
+    """Uploads and syncs multiple files uncompressed and empty folders."""
+
+    self._do_test_with_empty_folders(compressed=False)
+
+  def test_compressed_with_empty_folders(self):
+    """Uploads and syncs multiple files compress and empty folders."""
+
+    self._do_test_with_empty_folders(compressed=True)
+
+  def _do_test_with_empty_folders(self, compressed):
+    """Runs rsync with(out) -r for a non-trivial directory with empty folders.
+
+       1) Uploads a source directory with -r, checks sha1 hashes.
+       |-- rootdir
+       |   |-- dir1
+       |      |-- emptydir2
+       |      |-- file1_1.txt
+       |      |-- file1_2.txt
+       |   |-- dir2
+       |      |-- file2_1.txt
+       |   |-- emptydir1
+       |   |-- file0.txt
+       2) Uploads the same source directory again without -r,
+       checks nothing has changed. The directory should be just skipped.
+       3) Uploads the same source directory with --delete option and with -r.
+       Nothing should change.
+       4) Removes dir1 and dir2 locally.
+       Uploads the same source directory with --delete option and with -r.
+       dir1 and dir2 should be removed from the remote instance.
+
+    Args:
+        compressed (bool): Whether to append '--compress' or not.
+    """
+    compressed_arg = '--compress' if compressed else None
+    local_root_path = self.local_base_dir + 'rootdir'
+    remote_root_path = self.remote_base_dir + 'rootdir/'
+    utils.create_test_file(local_root_path + '\\dir1\\file1_1.txt', 1024)
+    utils.create_test_file(local_root_path + '\\dir1\\file1_2.txt', 1024)
+    utils.create_test_directory(local_root_path + '\\dir1\\emptydir2\\')
+    utils.create_test_file(local_root_path + '\\dir2\\file2_1.txt', 1024)
+    utils.create_test_file(local_root_path + '\\file0.txt', 1024)
+    utils.create_test_directory(local_root_path + '\\emptydir1\\')
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=5))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir1\\file1_1.txt',
+                           remote_root_path + 'dir1/file1_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir1\\file1_2.txt',
+                           remote_root_path + 'dir1/file1_2.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir2\\file2_1.txt',
+                           remote_root_path + 'dir2/file2_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\file0.txt',
+                           remote_root_path + 'file0.txt'))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, extraneous_dir=1))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=5))
+
+    utils.remove_test_directory(local_root_path + '\\dir1\\')
+    utils.remove_test_directory(local_root_path + '\\dir2\\')
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(
+            res, matching=1, extraneous=3, matching_dir=2, extraneous_dir=3))
+    self.assertIn('3/3 file(s) and 3/3 folder(s) deleted', res.stdout)
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'dir1'))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'dir2'))
+
+  def test_upload_empty_file(self):
+    """Uploads an empty file and checks sha1 signatures."""
+
+    empty_local_data_path = self.local_base_dir + 'emptyfile.dat'
+    empty_remote_data_path = self.remote_base_dir + 'emptyfile.dat'
+    utils.create_test_file(empty_local_data_path, 0)
+    res = utils.run_rsync(empty_local_data_path, self.remote_base_dir, None)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1))
+    self.assertTrue(
+        utils.sha1_matches(empty_local_data_path, empty_remote_data_path))
+
+  def test_upload_empty_folder_with_backslash(self):
+    """Uploads an empty folder with a trailing backslash."""
+
+    self._do_test_upload_empty_folder(with_backslash=True)
+
+  def test_upload_empty_folder_no_backslash(self):
+    """Uploads an empty folder without a trailing backslash."""
+
+    self._do_test_upload_empty_folder(with_backslash=False)
+
+  def _do_test_upload_empty_folder(self, with_backslash=False):
+    """Uploads an empty folder."""
+
+    local_data_dir = (
+        self.local_base_dir +
+        'empty_folder\\' if with_backslash else self.local_base_dir +
+        'empty_folder')
+    res = utils.run_rsync(local_data_dir, self.remote_base_dir, None)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=0))
+
+  def test_whole_file_uncompressed(self):
+    """Uploads and syncs a file uncompressed with --whole-file."""
+
+    self._do_test_whole_file(compressed=False)
+
+  def test_whole_file_compressed(self):
+    """Uploads and syncs a file compressed with --whole-file."""
+
+    self._do_test_whole_file(compressed=True)
+
+  def _do_test_whole_file(self, compressed):
+    """Runs rsync 3 times with --whole-file -v options and validates results.
+
+       1) Uploads a file.
+       2) Modifies the file and uploads it with --whole-file and -v options.
+       Checks the output contains C100%, not D100%.
+       3) Modifies the file and uploads it with -W and -v options.
+       Checks the output contains C100%, not D100%.
+
+    Args:
+        compressed (bool): Whether to append '--compress' or not.
+    """
+    compressed_arg = '--compress' if compressed else None
+
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg)
+
+    utils.create_test_file(self.local_data_path, 2534)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg, '--whole-file', '-v')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    self.assertIn('will be copied due to -W/--whole-file', str(res.stdout))
+    self.assertIn('C100%', str(res.stdout))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+
+    utils.create_test_file(self.local_data_path, 3456)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg, '-W', '-v')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    self.assertIn('C100%', str(res.stdout))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+
+  def test_keep_file_permissions(self):
+    """Verifies that file permissions are kept for changed files."""
+
+    # Upload a file and check permissions.
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
+    self.assertIn('-rw-r--r--', ls_res)
+
+    # Add executable bit.
+    utils.get_ssh_command_output('chmod a+x %s*' % self.remote_data_path)
+    ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
+    self.assertIn('-rwxr-xr-x', ls_res)
+
+    # Sync file again and verify permissions don't change.
+    utils.create_test_file(self.local_data_path, 1337)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
+    self.assertIn('-rwxr-xr-x', ls_res)
+
+  def test_include_exclude(self):
+    """Verifies the --include and --exclude options."""
+
+    files = [
+        'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
+        'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
+    ]
+
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 987)
+
+    # Upload file2.txt and file3.dat.
+    res = utils.run_rsync(self.local_base_dir + '*', self.remote_base_dir, '-r',
+                          '--include=*\\file2.txt', '--exclude=*.txt')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=3))
+    self._assert_remote_dir_contains(['folder1/file2.txt', 'folder1/file3.dat'])
+
+    # Upload all except *.dat with --delete, make sure file3.dat is kept.
+    utils.remove_test_file(self.local_base_dir + 'folder1\\file3.dat')
+    res = utils.run_rsync(self.local_base_dir + '*', self.remote_base_dir, '-r',
+                          '--delete', '--exclude=*.dat')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(res, missing=3, matching=1, matching_dir=3))
+    self._assert_remote_dir_contains([
+        'file1.txt', 'folder1/file2.txt', 'folder1/file3.dat',
+        'folder1/folder2/file4.txt', 'folder3/file5.txt'
+    ])
+
+  def test_exclude_include_from(self):
+    """Verifies the --include-from and --exclude-from options."""
+
+    files = [
+        'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
+        'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
+    ]
+
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 987)
+
+    include_file = self.local_base_dir + 'include.txt'
+    with open(include_file, 'wt') as f:
+      f.writelines(['file1.txt\n', 'folder3\\file5.txt'])
+
+    exclude_file = self.local_base_dir + 'exclude.txt'
+    with open(exclude_file, 'wt') as f:
+      f.writelines(['*.txt'])
+
+    res = utils.run_rsync('-r', '--include-from', include_file,
+                          '--exclude-from', exclude_file,
+                          self.local_base_dir + '*', self.remote_base_dir)
+    self.assertTrue(utils.files_count_is(res, missing=3, missing_dir=3))
+    self._assert_remote_dir_contains(
+        ['file1.txt', 'folder1/file3.dat', 'folder3/file5.txt'])
+
+  def test_files_from(self):
+    """Verifies the --files-from option."""
+
+    files = [
+        'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
+        'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
+    ]
+
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 987)
+
+    sources_file = self.local_base_dir + 'sources.txt'
+    with open(sources_file, 'wt') as f:
+      f.writelines([
+          'file1.txt\n',
+          '\n',
+          '  folder1\\file3.dat  \n',
+          'folder1\\.\\folder2\\file4.txt\n',  # .\\ = rel path marker
+          '  folder3\\file5.txt\n',
+          '\n'
+      ])
+
+    res = utils.run_rsync('--files-from', sources_file, self.local_base_dir,
+                          self.remote_base_dir)
+    self.assertTrue(utils.files_count_is(res, missing=4))
+    self._assert_remote_dir_contains([
+        'file1.txt', 'folder1/file3.dat', 'folder2/file4.txt',
+        'folder3/file5.txt'
+    ])
+
+    # Upload again to check that nothing changes.
+    res = utils.run_rsync('--files-from', sources_file, self.local_base_dir,
+                          self.remote_base_dir)
+    self.assertTrue(utils.files_count_is(res, matching=4, extraneous_dir=3))
+
+  def test_checksum_file(self):
+    """Uploads and syncs a file with --checksum.
+
+       1) Uploads a file.
+       2) Uploads a file with --checksum option. As the file was not changed, it
+       is recognized as matched. The output should contain D100%.
+       3) Uploads the same file with --whole-file --checksum -v.
+       Checks the output contains C100%, not D100%.
+       4) Modifies the file without changing its content. The file is
+       synchronized, the output should contain D100%.
+
+    """
+
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          '--checksum', '-v')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, matching=1))
+    self.assertIn('D100%', str(res.stdout))
+    self.assertIn('will be synced due to -c/--checksum', str(res.stdout))
+
+    utils.create_test_file(self.local_data_path, 2534)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          '--checksum', '-v', '--whole-file')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    self.assertIn('C100%', str(res.stdout))
+    self.assertIn('will be copied due to -c/--checksum and -W/--whole-file',
+                  str(res.stdout))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+
+    utils.change_modified_time(self.local_data_path)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-c',
+                          '-v')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    self.assertIn('D100%', str(res.stdout))
+
+  def test_sync_folder_when_remote_file_non_recursive(self):
+    """Non-recursively uploads a folder while there is a remote file with the same name."""
+
+    local_folder = self.local_base_dir + 'foldertocopy\\'
+    utils.create_test_directory(local_folder)
+    utils.get_ssh_command_output(
+        'mkdir -p %s && touch %s' %
+        (self.remote_base_dir, self.remote_base_dir + 'foldertocopy'))
+
+    res = utils.run_rsync(self.local_base_dir + 'foldertocopy',
+                          self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, extraneous=1))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(self.remote_base_dir +
+                                            'foldertocopy'))
+    self.assertTrue(
+        utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy'))
+
+  def test_sync_folder_when_remote_file_recursive_with_delete(self):
+    """Recursively uploads a folder while removing a remote file with the same name with --delete."""
+
+    local_folder = self.local_base_dir + 'foldertocopy\\'
+    utils.create_test_directory(local_folder)
+    utils.get_ssh_command_output(
+        'mkdir -p %s && touch %s' %
+        (self.remote_base_dir, self.remote_base_dir + 'foldertocopy'))
+
+    res = utils.run_rsync(self.local_base_dir + 'foldertocopy',
+                          self.remote_base_dir, '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, extraneous=1, missing_dir=1))
+    self.assertTrue(
+        utils.does_directory_exist_remotely(self.remote_base_dir +
+                                            'foldertocopy'))
+    self.assertFalse(
+        utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy'))
+    self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout))
+
+  def test_sync_file_when_remote_folder_recursive_with_delete(self):
+    """Recursively uploads a file while removing a remote folder with the same name with --delete."""
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          '--delete', '-r')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+    self.assertFalse(utils.does_directory_exist_remotely(self.remote_data_path))
+    self.assertIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
+
+  def test_sync_file_when_remote_folder_empty_non_recursive(self):
+    """Non-recursively uploads a file while there is an empty remote folder with the same name."""
+    self._do_test_sync_file_when_remote_folder_empty(recursive=False)
+
+  def test_sync_file_when_remote_folder_empty_recursive(self):
+    """Recursively uploads a file while there is an empty remote folder with the same name."""
+    self._do_test_sync_file_when_remote_folder_empty(recursive=True)
+
+  def _do_test_sync_file_when_remote_folder_empty(self, recursive):
+    """Uploads a file while there is an empty remote folder with the same name.
+
+    Args:
+        recursive (bool): Whether to append '-r' or not.
+    """
+    flag = '-r' if recursive else None
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, flag)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+    self.assertFalse(utils.does_directory_exist_remotely(self.remote_data_path))
+    self.assertNotIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
+
+  def test_sync_file_when_remote_folder_non_empty_non_recursive(self):
+    """Non-recursively uploads a file while there is a non-empty remote folder with the same name."""
+    self._do_test_sync_file_when_remote_folder_non_empty(recursive=False)
+
+  def test_sync_file_when_remote_folder_non_empty_recursive(self):
+    """Recursively uploads a file while there is a non-empty remote folder with the same name."""
+    self._do_test_sync_file_when_remote_folder_non_empty(recursive=True)
+
+  def _do_test_sync_file_when_remote_folder_non_empty(self, recursive):
+    """Uploads a file while there is a non-empty remote folder with the same name.
+
+    Args:
+        recursive (bool): Whether to append '-r' or not.
+    """
+    flag = '-r' if recursive else None
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
+    utils.get_ssh_command_output(
+        'mkdir -p %s && touch %s' %
+        (self.remote_base_dir, self.remote_data_path + '/file1.txt'))
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, flag)
+    self.assertIn('remove() failed: Directory not empty.', str(res.stderr))
+    if recursive:
+      self.assertTrue(
+          utils.files_count_is(res, missing=1, extraneous=1, extraneous_dir=1))
+    else:
+      self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
+    self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path))
+    self.assertTrue(
+        utils.does_file_exist_remotely(self.remote_data_path + '/file1.txt'))
+    self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path))
+
+  def test_upload_from_dot(self):
+    """Uploads files from the current directory ('.')."""
+    utils.create_test_file(self.local_base_dir + 'file1.txt', 1024)
+    utils.create_test_file(self.local_base_dir + 'dir\\file2.txt', 1024)
+
+    prev_cwd = os.getcwd()
+    os.chdir(self.local_base_dir)
+    try:
+      # Uploading recursivly should pick up all files and dirs.
+      res = utils.run_rsync('.', self.remote_base_dir, '-r')
+      self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=1))
+      self._assert_remote_dir_contains(['file1.txt', 'dir/file2.txt'])
+
+      # Uploading again should not change anything.
+      res = utils.run_rsync('.', self.remote_base_dir, '-r')
+      self.assertTrue(utils.files_count_is(res, matching=2, matching_dir=1))
+
+      # Verify that non-recursive uploads do nothing.
+      res = utils.run_rsync('.', self.remote_base_dir)
+      self.assertTrue(utils.files_count_is(res, extraneous=1, extraneous_dir=1))
+    finally:
+      os.chdir(prev_cwd)
+
+  def test_upload_from_dotdot(self):
+    """Uploads files from the parent directory ('..')."""
+    utils.create_test_file(self.local_base_dir + 'file1.txt', 1024)
+    utils.create_test_file(self.local_base_dir + 'dir\\file2.txt', 1024)
+
+    prev_cwd = os.getcwd()
+    os.chdir(self.local_base_dir + 'dir')
+    try:
+      # Uploading recursivly should pick up all files and dirs.
+      res = utils.run_rsync('..', self.remote_base_dir, '-r')
+      self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=1))
+      self._assert_remote_dir_contains(['file1.txt', 'dir/file2.txt'])
+
+      # Uploading again should not change anything.
+      res = utils.run_rsync('..', self.remote_base_dir, '-r')
+      self.assertTrue(utils.files_count_is(res, matching=2, matching_dir=1))
+
+      # Verify that non-recursive uploads do nothing.
+      res = utils.run_rsync('..', self.remote_base_dir)
+      self.assertTrue(utils.files_count_is(res, extraneous=1, extraneous_dir=1))
+    finally:
+      os.chdir(prev_cwd)
+
+  def test_existing(self):
+    """Runs rsync with --existing for a non-trivial directory.
+
+       1) Uploads a source directory with -r.
+       |-- rootdir
+       |   |-- dir1
+       |      |-- emptydir2
+       |      |-- file1_1.txt
+       |      |-- file1_2.txt -> rename to file1_3.txt (step 2)
+       |      |-- (step2) emptydir3
+       |   |-- dir2
+       |      |-- file2_1.txt
+       |   |-- emptydir1      -> rename emptydir4 (step 2)
+       |   |-- file0.txt      -> change (step 2)
+       2) Add new files/folders, remove and change some files/folders.
+       3) Uploads the same source directory with --existing option and with -r.
+       Only files existing on the server are changed, nothing is removed.
+       4) Uploads the same source directory with --existing --delete -r.
+       Files non-existing on the server are deleted.
+    """
+    local_root_path = self.local_base_dir + 'rootdir'
+    remote_root_path = self.remote_base_dir + 'rootdir/'
+
+    files = [
+        '\\dir1\\file1_1.txt', '\\dir1\\file1_2.txt', '\\dir2\\file2_1.txt',
+        '\\file0.txt'
+    ]
+    for file in files:
+      utils.create_test_file(local_root_path + file, 1024)
+    dirs = ['\\dir1\\emptydir2\\', '\\emptydir1\\']
+    for directory in dirs:
+      utils.create_test_directory(local_root_path + directory)
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r')
+    self._assert_rsync_success(res)
+
+    utils.remove_test_file(local_root_path + '\\dir1\\file1_2.txt')
+    utils.create_test_file(local_root_path + '\\dir1\\file1_3.txt', 1024)
+    utils.create_test_directory(local_root_path + '\\dir1\\emptydir3\\')
+    utils.remove_test_directory(local_root_path + '\\emptydir1\\')
+    utils.create_test_directory(local_root_path + '\\emptydir4\\')
+    utils.create_test_file(local_root_path + '\\file0.txt', 2034)
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r',
+                          '--existing')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(
+            res,
+            missing=1,
+            missing_dir=2,
+            matching=2,
+            matching_dir=4,
+            changed=1,
+            extraneous=1,
+            extraneous_dir=1))
+    self.assertTrue(
+        utils.does_directory_exist_remotely(remote_root_path + 'emptydir1'))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'emptydir4'))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path +
+                                            'dir1/emptydir3'))
+    self.assertTrue(
+        utils.does_file_exist_remotely(remote_root_path + 'dir1/file1_2.txt'))
+    self.assertFalse(
+        utils.does_file_exist_remotely(remote_root_path + 'dir1/file1_3.txt'))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r',
+                          '--existing', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(
+            res,
+            missing=1,
+            missing_dir=2,
+            matching=3,
+            matching_dir=4,
+            extraneous=1,
+            extraneous_dir=1))
+    self.assertIn('1/1 file(s) and 1/1 folder(s) deleted', res.stdout)
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'emptydir1'))
+    self.assertFalse(
+        utils.does_file_exist_remotely(remote_root_path + 'dir2/file1_2.txt'))
+
+  def test_copy_dest(self):
+    r"""Runs rsync with --copy-dest option.
+
+       Copies testdata.dat to 
+       Copies the "cdc_rsync_e2e_test" package locally and syncs it with
+       --copy-dest. Verifies that the files are actually sync'ed (D), not
+       copied (C).
+
+    Raises:
+        Exception: On timeout waiting for mount to appear (after 20 seconds)
+    """
+
+    copy_dest_dir = self.remote_base_dir + 'copy_dest_dir'
+
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, copy_dest_dir)
+    self._assert_rsync_success(res)
+
+    # Upload package using --package.
+    res = utils.run_rsync('--copy-dest', copy_dest_dir, self.local_data_path,
+                          self.remote_base_dir, '-v')
+    self._assert_rsync_success(res)
+    self.assertIn('D100%', res.stdout)
+    self.assertNotIn('C100%', res.stdout)
+
+  def test_upload_executables(self):
+    """Uploads executable files and checks that they have the x bit set."""
+
+    # Use the cdc rsync binaries as test executables.
+    local_exe_path = utils.CDC_RSYNC_PATH
+    local_elf_path = os.path.join(
+        os.path.dirname(local_exe_path), 'cdc_rsync_server')
+
+    remote_exe_path = self.remote_base_dir + os.path.basename(local_exe_path)
+    remote_elf_path = self.remote_base_dir + os.path.basename(local_elf_path)
+
+    # Copy the files to the gamelet.
+    res = utils.run_rsync(local_exe_path, local_elf_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+
+    # Check that both files have the executable bit set.
+    stats = utils.get_ssh_command_output('stat -c "%%a" %s %s' %
+                                         (remote_exe_path, remote_elf_path))
+    self.assertEqual(stats.count('755'), 2, stats)
+
+    # Remove executable bits.
+    utils.get_ssh_command_output('chmod -x %s %s' %
+                                 (remote_exe_path, remote_elf_path))
+
+    # Sync again, using -c to force a sync.
+    res = utils.run_rsync('-c', local_exe_path, local_elf_path,
+                          self.remote_base_dir)
+    self._assert_rsync_success(res)
+
+    # Validate that the executable bits were restored.
+    stats = utils.get_ssh_command_output('stat -c "%%a" %s %s' %
+                                         (remote_exe_path, remote_elf_path))
+    self.assertEqual(stats.count('755'), 2, stats)
+
+  def _run(self, args):
+    logging.debug('Running %s', ' '.join(args))
+    res = subprocess.run(args, capture_output=True)
+    self.assertEqual(res.returncode, 0, 'Command failed: ' + str(res))
+    res.stdout = res.stdout.decode('ascii')
+    logging.debug('\r\n%s', res.stdout)
+    return res
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/framework/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

integration_tests/framework/test_base.py

@@ -0,0 +1,63 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Test main and flags."""
+
+import argparse
+import contextlib
+import logging
+import sys
+import unittest
+
+from integration_tests.framework import test_runner
+
+
+class Flags(object):
+  binary_path = None
+  user_host = None
+  ssh_port = 22
+
+
+def main():
+  parser = argparse.ArgumentParser(description='End-to-end integration test.')
+  parser.add_argument('--binary_path', help='Target [user@]host', required=True)
+  parser.add_argument('--user_host', help='Target [user@]host', required=True)
+  parser.add_argument(
+      '--ssh_port',
+      type=int,
+      help='SSH port for connecting to the host',
+      default=22)
+  parser.add_argument('--log_file', help='Log file path')
+
+  # Capture all remaining arguments to pass to unittest.main().
+  args, unittest_args = parser.parse_known_args()
+  Flags.binary_path = args.binary_path
+  Flags.user_host = args.user_host
+  Flags.ssh_port = args.ssh_port
+
+  # Log to STDERR
+  log_format = ('%(levelname)-8s%(asctime)s '
+                '%(filename)s:%(lineno)-3d %(message)s')
+  log_stream = sys.stderr
+
+  if args.log_file:
+    log_stream = open(args.log_file, 'w')
+
+  with log_stream:
+    logging.basicConfig(
+        format=log_format, level=logging.DEBUG, stream=log_stream)
+
+    unittest.main(
+        argv=sys.argv[:1] + unittest_args, testRunner=test_runner.TestRunner())

integration_tests/framework/test_runner.py

@@ -0,0 +1,66 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Test runner, adds some sugar around logs to make them easier to read."""
+
+import logging
+import traceback
+import unittest
+
+
+class TestRunner(object):
+  """Runner producing test xml output."""
+
+  def run(self, test):  # pylint: disable=invalid-name
+    result = TestResult()
+    logging.info('Running tests...')
+    test(result)
+    logging.info('\n\n******************* TESTS FINISHED *******************\n')
+    logging.info('Ran %d tests with %d errors and %d failures', result.testsRun,
+                 len(result.errors), len(result.failures))
+    for test_and_stack in result.failures:
+      logging.info('\n\n[  TEST FAILED  ] %s\n', test_and_stack[0])
+      logging.info(
+          '%s', test_and_stack[1].replace('\\\\r',
+                                          '\r').replace('\\\\n', '\n').replace(
+                                              '\\r', '\r').replace('\\n', '\n'))
+
+    return result
+
+
+class TestResult(unittest.TestResult):
+
+  def startTest(self, test):
+    """Called when the given test is about to be run."""
+    logging.info('\n\n===== BEGIN TEST CASE: %s =====\n', test)
+    unittest.TestResult.startTest(self, test)
+
+  def stopTest(self, test):
+    """Called when the given test has been run."""
+    unittest.TestResult.stopTest(self, test)
+    logging.info('\n\n===== END TEST CASE: %s =====\n', test)
+
+  def addError(self, test, err):
+    unittest.TestResult.addError(self, test, err)
+    self._LogFailureInfo(err)
+
+  def addFailure(self, test, err):
+    unittest.TestResult.addFailure(self, test, err)
+    self._LogFailureInfo(err)
+
+  def _LogFailureInfo(self, err):
+    exctype, exc, tb = err
+    detail = ''.join(traceback.format_exception(exctype, exc, tb))
+    logging.error('FAILURE: %s', detail)

integration_tests/framework/utils.py

@@ -0,0 +1,331 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Utils for file transfer tests."""
+
+import hashlib
+import logging
+import os
+import pathlib
+import random
+import shutil
+import string
+import subprocess
+import time
+
+CDC_RSYNC_PATH = None
+USER_HOST = None
+
+SHA1_LEN = 40
+SHA1_BUF_SIZE = 65536
+RANDOM = random.Random()
+
+
+def initialize(cdc_rsync_path, user_host):
+  """Sets global variables."""
+  global CDC_RSYNC_PATH, USER_HOST
+
+  CDC_RSYNC_PATH = cdc_rsync_path
+  USER_HOST = user_host
+
+
+def initialize_random():
+  """Sets random seed."""
+  global RANDOM
+  seed = int(time.time())
+  logging.debug('Use random seed %i', seed)
+  RANDOM.seed(seed)
+
+
+def _remove_carriage_return_lines(text):
+  r"""Removes *\r, keeps only *\r\n lines.
+
+  Args:
+      text (string): Text to remove lines from (usually cdc_rsync output).
+
+  Returns:
+      string: Text with lines removed.
+  """
+
+  # Some lines have \r\r\n, treat them properly.
+  ret = ''
+  for line in text.replace('\r\r', '\r').split('\r\n'):
+    ret += line.split('\r')[-1] + '\r\n'
+  return ret
+
+
+def run_rsync(*args):
+  """Runs cdc_rsync with given args.
+
+  The last positional argument is assumed to be the destination. The user/host
+  prefix [user@]host: is optional. If it does not have one, then it is prefixed
+  by |USER_HOST|:.
+
+  Args:
+      *args (string): cdc_rsync arguments.
+
+  Returns:
+      CompletedProcess: cdc_rsync process info with exit code and stdout/stderr.
+  """
+
+  # Prefix last positional argument with [user@]host: if it doesn't have such
+  # a prefix yet. Note that this won't work in all cases, e.g. if
+  # '--exclude', 'file' is passed. Use '--exclude=file' instead.
+  args_list = list(filter(None, args))
+  for n in range(len(args_list) - 1, 0, -1):
+    if args_list[n][0] != '-' and not ':' in args_list[n]:
+      args_list[n] = USER_HOST + ":" + args_list[n]
+      break
+
+  command = [CDC_RSYNC_PATH, *args_list]
+
+  # Workaround issue with unicode logging.
+  logging.debug(
+      'Executing %s ',
+      ' '.join(command).encode('utf-8').decode('ascii', 'backslashreplace'))
+  res = subprocess.run(command, capture_output=True)
+  # Remove lines ending with \r since those are temp display lines.
+  res.stdout = _remove_carriage_return_lines(res.stdout.decode('ascii'))
+  if res.stdout.strip():
+    logging.debug('\r\n%s', res.stdout)
+  return res
+
+
+def files_count_is(cdc_rsync_res,
+                   missing=0,
+                   missing_dir=0,
+                   changed=0,
+                   matching=0,
+                   matching_dir=0,
+                   extraneous=0,
+                   extraneous_dir=0):
+  r"""Verifies that the output of cdc_rsync indicates the given file counts.
+
+  Args:
+      cdc_rsync_res (CompletedProcess): Completed cdc_rsync process
+      missing (int, optional): Number of missing files. Defaults to 0.
+      missing_dir (int, optional): Number of missing folders. Defaults to 0.
+      changed (int, optional): Number of changed files. Defaults to 0.
+      matching (int, optional): Number of matching files. Defaults to 0.
+      matching_dir (int, optional): Number of matching folders. Defaults to 0.
+      extraneous (int, optional): Number of extraneous files. Defaults to 0.
+      extraneous_dir (int, optional): Number of extraneous folders. \ Defaults
+        to 0.
+
+  Returns:
+      bool: True if all file counts match.
+  """
+  missing_ok = '%i file(s) and %i folder(s) are not present' % (
+      missing, missing_dir) in cdc_rsync_res.stdout
+  changed_ok = '%i file(s) changed' % (changed) in cdc_rsync_res.stdout
+  matching_ok = '%i file(s) and %i folder(s) match' % (
+      matching, matching_dir) in cdc_rsync_res.stdout or """%i file(s) and %i \
+folder(s) have matching modified time and size""" % (
+          matching, matching_dir) in cdc_rsync_res.stdout
+  extraneous_ok = """%i file(s) and %i folder(s) on the instance do not exist \
+on this machine""" % (extraneous, extraneous_dir) in cdc_rsync_res.stdout
+  return missing_ok and changed_ok and matching_ok and extraneous_ok
+
+
+def sha1sum_local(filepath):
+  """Computes the sha1 hash of a local file.
+
+  Args:
+      filepath (string): Path of the local (Windows) file
+
+  Returns:
+      string: sha1 hash
+  """
+  sha1 = hashlib.sha1()
+  with open(filepath, 'rb') as f:
+    while True:
+      data = f.read(SHA1_BUF_SIZE)
+      if not data:
+        break
+      sha1.update(data)
+  return sha1.hexdigest()
+
+
+def sha1sum_remote(filepath):
+  """Computes the sha1 hash of a remote file.
+
+  Args:
+      filepath (string): Path of the remote (Linux) file
+
+  Returns:
+      string: sha1 hash
+  """
+  return get_ssh_command_output('sha1sum %s' % filepath)[0:SHA1_LEN]
+
+
+def sha1_matches(local_path, remote_path):
+  """Compares the sha1 hashes of a local and a remote file.
+
+  Args:
+      local_path (string): Path of the local (Windows) file
+      remote_path (string): Path of the remote (Linux) file
+
+  Returns:
+      bool: True if the sha1 hashes match
+  """
+
+  sha1_local = sha1sum_local(local_path)
+  sha1_remote = sha1sum_remote(remote_path)
+  return sha1_local == sha1_remote
+
+
+def create_test_file(local_path, size, printable_data=True, append=False):
+  """Creates a test file with random text of given size.
+
+  Args:
+      local_path (string): Local path of the file to create.
+      size (integer): Size of the file to create (bytes).
+      printable_data (bool, optional): If the data should be printable. Writing
+          a file with printable data is slower, for 1GB of data this takes ~5
+          minutes, in comparison to ~2 seconds for non printable data. Defaults
+          to True.
+      append (bool, optional): If append mode should be used. Defaults to False.
+  """
+  pathlib.Path(os.path.dirname(local_path)).mkdir(parents=True, exist_ok=True)
+
+  mode = None
+  random_bytes = None
+  if printable_data:
+    mode = 'at' if append else 'wt'
+    random_bytes = ''.join(
+        RANDOM.choices(string.ascii_uppercase + string.digits, k=size))
+  else:
+    mode = 'ab' if append else 'wb'
+    random_bytes = os.urandom(size)
+
+  with open(local_path, mode) as f:
+    if size > 0:
+      f.write(random_bytes)
+
+
+def remove_test_file(local_path):
+  """Deletes a test file.
+
+  Args:
+      local_path (string): Local path of the file to delete.
+  """
+  os.remove(local_path)
+
+
+def create_test_directory(local_path):
+  """Creates a directory.
+
+  Args:
+      local_path (string): Local path of the directory to create.
+  """
+  pathlib.Path(os.path.dirname(local_path)).mkdir(parents=True, exist_ok=True)
+
+
+def remove_test_directory(local_path):
+  """Removes a directory with its content.
+
+  Args:
+      local_path (string): Local path of the directory to remove.
+  """
+  shutil.rmtree(pathlib.Path(os.path.dirname(local_path)), ignore_errors=True)
+
+
+def does_directory_exist_remotely(path):
+  """Checks if a directory exists on the remote instance.
+
+  Args:
+      path (string): Path of the remote (Linux) directory
+
+  Returns:
+      bool: True if a directory exists.
+  """
+  return 'yes' in get_ssh_command_output('test -d %s && echo "yes"' % path)
+
+
+def does_file_exist_remotely(path):
+  """Checks if a file exists on the remote instance.
+
+  Args:
+      path (string): Path of the remote (Linux) file
+
+  Returns:
+      bool: True if a file exists.
+  """
+  return 'yes' in get_ssh_command_output('test -f %s && echo "yes"' % path)
+
+
+def change_modified_time(path):
+  """Changes the modified time of the given file.
+
+  Args:
+      path (string): Path of the local file
+  """
+  stats = os.stat(path)
+  os.utime(path, (stats.st_atime, stats.st_mtime + 1))
+
+
+def get_ssh_command_output(cmd):
+  """Runs an SSH command using the command from the CDC_SSH_COMMAND env var.
+
+  Args:
+      cmd (string): Command that is being run remotely
+
+  Returns:
+      string: The output of the ssh command.
+  """
+  ssh_command = os.environ.get('CDC_SSH_COMMAND') or "ssh"
+  full_ssh_cmd = '%s -tt "%s" -- %s' % (ssh_command, USER_HOST,
+                                        quote_argument(cmd))
+  res = subprocess.run(full_ssh_cmd, capture_output=True)
+  if res.returncode != 0:
+    logging.warning('SSH command %s failed with code %i, stderr: %s', cmd,
+                    res.returncode, res.stderr)
+  return res.stdout.decode('ascii', errors='replace')
+
+
+def quote_argument(argument):
+  # This isn't fully generic, but does the job... It doesn't handle when the
+  # argument already escapes quotes, for instance.
+  return '"' + argument.replace('"', '\\"') + '"'
+
+
+def get_sorted_files(remote_dir, pattern='"*.[t|d]*"'):
+  """Returns a sorted list of files in the remote_dir.
+
+  Args:
+      remote_dir (string): Remote directory.
+      pattern (string, optional): Pattern for matching file names.
+
+  Returns:
+      string: Sorted list of files found in the remote directory.
+  """
+  find_res = get_ssh_command_output('cd %s && find -name %s -print' %
+                                    (remote_dir, pattern))
+
+  found = sorted(
+      filter(lambda item: item and item != '.', find_res.split('\r\n')))
+  return found
+
+
+def write_file(path, content):
+  """Writes a file and creates the parent directory if it does not exist yet.
+
+  Args:
+      path (string): File path to create.
+      content (string): File content.
+  """
+  pathlib.Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True)
+  with open(path, 'wt') as file:
+    file.write(content)