Add Integration tests (canonical#294)

Add integration tests and remove the current CI running example and
cloud-init tests. Since running on real clouds require credentials, but
GH Actions doesn't allow the use of secrets in forks, the cloud tests
have been changed to run post-merge. Late is better than never.

Since it's hard to check post-merge behavior on a PR, I temporary had
the `main_check.yaml` workflow run on PR. Got a successful run of the
post-merge tests at
https://github.com/canonical/pycloudlib/actions/runs/4296379520/jobs/7488021663
, so it has now been changed to run post-merge.

Author: TheRealFalcon

.github/workflows/ci.yaml

@@ -2,8 +2,12 @@ name: Pycloudlib CI
 
 on: [pull_request]
 
+concurrency:
+  group: "ci-${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
 jobs:
-  tox:
+  tox-defaults:
     runs-on: ubuntu-20.04
     steps:
       - name: Install dependencies
@@ -12,3 +16,26 @@ jobs:
         uses: actions/checkout@v3
       - name: Run tox
         run: tox
+  integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -q
+          sudo apt-get install -qqy distro-info tox
+          sudo snap install lxd
+      - name: Initialize LXD
+        run: |
+          ssh-keygen -P "" -q -f ~/.ssh/id_rsa
+          mkdir -p ~/.config
+          echo "[lxd]" > ~/.config/pycloudlib.toml
+          sudo adduser $USER lxd
+          # Jammy GH Action runners have docker installed, which edits iptables
+          # in a way that is incompatible with lxd.
+          # https://linuxcontainers.org/lxd/docs/master/howto/network_bridge_firewalld/#prevent-issues-with-lxd-and-docker
+          sudo iptables -I DOCKER-USER -j ACCEPT
+          sudo lxd init --auto
+      - name: Git checkout
+        uses: actions/checkout@v3
+      - name: Run CI integration tests
+        run: sg lxd -c 'tox -e integration-tests-ci'

.github/workflows/main_check.yaml

@@ -0,0 +1,29 @@
+name: Pycloudlib Post Merge Check
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  post-merge-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -q
+          sudo apt-get install -qqy distro-info tox
+          sudo apt-get remove --yes --purge azure-cli
+      - name: Initialize Pycloudlib
+        env:
+          GCE_CREDENTIALS_JSON: ${{ secrets.GCE_CREDENTIALS_JSON }}
+          PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }}
+        run: |
+          ssh-keygen -P "" -q -f ~/.ssh/cloudinit_id_rsa
+          mkdir -p ~/.config
+          echo "$GCE_CREDENTIALS_JSON" > ~/.config/gce_credentials
+          echo "$PYCLOUDLIB_TOML" > ~/.config/pycloudlib.toml
+      - name: Git checkout
+        uses: actions/checkout@v3
+      - name: Run CI integration tests
+        run: tox -e integration-tests-main-check

.travis.yml

@@ -1 +1 @@
-1!2.0.0
+1!2.0.1

VERSION

@@ -55,14 +55,10 @@ def __init__(
         user = getpass.getuser()
         self.key_pair = KeyPair(
             public_key_path=os.path.expandvars(
-                os.path.expanduser(
-                    self.config.get(
-                        "public_key_path", f"~{user}/.ssh/id_rsa.pub"
-                    )
-                )
+                self.config.get("public_key_path", f"~{user}/.ssh/id_rsa.pub")
             ),
             private_key_path=os.path.expandvars(
-                os.path.expanduser(self.config.get("private_key_path", ""))
+                self.config.get("private_key_path", "")
             ),
             name=self.config.get("key_name", user),
         )

pycloudlib/cloud.py

@@ -199,7 +199,12 @@ def clean(self):
 
         This will clean out specifically the cloud-init files and system logs.
         """
-        self.execute("sudo cloud-init clean --logs")
+        result = self.execute("sudo cloud-init clean --logs --machine-id")
+        if result.failed:
+            # --machine-id likely isn't supported on this version.
+            # Manually reset machine-id even though this is less portable
+            self.execute("sudo cloud-init clean --logs")
+            self.execute("sudo echo 'uninitialized' > /etc/machine-id")
         self.execute("sudo rm -rf /var/log/syslog")
 
     def _run_command(self, command, stdin):

pycloudlib/instance.py

@@ -1,6 +1,8 @@
 # This file is part of pycloudlib. See LICENSE file for license information.
 """Base Key Class."""
 
+import os
+
 
 class KeyPair:
     """Key Class."""
@@ -24,6 +26,9 @@ def __init__(self, public_key_path, private_key_path=None, name=None):
         else:
             self.private_key_path = self.public_key_path.replace(".pub", "")
 
+        self.public_key_path = os.path.expanduser(self.public_key_path)
+        self.private_key_path = os.path.expanduser(self.private_key_path)
+
     def __str__(self):
         """Create string representation of class."""
         return "KeyPair({}, {}, name={})".format(

pycloudlib/key.py

@@ -2,6 +2,8 @@
 # This file is part of pycloudlib. See LICENSE file for license information.
 """OCI instance."""
 
+from time import sleep
+
 import oci
 
 from pycloudlib.errors import PycloudlibError
@@ -96,7 +98,23 @@ def delete(self, wait=True):
 
     def _do_restart(self, **kwargs):
         """Restart the instance."""
-        self.compute_client.instance_action(self.instance_data.id, "RESET")
+        last_exception = None
+        for _ in range(30):
+            try:
+                self.compute_client.instance_action(
+                    self.instance_data.id, "RESET"
+                )
+                return
+            except oci.exceptions.ServiceError as e:
+                last_exception = e
+                if last_exception.status != 409:
+                    raise
+                self._log.debug(
+                    "Received 409 attempting to RESET instance. Retrying"
+                )
+                sleep(0.5)
+        if last_exception:
+            raise last_exception
 
     def shutdown(self, wait=True, **kwargs):
         """Shutdown the instance.

pycloudlib/oci/instance.py

@@ -0,0 +1,146 @@
+import ipaddress
+from contextlib import contextmanager, suppress
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Generator
+
+import pytest
+
+import pycloudlib
+from pycloudlib.cloud import BaseCloud
+from pycloudlib.instance import BaseInstance
+from pycloudlib.util import LTS_RELEASES
+
+cloud_config = """\
+#cloud-config
+runcmd:
+  - echo 'hello' >> /var/tmp/example.txt
+"""
+
+
+@pytest.fixture
+def cloud(request):
+    cloud_instance: BaseCloud = request.param(
+        tag="pycl-test",
+        timestamp_suffix=True,
+    )
+
+    if isinstance(cloud_instance, pycloudlib.EC2):
+        cloud_instance.upload_key(
+            public_key_path=cloud_instance.config["public_key_path"],
+            private_key_path=cloud_instance.config["private_key_path"],
+            name=cloud_instance.tag,
+        )
+
+    yield cloud_instance
+
+    if isinstance(cloud_instance, pycloudlib.EC2):
+        cloud_instance.delete_key(name=cloud_instance.tag)
+
+
+@contextmanager
+def launch_instance(
+    cloud_instance: BaseCloud, **kwargs
+) -> Generator[BaseInstance, None, None]:
+    instance = cloud_instance.launch(**kwargs)
+    try:
+        yield instance
+    finally:
+        instance.delete()
+
+
+def assert_example_output(instance: BaseInstance):
+    example_output = instance.execute("cat /var/tmp/example.txt").stdout
+    assert example_output == "hello"
+
+
+def exercise_push_pull(instance: BaseInstance):
+    with TemporaryDirectory() as tmpdir:
+        push_path = Path(tmpdir).joinpath("pushed")
+        push_path.write_text("pushed", encoding="utf-8")
+        instance.push_file(str(push_path), "/var/tmp/pushed")
+        assert (
+            "pushed" == instance.execute("cat /var/tmp/pushed").stdout.strip()
+        )
+
+        instance.execute("echo 'pulled' > /var/tmp/pulled")
+        pull_path = Path(tmpdir).joinpath("pulled")
+        instance.pull_file("/var/tmp/pulled", str(pull_path))
+        assert pull_path.read_text(encoding="utf-8").strip() == "pulled"
+
+
+def exercise_instance(instance: BaseInstance):
+    assert instance.name is not None
+    assert ipaddress.ip_address(instance.ip)
+
+    with suppress(NotImplementedError):
+        ip_address = instance.add_network_interface()
+        try:
+            assert ipaddress.ip_address(ip_address)
+        finally:
+            instance.remove_network_interface(ip_address)
+
+    exercise_push_pull(instance)
+
+    assert_example_output(instance)
+    boot_id = instance.get_boot_id()
+    instance.restart()
+    assert boot_id != instance.get_boot_id()
+
+    assert_example_output(instance)
+    instance.shutdown()
+
+    instance.start()
+    assert_example_output(instance)
+
+
+@pytest.mark.parametrize(
+    "cloud",
+    [
+        pytest.param(
+            pycloudlib.Azure, id="azure", marks=pytest.mark.main_check
+        ),
+        pytest.param(pycloudlib.EC2, id="ec2", marks=pytest.mark.main_check),
+        pytest.param(pycloudlib.GCE, id="gce", marks=pytest.mark.main_check),
+        pytest.param(pycloudlib.IBM, id="ibm"),
+        pytest.param(
+            pycloudlib.LXDContainer, id="lxd_container", marks=pytest.mark.ci
+        ),
+        pytest.param(pycloudlib.LXDVirtualMachine, id="lxd_vm"),
+        pytest.param(pycloudlib.OCI, id="oci"),
+        # For openstack we first need a reliable way of obtaining the
+        # image id
+        # pytest.param(pycloudlib.Openstack, id="openstack"),
+    ],
+    indirect=True,
+)
+def test_public_api(cloud: BaseCloud):
+    """Shallow test of (most) public functions in the base API."""
+    latest_lts = LTS_RELEASES[-1]
+    print(f"Using Ubuntu {latest_lts} release")
+    try:
+        image_id = cloud.released_image(release=latest_lts)
+    except NotImplementedError:
+        image_id = cloud.daily_image(release=latest_lts)
+    with suppress(NotImplementedError):
+        # Not sure there's a great way to test this other than not raising
+        cloud.image_serial(image_id)
+
+    with launch_instance(
+        cloud, image_id=image_id, user_data=cloud_config, wait=False
+    ) as instance:
+        instance.wait()
+        exercise_instance(instance)
+
+        instance.clean()
+        result = instance.execute("sudo rm /var/tmp/example.txt")
+        snapshot_id = cloud.snapshot(instance)
+
+    try:
+        with launch_instance(
+            cloud, image_id=snapshot_id, user_data=cloud_config, wait=False
+        ) as instance_from_snapshot:
+            instance_from_snapshot.wait()
+            exercise_instance(instance_from_snapshot)
+    finally:
+        cloud.delete_image(snapshot_id)

tests/__init__.py

@@ -84,7 +84,32 @@ envdir = {[tip]envdir}
 deps = {[tip]deps}
 commands = {envpython} -m flake8 pycloudlib examples setup.py
 
+[testenv:integration-tests]
+commands = {envpython} -m pytest --log-cli-level=INFO -svv {posargs:tests/integration_tests}
+deps =
+    -rrequirements.txt
+    -rtest-requirements.txt
+
+[testenv:integration-tests-ci]
+commands = {envpython} -m pytest -m ci --log-cli-level=INFO -svv {posargs:tests/integration_tests}
+deps =
+  -rrequirements.txt
+  -rtest-requirements.txt
+  pytest-xdist
+
+[testenv:integration-tests-main-check]
+# Since we can't use GH secrets from a forked PR, run the cloud-based
+# tests after the branch has merged. Better late than never
+commands = {envpython} -m pytest -n 5 -m main_check --log-cli-level=DEBUG -svv {posargs:tests/integration_tests}
+deps = {[testenv:integration-tests-ci]deps}
+
 [flake8]
 # E203: whitespace before ':' ... This goes against pep8 and black formatting
 # W503: line break before binary operator
 ignore = E203, W503
+
+[pytest]
+testpaths = tests/unit_tests
+markers =
+    ci: run test on as part of continous integration
+    main_check: run test after branch has merged to main