|
@@ -1,11 +1,14 @@
|
|
import contextlib
|
|
import contextlib
|
|
import logging
|
|
import logging
|
|
import os
|
|
import os
|
|
|
|
+import pathlib
|
|
|
|
+import platform
|
|
import re
|
|
import re
|
|
import shlex
|
|
import shlex
|
|
import socket
|
|
import socket
|
|
import subprocess
|
|
import subprocess
|
|
import time
|
|
import time
|
|
|
|
+from io import StringIO
|
|
from typing import Iterator, List, Optional
|
|
from typing import Iterator, List, Optional
|
|
|
|
|
|
import backoff
|
|
import backoff
|
|
@@ -20,12 +23,13 @@ from packaging.version import Version
|
|
from requests import Response
|
|
from requests import Response
|
|
from urllib3.util.connection import HAS_IPV6
|
|
from urllib3.util.connection import HAS_IPV6
|
|
|
|
|
|
|
|
+
|
|
logging.basicConfig(level=logging.INFO)
|
|
logging.basicConfig(level=logging.INFO)
|
|
logging.getLogger('backoff').setLevel(logging.INFO)
|
|
logging.getLogger('backoff').setLevel(logging.INFO)
|
|
logging.getLogger('DNS').setLevel(logging.DEBUG)
|
|
logging.getLogger('DNS').setLevel(logging.DEBUG)
|
|
logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARN)
|
|
logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARN)
|
|
|
|
|
|
-CA_ROOT_CERTIFICATE = os.path.join(os.path.dirname(__file__), 'certs/ca-root.crt')
|
|
|
|
|
|
+CA_ROOT_CERTIFICATE = pathlib.Path(__file__).parent.joinpath("certs/ca-root.crt")
|
|
PYTEST_RUNNING_IN_CONTAINER = os.environ.get('PYTEST_RUNNING_IN_CONTAINER') == "1"
|
|
PYTEST_RUNNING_IN_CONTAINER = os.environ.get('PYTEST_RUNNING_IN_CONTAINER') == "1"
|
|
FORCE_CONTAINER_IPV6 = False # ugly global state to consider containers' IPv6 address instead of IPv4
|
|
FORCE_CONTAINER_IPV6 = False # ugly global state to consider containers' IPv6 address instead of IPv4
|
|
|
|
|
|
@@ -71,8 +75,8 @@ class RequestsForDocker:
|
|
"""
|
|
"""
|
|
def __init__(self):
|
|
def __init__(self):
|
|
self.session = requests.Session()
|
|
self.session = requests.Session()
|
|
- if os.path.isfile(CA_ROOT_CERTIFICATE):
|
|
|
|
- self.session.verify = CA_ROOT_CERTIFICATE
|
|
|
|
|
|
+ if CA_ROOT_CERTIFICATE.is_file():
|
|
|
|
+ self.session.verify = CA_ROOT_CERTIFICATE.as_posix()
|
|
|
|
|
|
@staticmethod
|
|
@staticmethod
|
|
def get_nginx_proxy_container() -> Container:
|
|
def get_nginx_proxy_container() -> Container:
|
|
@@ -217,8 +221,8 @@ def nginx_proxy_dns_resolver(domain_name: str) -> Optional[str]:
|
|
|
|
|
|
def docker_container_dns_resolver(domain_name: str) -> Optional[str]:
|
|
def docker_container_dns_resolver(domain_name: str) -> Optional[str]:
|
|
"""
|
|
"""
|
|
- if domain name is of the form "XXX.container.docker" or "anything.XXX.container.docker", return the ip address of the docker container
|
|
|
|
- named XXX.
|
|
|
|
|
|
+ if domain name is of the form "XXX.container.docker" or "anything.XXX.container.docker",
|
|
|
|
+ return the ip address of the docker container named XXX.
|
|
|
|
|
|
:return: IP or None
|
|
:return: IP or None
|
|
"""
|
|
"""
|
|
@@ -248,7 +252,10 @@ def monkey_patch_urllib_dns_resolver():
|
|
"""
|
|
"""
|
|
Alter the behavior of the urllib DNS resolver so that any domain name
|
|
Alter the behavior of the urllib DNS resolver so that any domain name
|
|
containing substring 'nginx-proxy' will resolve to the IP address
|
|
containing substring 'nginx-proxy' will resolve to the IP address
|
|
- of the container created from image 'nginxproxy/nginx-proxy:test'.
|
|
|
|
|
|
+ of the container created from image 'nginxproxy/nginx-proxy:test',
|
|
|
|
+ or to 127.0.0.1 on Darwin.
|
|
|
|
+
|
|
|
|
+ see https://docs.docker.com/desktop/features/networking/#i-want-to-connect-to-a-container-from-the-host
|
|
"""
|
|
"""
|
|
prv_getaddrinfo = socket.getaddrinfo
|
|
prv_getaddrinfo = socket.getaddrinfo
|
|
dns_cache = {}
|
|
dns_cache = {}
|
|
@@ -262,7 +269,12 @@ def monkey_patch_urllib_dns_resolver():
|
|
pytest.skip("This system does not support IPv6")
|
|
pytest.skip("This system does not support IPv6")
|
|
|
|
|
|
# custom DNS resolvers
|
|
# custom DNS resolvers
|
|
- ip = nginx_proxy_dns_resolver(args[0])
|
|
|
|
|
|
+ ip = None
|
|
|
|
+ # Docker Desktop can't route traffic directly to Linux containers.
|
|
|
|
+ if platform.system() == "Darwin":
|
|
|
|
+ ip = "127.0.0.1"
|
|
|
|
+ if ip is None:
|
|
|
|
+ ip = nginx_proxy_dns_resolver(args[0])
|
|
if ip is None:
|
|
if ip is None:
|
|
ip = docker_container_dns_resolver(args[0])
|
|
ip = docker_container_dns_resolver(args[0])
|
|
if ip is not None:
|
|
if ip is not None:
|
|
@@ -298,20 +310,40 @@ def get_nginx_conf_from_container(container: Container) -> bytes:
|
|
return conffile.read()
|
|
return conffile.read()
|
|
|
|
|
|
|
|
|
|
-def docker_compose_up(compose_file: str):
|
|
|
|
- logging.info(f'{DOCKER_COMPOSE} -f {compose_file} up -d')
|
|
|
|
|
|
+def __prepare_and_execute_compose_cmd(compose_files: List[str], project_name: str, cmd: str):
|
|
|
|
+ """
|
|
|
|
+ Prepare and execute the Docker Compose command with the provided compose files and project name.
|
|
|
|
+ """
|
|
|
|
+ compose_cmd = StringIO()
|
|
|
|
+ compose_cmd.write(DOCKER_COMPOSE)
|
|
|
|
+ compose_cmd.write(f" --project-name {project_name}")
|
|
|
|
+ for compose_file in compose_files:
|
|
|
|
+ compose_cmd.write(f" --file {compose_file}")
|
|
|
|
+ compose_cmd.write(f" {cmd}")
|
|
|
|
+
|
|
|
|
+ logging.info(compose_cmd.getvalue())
|
|
try:
|
|
try:
|
|
- subprocess.check_output(shlex.split(f'{DOCKER_COMPOSE} -f {compose_file} up -d'), stderr=subprocess.STDOUT)
|
|
|
|
|
|
+ subprocess.check_output(shlex.split(compose_cmd.getvalue()), stderr=subprocess.STDOUT)
|
|
except subprocess.CalledProcessError as e:
|
|
except subprocess.CalledProcessError as e:
|
|
- pytest.fail(f"Error while running '{DOCKER_COMPOSE} -f {compose_file} up -d':\n{e.output}", pytrace=False)
|
|
|
|
|
|
+ pytest.fail(f"Error while running '{compose_cmd.getvalue()}':\n{e.output}", pytrace=False)
|
|
|
|
|
|
|
|
|
|
-def docker_compose_down(compose_file: str):
|
|
|
|
- logging.info(f'{DOCKER_COMPOSE} -f {compose_file} down -v')
|
|
|
|
- try:
|
|
|
|
- subprocess.check_output(shlex.split(f'{DOCKER_COMPOSE} -f {compose_file} down -v'), stderr=subprocess.STDOUT)
|
|
|
|
- except subprocess.CalledProcessError as e:
|
|
|
|
- pytest.fail(f"Error while running '{DOCKER_COMPOSE} -f {compose_file} down -v':\n{e.output}", pytrace=False)
|
|
|
|
|
|
+def docker_compose_up(compose_files: List[str], project_name: str):
|
|
|
|
+ """
|
|
|
|
+ Execute compose up --detach with the provided compose files and project name.
|
|
|
|
+ """
|
|
|
|
+ if compose_files is None or len(compose_files) == 0:
|
|
|
|
+ pytest.fail(f"No compose file passed to docker_compose_up", pytrace=False)
|
|
|
|
+ __prepare_and_execute_compose_cmd(compose_files, project_name, cmd="up --detach")
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def docker_compose_down(compose_files: List[str], project_name: str):
|
|
|
|
+ """
|
|
|
|
+ Execute compose down --volumes with the provided compose files and project name.
|
|
|
|
+ """
|
|
|
|
+ if compose_files is None or len(compose_files) == 0:
|
|
|
|
+ pytest.fail(f"No compose file passed to docker_compose_up", pytrace=False)
|
|
|
|
+ __prepare_and_execute_compose_cmd(compose_files, project_name, cmd="down --volumes")
|
|
|
|
|
|
|
|
|
|
def wait_for_nginxproxy_to_be_ready():
|
|
def wait_for_nginxproxy_to_be_ready():
|
|
@@ -330,35 +362,47 @@ def wait_for_nginxproxy_to_be_ready():
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
@pytest.fixture
|
|
-def docker_compose_file(request: FixtureRequest) -> Iterator[Optional[str]]:
|
|
|
|
- """Fixture naming the docker compose file to consider.
|
|
|
|
|
|
+def docker_compose_files(request: FixtureRequest) -> List[str]:
|
|
|
|
+ """Fixture returning the docker compose files to consider:
|
|
|
|
+
|
|
|
|
+ If a YAML file exists with the same name as the test module (with the `.py` extension
|
|
|
|
+ replaced with `.base.yml`, ie `test_foo.py`-> `test_foo.base.yml`) and in the same
|
|
|
|
+ directory as the test module, use only that file.
|
|
|
|
|
|
- If a YAML file exists with the same name as the test module (with the `.py` extension replaced
|
|
|
|
- with `.yml` or `.yaml`), use that. Otherwise, use `docker-compose.yml` in the same directory
|
|
|
|
- as the test module.
|
|
|
|
|
|
+ Otherwise, merge the following files in this order:
|
|
|
|
+
|
|
|
|
+ - the `compose.base.yml` file in the parent `test` directory.
|
|
|
|
+ - if present in the same directory as the test module, the `compose.base.override.yml` file.
|
|
|
|
+ - the YAML file named after the current test module (ie `test_foo.py`-> `test_foo.yml`)
|
|
|
|
|
|
Tests can override this fixture to specify a custom location.
|
|
Tests can override this fixture to specify a custom location.
|
|
"""
|
|
"""
|
|
- test_module_dir = os.path.dirname(request.module.__file__)
|
|
|
|
- yml_file = os.path.join(test_module_dir, f"{request.module.__name__}.yml")
|
|
|
|
- yaml_file = os.path.join(test_module_dir, f"{request.module.__name__}.yaml")
|
|
|
|
- default_file = os.path.join(test_module_dir, 'docker-compose.yml')
|
|
|
|
|
|
+ compose_files: List[str] = []
|
|
|
|
+ test_module_path = pathlib.Path(request.module.__file__).parent
|
|
|
|
|
|
- docker_compose_file = None
|
|
|
|
|
|
+ module_base_file = test_module_path.joinpath(f"{request.module.__name__}.base.yml")
|
|
|
|
+ if module_base_file.is_file():
|
|
|
|
+ return [module_base_file.as_posix()]
|
|
|
|
|
|
- if os.path.isfile(yml_file):
|
|
|
|
- docker_compose_file = yml_file
|
|
|
|
- elif os.path.isfile(yaml_file):
|
|
|
|
- docker_compose_file = yaml_file
|
|
|
|
- elif os.path.isfile(default_file):
|
|
|
|
- docker_compose_file = default_file
|
|
|
|
|
|
+ global_base_file = test_module_path.parent.joinpath("compose.base.yml")
|
|
|
|
+ if global_base_file.is_file():
|
|
|
|
+ compose_files.append(global_base_file.as_posix())
|
|
|
|
|
|
- if docker_compose_file is None:
|
|
|
|
- logging.error("Could not find any docker compose file named either '{0}.yml', '{0}.yaml' or 'docker-compose.yml'".format(request.module.__name__))
|
|
|
|
- else:
|
|
|
|
- logging.debug(f"using docker compose file {docker_compose_file}")
|
|
|
|
|
|
+ module_base_override_file = test_module_path.joinpath("compose.base.override.yml")
|
|
|
|
+ if module_base_override_file.is_file():
|
|
|
|
+ compose_files.append(module_base_override_file.as_posix())
|
|
|
|
+
|
|
|
|
+ module_compose_file = test_module_path.joinpath(f"{request.module.__name__}.yml")
|
|
|
|
+ if module_compose_file.is_file():
|
|
|
|
+ compose_files.append(module_compose_file.as_posix())
|
|
|
|
|
|
- yield docker_compose_file
|
|
|
|
|
|
+ if not module_base_file.is_file() and not module_compose_file.is_file():
|
|
|
|
+ logging.error(
|
|
|
|
+ f"Could not find any docker compose file named '{module_base_file.name}' or '{module_compose_file.name}'"
|
|
|
|
+ )
|
|
|
|
+
|
|
|
|
+ logging.debug(f"using docker compose files {compose_files}")
|
|
|
|
+ return compose_files
|
|
|
|
|
|
|
|
|
|
def connect_to_network(network: Network) -> Optional[Network]:
|
|
def connect_to_network(network: Network) -> Optional[Network]:
|
|
@@ -428,30 +472,33 @@ def connect_to_all_networks() -> List[Network]:
|
|
class DockerComposer(contextlib.AbstractContextManager):
|
|
class DockerComposer(contextlib.AbstractContextManager):
|
|
def __init__(self):
|
|
def __init__(self):
|
|
self._networks = None
|
|
self._networks = None
|
|
- self._docker_compose_file = None
|
|
|
|
|
|
+ self._docker_compose_files = None
|
|
|
|
+ self._project_name = None
|
|
|
|
|
|
def __exit__(self, *exc_info):
|
|
def __exit__(self, *exc_info):
|
|
self._down()
|
|
self._down()
|
|
|
|
|
|
def _down(self):
|
|
def _down(self):
|
|
- if self._docker_compose_file is None:
|
|
|
|
|
|
+ if self._docker_compose_files is None:
|
|
return
|
|
return
|
|
for network in self._networks:
|
|
for network in self._networks:
|
|
disconnect_from_network(network)
|
|
disconnect_from_network(network)
|
|
- docker_compose_down(self._docker_compose_file)
|
|
|
|
|
|
+ docker_compose_down(self._docker_compose_files, self._project_name)
|
|
self._docker_compose_file = None
|
|
self._docker_compose_file = None
|
|
|
|
+ self._project_name = None
|
|
|
|
|
|
- def compose(self, docker_compose_file: Optional[str]):
|
|
|
|
- if docker_compose_file == self._docker_compose_file:
|
|
|
|
|
|
+ def compose(self, docker_compose_files: List[str], project_name: str):
|
|
|
|
+ if docker_compose_files == self._docker_compose_files and project_name == self._project_name:
|
|
return
|
|
return
|
|
self._down()
|
|
self._down()
|
|
- if docker_compose_file is None:
|
|
|
|
|
|
+ if docker_compose_files is None or project_name is None:
|
|
return
|
|
return
|
|
- docker_compose_up(docker_compose_file)
|
|
|
|
|
|
+ docker_compose_up(docker_compose_files, project_name)
|
|
self._networks = connect_to_all_networks()
|
|
self._networks = connect_to_all_networks()
|
|
wait_for_nginxproxy_to_be_ready()
|
|
wait_for_nginxproxy_to_be_ready()
|
|
time.sleep(3) # give time to containers to be ready
|
|
time.sleep(3) # give time to containers to be ready
|
|
- self._docker_compose_file = docker_compose_file
|
|
|
|
|
|
+ self._docker_compose_files = docker_compose_files
|
|
|
|
+ self._project_name = project_name
|
|
|
|
|
|
|
|
|
|
###############################################################################
|
|
###############################################################################
|
|
@@ -462,14 +509,14 @@ class DockerComposer(contextlib.AbstractContextManager):
|
|
|
|
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
@pytest.fixture(scope="module")
|
|
-def docker_composer() -> Iterator[DockerComposer]:
|
|
|
|
|
|
+def docker_composer() -> Iterator[DockerComposer]:
|
|
with DockerComposer() as d:
|
|
with DockerComposer() as d:
|
|
yield d
|
|
yield d
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
@pytest.fixture
|
|
-def ca_root_certificate() -> Iterator[str]:
|
|
|
|
- yield CA_ROOT_CERTIFICATE
|
|
|
|
|
|
+def ca_root_certificate() -> str:
|
|
|
|
+ return CA_ROOT_CERTIFICATE.as_posix()
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
@pytest.fixture
|
|
@@ -480,16 +527,29 @@ def monkey_patched_dns():
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
@pytest.fixture
|
|
-def docker_compose(monkey_patched_dns, docker_composer, docker_compose_file) -> Iterator[DockerClient]:
|
|
|
|
- """Ensures containers described in a docker compose file are started.
|
|
|
|
|
|
+def docker_compose(
|
|
|
|
+ request: FixtureRequest,
|
|
|
|
+ monkeypatch,
|
|
|
|
+ monkey_patched_dns,
|
|
|
|
+ docker_composer,
|
|
|
|
+ docker_compose_files
|
|
|
|
+) -> Iterator[DockerClient]:
|
|
|
|
+ """
|
|
|
|
+ Ensures containers necessary for the test module are started in a compose project,
|
|
|
|
+ and set the environment variable `PYTEST_MODULE_PATH` to the test module's parent folder.
|
|
|
|
|
|
- A custom docker compose file name can be specified by overriding the `docker_compose_file`
|
|
|
|
- fixture.
|
|
|
|
|
|
+ A list of custom docker compose files path can be specified by overriding
|
|
|
|
+ the `docker_compose_file` fixture.
|
|
|
|
|
|
- Also, in the case where pytest is running from a docker container, this fixture makes sure
|
|
|
|
- our container will be attached to all the docker networks.
|
|
|
|
|
|
+ Also, in the case where pytest is running from a docker container, this fixture
|
|
|
|
+ makes sure our container will be attached to all the docker networks.
|
|
"""
|
|
"""
|
|
- docker_composer.compose(docker_compose_file)
|
|
|
|
|
|
+ pytest_module_path = pathlib.Path(request.module.__file__).parent
|
|
|
|
+ monkeypatch.setenv("PYTEST_MODULE_PATH", pytest_module_path.as_posix())
|
|
|
|
+
|
|
|
|
+ project_name = request.module.__name__
|
|
|
|
+ docker_composer.compose(docker_compose_files, project_name)
|
|
|
|
+
|
|
yield docker_client
|
|
yield docker_client
|
|
|
|
|
|
|
|
|
|
@@ -511,11 +571,11 @@ def nginxproxy() -> Iterator[RequestsForDocker]:
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
@pytest.fixture
|
|
-def acme_challenge_path() -> Iterator[str]:
|
|
|
|
|
|
+def acme_challenge_path() -> str:
|
|
"""
|
|
"""
|
|
Provides fake Let's Encrypt ACME challenge path used in certain tests
|
|
Provides fake Let's Encrypt ACME challenge path used in certain tests
|
|
"""
|
|
"""
|
|
- yield ".well-known/acme-challenge/test-filename"
|
|
|
|
|
|
+ return ".well-known/acme-challenge/test-filename"
|
|
|
|
|
|
###############################################################################
|
|
###############################################################################
|
|
#
|
|
#
|