Improve testing against distro and podman-next

1. Split upstream into distro/pnext test runs

A new CI run targets tests that are exclusive for podman-next.
/upstream plans are now split into /distro and /pnext to be explicit
against which podman version is used in the tests. Also, python version
used are now called base_python and all_python, to be more self
explainatory.

2. Add pytest marker `pnext` to filter tests

Tests that should run against podman-next are now skipped by default in
tox runs. They can be enabled with --pnext and filtered in using
`-m pnext`. These two options should be used together since `pnext` and
`non-pnext` tests are thought to be incompatible.

3. Split `test_container_mounts` to test a breaking change and update
   docs

The scenario `test_container_mounts` was split to demonstrates the
usage of the marker against an upstream breaking change.
CONTRIBUTING.md now reflects the new feature and explains how to use tox
locally to leverage on this feature.

Fixes: containers#547

Signed-off-by: Nicola Sella <[email protected]>

Author: inknos

.packit.yaml

@@ -102,12 +102,13 @@ jobs:
     dist_git_branches:
       - fedora-branched # rawhide updates are created automatically
 
+
   # Test linting on the codebase
   # This test might break based on the OS and lint used, so we follow fedora-latest as a reference
   - job: tests
     trigger: pull_request
-    identifier: upstream-sanity
-    tmt_plan: /upstream/sanity
+    identifier: distro-sanity
+    tmt_plan: /distro/sanity
     packages: [python-podman-fedora]
     targets:
       - fedora-latest-stable
@@ -118,7 +119,7 @@ jobs:
   - job: tests
     trigger: pull_request
     identifier: unittest-coverage
-    tmt_plan: /upstream/unittest_coverage
+    tmt_plan: /distro/unittest_coverage
     packages: [python-podman-fedora]
     targets:
       - fedora-latest-stable
@@ -130,30 +131,38 @@ jobs:
   # run all tests for all python versions on all fedoras
   - job: tests
     trigger: pull_request
-    identifier: upstream-all-fedora
-    tmt_plan: /upstream/all
+    identifier: distro-fedora-all
+    tmt_plan: /distro/all_python
     packages: [python-podman-fedora]
     targets:
       - fedora-all
+
+  # run tests for the rawhide python version using podman-next packages
+  - job: tests
+    trigger: pull_request
+    identifier: podman-next-fedora-base
+    tmt_plan: /pnext/base_python_pnext
+    packages: [python-podman-fedora]
+    targets:
+      - fedora-rawhide
     tf_extra_params:
       environments:
         - artifacts:
           - type: repository-file
             id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/fedora-$releasever/rhcontainerbot-podman-next-fedora-$releasever.repo
-
   - job: tests
     trigger: pull_request
-    identifier: upstream-base-centos
-    tmt_plan: /upstream/base
+    identifier: distro-centos-base
+    tmt_plan: /distro/base_python
     packages: [python-podman-centos]
     targets:
       - centos-stream-9
       - centos-stream-10
 
   - job: tests
     trigger: pull_request
-    identifier: upstream-base-rhel
-    tmt_plan: /upstream/base
+    identifier: distro-rhel-base
+    tmt_plan: /distro/base_python
     packages: [python-podman-rhel]
     targets:
       - epel-9

CONTRIBUTING.md

@@ -45,6 +45,45 @@ pip install tox
 tox -e coverage
 ```
 
+#### Advanced testing
+
+Always prefer to run `tox` directly, even when you want to run a specific test or scenario.
+Instead of running `pytest` directly, you should run:
+
+```
+tox -e py -- podman/tests/integration/test_container_create.py -k test_container_directory_volume_mount
+```
+
+If you'd like to test against a specific `tox` environment you can do:
+
+```
+tox -e py12 -- podman/tests/integration/test_container_create.py -k test_container_directory_volume_mount
+```
+
+Pass pytest options after `--`.
+
+#### Testing future features
+
+Since `podman-py` follows stable releases of `podman`, tests are thought to be run against
+libpod's versions that are commonly installed in the distributions. Tests can be versioned,
+but preferably they should not. Occasionally, upstream can have diverge and have features that
+are not included in a specific version of libpod, or that will be included eventually.
+To run a test against such changes, you need to have
+[podman-next](https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next) installed.
+Then, you need to mark the test as `@pytest.mark.pnext`. Marked tests willbe excluded from the
+runs, unless you pass `--pnext` as a cli option.
+Preferably, this should be a rare case and it's better to use this marker as a temporary solution,
+with the goal of removing the marker within few PRs.
+
+To run these tests use:
+
+```
+tox -e py -- --pnext -m pnext podman/tests/integration/test_container_create.py -k test_container_mounts_without_rw_as_default
+```
+
+The option `--pnext` **enables** the tests with the `pnext` pytest marker, and `-m pnext` will run
+the marked tests **only**.
+
 ## Submitting changes
 
 - Create a github pull request (PR)

Makefile

@@ -26,6 +26,10 @@ tests: tox
 	# see tox.ini for environment variable settings
 	$(PYTHON) -m tox -e coverage,py39,py310,py311,py312,py313
 
+.PHONY: tests-ci-base-python-podman-next
+tests-ci-base-python-podman-next:
+	$(PYTHON) -m tox -e coverage,py -- --pnext -m -pnext
+
 .PHONY: tests-ci-base-python
 tests-ci-base-python:
 	$(PYTHON) -m tox -e coverage,py

plans/main.fmf

@@ -12,16 +12,6 @@ prepare:
       - python3-pip
       - podman
 
-  - name: enable rhcontainerbot/podman-next update podman
-    when: initiator == packit
-    how: shell
-    script: |
-      COPR_REPO_FILE="/etc/yum.repos.d/*podman-next*.repo"
-      if compgen -G $COPR_REPO_FILE > /dev/null; then
-          sed -i -n '/^priority=/!p;$apriority=1' $COPR_REPO_FILE
-      fi
-      dnf -y upgrade --allowerasing
-
   - name: pip dependencies
     how: shell
     script:
@@ -34,20 +24,46 @@ prepare:
         - cp /root/.ssh/authorized_keys /root/.ssh/authorized_keys%
         - cat /root/.ssh/id_ecdsa.pub >>/root/.ssh/authorized_keys
 
-/upstream:
+# Run tests agains Podman Next builds.
+# These tests should NOT overlap with the ones who run in the distro plan and should only include
+# tests against upcoming features or upstream tests that we need to run for reasons.
+/pnext:
+    prepare+:
+      - name: enable rhcontainerbot/podman-next update podman
+        when: initiator == packit
+        how: shell
+        script: |
+          COPR_REPO_FILE="/etc/yum.repos.d/*podman-next*.repo"
+          if compgen -G $COPR_REPO_FILE > /dev/null; then
+              sed -i -n '/^priority=/!p;$apriority=1' $COPR_REPO_FILE
+          fi
+          dnf -y upgrade --allowerasing
+
+    /base_python:
+        summary: Run Tests Upstream PRs for base Python
+        discover+:
+            filter: tag:pnext
+
+    adjust+:
+        enabled: false
+        when: initiator is not defined or initiator != packit
+
+
+# Run tests against Podman buids installed from the distribution.
+/distro:
     /sanity:
         summary: Run Sanity and Coverage checks on Python Podman
         discover+:
             # we want to change this to tag:stable once all the coverage tests are fixed
             filter: tag:lint
 
-    /base:
-        summary: Run Python Podman Tests on Upstream PRs for base Python
+    /base_python:
+        summary: Run Tests Upstream for base Python
         discover+:
             filter: tag:base
 
-    /all:
-        summary: Run Python Podman Tests on Upstream PRs for all Python versions
+    /all_python:
+        summary: Run Tests Upstream PRs for all Python versions
         prepare+:
           - name: install all python versions
             how: install
@@ -60,6 +76,7 @@ prepare:
         discover+:
             filter: tag:matrix
 
+    # TODO: replace with /coverage and include integration tests coverage
     /unittest_coverage:
         summary: Run Unit test coverage
         discover+:
@@ -69,9 +86,11 @@ prepare:
         enabled: false
         when: initiator is not defined or initiator != packit
 
+# Run tests against downstream Podman. These tests should be the all_python only since the sanity
+# of code is tested in the distro environment
 /downstream:
     /all:
-      summary: Run Python Podman Tests on bodhi / errata and dist-git PRs
+      summary: Run Tests on bodhi / errata and dist-git PRs
       prepare+:
         - name: install all python versions
           how: install

podman/tests/conftest.py

@@ -0,0 +1,21 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--pnext", action="store_true", default=False, help="run tests against podman_next copr"
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "pnext: mark test as run against podman_next")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--pnext"):
+        # --pnext given in cli: run tests marked as pnext
+        return
+    podman_next = pytest.mark.skip(reason="need --pnext option to run")
+    for item in items:
+        if "pnext" in item.keywords:
+            item.add_marker(podman_next)

podman/tests/integration/test_container_create.py

@@ -1,5 +1,6 @@
 import unittest
 
+import pytest
 import re
 
 import podman.tests.integration.base as base
@@ -311,6 +312,45 @@ def test_container_mounts(self):
 
             self.assertEqual(container.attrs.get('State', dict()).get('ExitCode', 256), 0)
 
+    @pytest.mark.pnext
+    # repeat this test against this upstream change
+    # https://github.com/containers/podman/pull/25942
+    def test_container_mounts_without_rw_as_default(self):
+        """Test passing mounts"""
+        with self.subTest("Check bind mount"):
+            mount = {
+                "type": "bind",
+                "source": "/etc/hosts",
+                "target": "/test",
+                "read_only": True,
+                "relabel": "Z",
+            }
+            container = self.client.containers.create(
+                self.alpine_image, command=["cat", "/test"], mounts=[mount]
+            )
+            self.containers.append(container)
+            self.assertIn(
+                f"{mount['source']}:{mount['target']}:ro,Z,rprivate,rbind",
+                container.attrs.get('HostConfig', {}).get('Binds', list()),
+            )
+
+            # check if container can be started and exits with EC == 0
+            container.start()
+            container.wait()
+
+            self.assertEqual(container.attrs.get('State', dict()).get('ExitCode', 256), 0)
+
+        with self.subTest("Check tmpfs mount"):
+            mount = {"type": "tmpfs", "source": "tmpfs", "target": "/test", "size": "456k"}
+            container = self.client.containers.create(
+                self.alpine_image, command=["df", "-h"], mounts=[mount]
+            )
+            self.containers.append(container)
+            self.assertEqual(
+                container.attrs.get('HostConfig', {}).get('Tmpfs', {}).get(mount['target']),
+                f"size={mount['size']},rprivate,nosuid,nodev,tmpcopyup",
+            )
+
     def test_container_devices(self):
         devices = ["/dev/null:/dev/foo", "/dev/zero:/dev/bar"]
         container = self.client.containers.create(

tests/main.fmf

@@ -24,6 +24,12 @@ require:
         test: cd .. && make tests-ci-base-python
         duration: 10m
 
+    /base_python_pnext:
+        tag: [ pnext ]
+        summary: Run all tests on the base python version and podman-next
+        test: cd .. && make tests-ci-base-python-podman-next
+        duration: 5m
+
     /all_python:
         tag: [ matrix ]
         summary: Run all tests for all ptyhon versions available