diff --git a/.github/workflows/alpine-unittests.yml b/.github/workflows/alpine-unittests.yml index fa6355c2..a525415e 100644 --- a/.github/workflows/alpine-unittests.yml +++ b/.github/workflows/alpine-unittests.yml @@ -29,9 +29,7 @@ jobs: fetch-depth: 0 - name: Setup LXD - uses: canonical/setup-lxd@v0.1.1 - with: - channel: latest/candidate + uses: canonical/setup-lxd@v0.1.2 - name: Create alpine container # the current shell doesn't have lxd as one of the groups diff --git a/.github/workflows/check_format.yml b/.github/workflows/check_format.yml index 8f2a3357..6e9b0bbc 100644 --- a/.github/workflows/check_format.yml +++ b/.github/workflows/check_format.yml @@ -104,11 +104,3 @@ jobs: - name: Run ShellCheck run: | shellcheck ./tools/ds-identify - - check-cla-signers: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - - name: Check CLA signers file - run: tools/check-cla-signers diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 1022e7ba..fc72874a 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -38,3 +38,12 @@ jobs: cat unsigned-cla.txt exit 1 fi + + check-cla-signers: + name: Verify that ./tools/.github-cla-signers is in alphabetical order + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Check CLA signers file + run: tools/check-cla-signers diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 438bbfcd..612c8420 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -54,9 +54,7 @@ jobs: path: '${{ runner.temp }}/cloud-init*.deb' retention-days: 3 - name: Setup LXD - uses: canonical/setup-lxd@v0.1.1 - with: - channel: latest/candidate + uses: canonical/setup-lxd@v0.1.2 - name: Verify deb package run: | ls -hal '${{ runner.temp }}' diff --git a/.github/workflows/packaging-tests.yml b/.github/workflows/packaging-tests.yml index ac647f31..a08e5b5d 100644 --- a/.github/workflows/packaging-tests.yml +++ b/.github/workflows/packaging-tests.yml @@ -1,4 +1,8 @@ -name: Packaging +# This test runs against packaging PRs and verifies that +# patches apply and unit tests pass. +# +# TODO: add full build-package / sbuild test +name: "Packaging (downstream branch) - patches apply cleanly unit tests pass" on: pull_request: @@ -13,40 +17,65 @@ defaults: run: shell: sh -ex {0} -env: - RELEASE: focal - jobs: - check-patches: - name: Check patches + patch-conflicts-upstream: runs-on: ubuntu-24.04 + name: Check patches apply cleanly and unit tests pass steps: - - name: Checkout + + - name: Setup - checkout branch uses: actions/checkout@v4 - with: - # Fetch all branches for merging - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - name: Prepare dependencies + + - name: Setup - install dependencies run: | + # This stage is slow, and unnecessecary when no series exists, so + # only run it if necessary to save some cycles. + # Github Actions doesn't appear to have a simple mechanism for + # early exit without failure + if [ ! -f debian/patches/series ]; then + echo "no patches, skipping" + exit 0 + fi sudo DEBIAN_FRONTEND=noninteractive apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get -y install tox quilt - - name: Setup quilt environment - run: | - echo 'QUILT_PATCHES=debian/patches' >> ~/.quiltrc - echo 'QUILT_SERIES=debian/patches/series' >> ~/.quiltrc - - name: Configure git and merge + - name: Setup - Configure quilt run: | - git config user.name "GitHub Actions" - git config user.email "actions@github.com" - git merge origin/main - - name: Quilt patches apply successfully and tests run + # Github Actions doesn't appear to have a simple mechanism for + # early exit without failure + if [ ! -f debian/patches/series ]; then + echo "no patches, skipping" + exit 0 + fi + # The quilt default setting is --fuzz=2, but debian packaging has + # stricter requirements + sudo sed -i 's/QUILT_PUSH_ARGS=.*$/QUILT_PUSH_ARGS="--fuzz=0"/g' /etc/quilt.quiltrc + # Standardize patches to use this format. Sorted patches reduce patch size. + sudo sed -i 's/QUILT_REFRESH_ARGS=.*$/QUILT_REFRESH_ARGS="-p ab --no-timestamps --no-index --sort"/g' /etc/quilt.quiltrc + # quilt defaults to QUILT_PATCHES=patches, but debian uses debian/patches + sudo sed -i 's|.*QUILT_PATCHES=.*$|QUILT_PATCHES=debian/patches|g' /etc/quilt.quiltrc + + - name: Run test - apply patches and run unit tests run: | + # Github Actions doesn't appear to have a simple mechanism for + # early exit without failure if [ ! -f debian/patches/series ]; then echo "no patches, skipping" exit 0 fi quilt push -a tox -e py3 - quilt pop -a + quilt pop -a --refresh + - name: Enforce sorted patches + run: | + # Github Actions doesn't appear to have a simple mechanism for + # early exit without failure + if [ ! -f debian/patches/series ]; then + echo "no patches, skipping" + exit 0 + fi + # check for any changes from the refresh above + if [ -n "$(git diff)" ]; then + # if patches were refreshed then they weren't sorted + exit 1 + fi diff --git a/.github/workflows/packaging-upstream.yml b/.github/workflows/packaging-upstream.yml new file mode 100644 index 00000000..3a7429d1 --- /dev/null +++ b/.github/workflows/packaging-upstream.yml @@ -0,0 +1,69 @@ +# This test runs after merging PRs into upstream to notify maintainers when +# a new PR has caused a patch to not apply against the merged branch. +name: "Packaging (main branch) - patches apply cleanly and unit tests pass (after merging)" + +on: + push: + branches: + - main + +concurrency: + group: 'ci-${{ github.workflow }}-${{ github.ref }}' + cancel-in-progress: true + +defaults: + run: + shell: bash -ex {0} + +jobs: + patch-conflicts-ubuntu: + runs-on: ubuntu-24.04 + name: Check patches + steps: + + - name: Setup - checkout branches + uses: actions/checkout@v4 + with: + # Fetch all history for merging + fetch-depth: 0 + ref: main + + - name: Setup - install dependencies + run: | + sudo DEBIAN_FRONTEND=noninteractive apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get -y install tox quilt + + - name: Setup - configure quilt + run: | + # The quilt default setting is --fuzz=2, but debian packaging has + # stricter requirements. + sudo sed -i 's/QUILT_PUSH_ARGS=.*$/QUILT_PUSH_ARGS="--fuzz=0"/g' /etc/quilt.quiltrc + # quilt defaults to QUILT_PATCHES=patches, but debian uses debian/patches + sudo sed -i 's|.*QUILT_PATCHES=.*$|QUILT_PATCHES=debian/patches|g' /etc/quilt.quiltrc + + - name: Setup - configure git + run: | + git config user.name "Github Actions" + git config user.email "noreply@github.com" + + - name: Run test - apply patches and run unit tests for each series + run: | + # Modify the following line to add / remove ubuntu series + for BRANCH in ubuntu/devel ubuntu/oracular ubuntu/noble ubuntu/jammy ubuntu/focal; do + # merge - this step is not expected to fail + git merge "origin/$BRANCH" + if [ ! -f debian/patches/series ]; then + echo "no patches, skipping $BRANCH" + # undo merge - this step is not expected to fail + git reset --hard origin/main + continue + fi + # did patches apply cleanly? + quilt push -a + # run unit tests + tox -e py3 + # a patch didn't un-apply cleanly if this step fails + quilt pop -a + # undo merge - this step is not expected to fail + git reset --hard origin/main + done diff --git a/.pc/applied-patches b/.pc/applied-patches index 4fb6fca6..8c0008e5 100644 --- a/.pc/applied-patches +++ b/.pc/applied-patches @@ -2,10 +2,4 @@ deprecation-version-boundary.patch no-single-process.patch no-nocloud-network.patch grub-dpkg-support.patch -cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting no-remove-networkd-online.patch -cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995 -cpick-c60771d8-test-pytestify-test_url_helper.py -cpick-8810a2dc-test-Remove-CiTestCase-from-test_url_helper.py -cpick-582f16c1-test-add-OauthUrlHelper-tests -cpick-9311e066-fix-Update-OauthUrlHelper-to-use-readurl-exception_cb diff --git a/.pc/cpick-582f16c1-test-add-OauthUrlHelper-tests/tests/unittests/test_url_helper.py b/.pc/cpick-582f16c1-test-add-OauthUrlHelper-tests/tests/unittests/test_url_helper.py deleted file mode 100644 index 56685177..00000000 --- a/.pc/cpick-582f16c1-test-add-OauthUrlHelper-tests/tests/unittests/test_url_helper.py +++ /dev/null @@ -1,949 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. -# pylint: disable=attribute-defined-outside-init - -import logging -import pathlib -from functools import partial -from threading import Event -from time import process_time -from unittest.mock import ANY, call - -import pytest -import requests -import responses - -from cloudinit import util, version -from cloudinit.url_helper import ( - REDACTED, - UrlError, - UrlResponse, - _handle_error, - dual_stack, - oauth_headers, - read_file_or_url, - readurl, - wait_for_url, -) -from tests.unittests.helpers import mock, skipIf - -try: - import oauthlib - - assert oauthlib # avoid pyflakes error F401: import unused - _missing_oauthlib_dep = False -except ImportError: - _missing_oauthlib_dep = True - - -M_PATH = "cloudinit.url_helper." - - -class TestOAuthHeaders: - def test_oauth_headers_raises_not_implemented_when_oathlib_missing(self): - """oauth_headers raises a NotImplemented error when oauth absent.""" - with mock.patch.dict("sys.modules", {"oauthlib": None}): - with pytest.raises(NotImplementedError) as context_manager: - oauth_headers(1, 2, 3, 4, 5) - assert "oauth support is not available" == str(context_manager.value) - - @skipIf(_missing_oauthlib_dep, "No python-oauthlib dependency") - @mock.patch("oauthlib.oauth1.Client") - def test_oauth_headers_calls_oathlibclient_when_available(self, m_client): - """oauth_headers calls oaut1.hClient.sign with the provided url.""" - - class fakeclient: - def sign(self, url): - # The first and 3rd item of the client.sign tuple are ignored - return ("junk", url, "junk2") - - m_client.return_value = fakeclient() - - return_value = oauth_headers( - "url", - "consumer_key", - "token_key", - "token_secret", - "consumer_secret", - ) - assert "url" == return_value - - -class TestReadFileOrUrl: - def test_read_file_or_url_str_from_file(self, tmp_path: pathlib.Path): - """Test that str(result.contents) on file is text version of contents. - It should not be "b'data'", but just "'data'" """ - tmpf = tmp_path / "myfile1" - data = b"This is my file content\n" - util.write_file(tmpf, data, omode="wb") - result = read_file_or_url(f"file://{tmpf}") - assert result.contents == data - assert str(result) == data.decode("utf-8") - - @responses.activate - def test_read_file_or_url_str_from_url(self): - """Test that str(result.contents) on url is text version of contents. - It should not be "b'data'", but just "'data'" """ - url = "http://hostname/path" - data = b"This is my url content\n" - responses.add(responses.GET, url, data) - result = read_file_or_url(url) - assert result.contents == data - assert str(result) == data.decode("utf-8") - - @responses.activate - def test_read_file_or_url_str_from_url_streamed(self): - """Test that str(result.contents) on url is text version of contents. - It should not be "b'data'", but just "'data'" """ - url = "http://hostname/path" - data = b"This is my url content\n" - responses.add(responses.GET, url, data) - result = read_file_or_url(url, stream=True) - assert isinstance(result, UrlResponse) - assert result.contents == data - assert str(result) == data.decode("utf-8") - - @responses.activate - def test_read_file_or_url_str_from_url_redacting_headers_from_logs( - self, caplog - ): - """Headers are redacted from logs but unredacted in requests.""" - url = "http://hostname/path" - headers = {"sensitive": "sekret", "server": "blah"} - - def _request_callback(request): - for k in headers.keys(): - assert headers[k] == request.headers[k] - return (200, request.headers, "does_not_matter") - - responses.add_callback(responses.GET, url, callback=_request_callback) - - read_file_or_url(url, headers=headers, headers_redact=["sensitive"]) - assert REDACTED in caplog.text - assert "sekret" not in caplog.text - - @responses.activate - def test_read_file_or_url_str_from_url_redacts_noheaders(self, caplog): - """When no headers_redact, header values are in logs and requests.""" - url = "http://hostname/path" - headers = {"sensitive": "sekret", "server": "blah"} - - def _request_callback(request): - for k in headers.keys(): - assert headers[k] == request.headers[k] - return (200, request.headers, "does_not_matter") - - responses.add_callback(responses.GET, url, callback=_request_callback) - - read_file_or_url(url, headers=headers) - assert REDACTED not in caplog.text - assert "sekret" in caplog.text - - def test_wb_read_url_defaults_honored_by_read_file_or_url_callers(self): - """Readurl param defaults used when unspecified by read_file_or_url - - Param defaults tested are as follows: - retries: 0, additional headers None beyond default, method: GET, - data: None, check_status: True and allow_redirects: True - """ - url = "http://hostname/path" - - m_response = mock.MagicMock() - - class FakeSessionRaisesHttpError(requests.Session): - @classmethod - def request(cls, **kwargs): - raise requests.exceptions.RequestException("broke") - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - assert { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": { - "User-Agent": "Cloud-Init/%s" - % (version.version_string()) - }, - "stream": False, - } == kwargs - return m_response - - with mock.patch(M_PATH + "requests.Session") as m_session: - m_session.side_effect = [ - FakeSessionRaisesHttpError(), - FakeSession(), - ] - # assert no retries and check_status == True - with pytest.raises(UrlError) as context_manager: - response = read_file_or_url(url) - assert "broke" == str(context_manager.value) - # assert default headers, method, url and allow_redirects True - # Success on 2nd call with FakeSession - response = read_file_or_url(url) - assert m_response == response._response - - -class TestReadFileOrUrlParameters: - @mock.patch(M_PATH + "readurl") - @pytest.mark.parametrize( - "timeout", [1, 1.2, "1", (1, None), (1, 1), (None, None)] - ) - def test_read_file_or_url_passes_params_to_readurl( - self, m_readurl, timeout - ): - """read_file_or_url passes all params through to readurl.""" - url = "http://hostname/path" - response = "This is my url content\n" - m_readurl.return_value = response - params = { - "url": url, - "timeout": timeout, - "retries": 2, - "headers": {"somehdr": "val"}, - "data": "data", - "sec_between": 1, - "ssl_details": {"cert_file": "/path/cert.pem"}, - "headers_cb": "headers_cb", - "exception_cb": "exception_cb", - "stream": True, - } - - assert response == read_file_or_url(**params) - params.pop("url") # url is passed in as a positional arg - assert m_readurl.call_args_list == [mock.call(url, **params)] - - @pytest.mark.parametrize( - "readurl_timeout,request_timeout", - [ - (-1, 0), - ("-1", 0), - (None, None), - (1, 1.0), - (1.2, 1.2), - ("1", 1.0), - ((1, None), (1, None)), - ((1, 1), (1, 1)), - ((None, None), (None, None)), - ], - ) - def test_readurl_timeout(self, readurl_timeout, request_timeout): - url = "http://hostname/path" - m_response = mock.MagicMock() - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - expected_kwargs = { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": { - "User-Agent": "Cloud-Init/%s" - % (version.version_string()) - }, - "timeout": request_timeout, - "stream": False, - } - if request_timeout is None: - expected_kwargs.pop("timeout") - - assert kwargs == expected_kwargs - return m_response - - with mock.patch( - M_PATH + "requests.Session", side_effect=[FakeSession()] - ): - response = read_file_or_url(url, timeout=readurl_timeout) - - assert response._response == m_response - - -def assert_time(func, max_time=1): - """Assert function time is bounded by a max (default=1s) - - The following async tests should canceled in under 1ms and have stagger - delay and max_ - It is possible that this could yield a false positive, but this should - basically never happen (esp under normal system load). - """ - start = process_time() - try: - out = func() - finally: - diff = process_time() - start - assert diff < max_time - return out - - -class TestReadUrl: - @pytest.mark.parametrize("headers", [{}, {"Metadata": "true"}]) - def test_headers(self, headers): - url = "http://hostname/path" - m_response = mock.MagicMock() - - expected_headers = headers.copy() - expected_headers["User-Agent"] = "Cloud-Init/%s" % ( - version.version_string() - ) - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - expected_kwargs = { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": expected_headers, - "stream": False, - } - - assert kwargs == expected_kwargs - return m_response - - with mock.patch( - M_PATH + "requests.Session", side_effect=[FakeSession()] - ): - response = readurl(url, headers=headers) - - assert response._response == m_response - - @pytest.mark.parametrize("headers", [{}, {"Metadata": "true"}]) - def test_headers_cb(self, headers): - url = "http://hostname/path" - m_response = mock.MagicMock() - - expected_headers = headers.copy() - expected_headers["User-Agent"] = "Cloud-Init/%s" % ( - version.version_string() - ) - headers_cb = lambda _: headers - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - expected_kwargs = { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": expected_headers, - "stream": False, - } - - assert kwargs == expected_kwargs - return m_response - - with mock.patch( - M_PATH + "requests.Session", side_effect=[FakeSession()] - ): - response = readurl(url, headers_cb=headers_cb) - - assert response._response == m_response - - def test_error_no_cb(self, mocker): - response = requests.Response() - response.status_code = 500 - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.return_value = response - - with pytest.raises(UrlError) as e: - readurl("http://some/path") - assert e.value.code == 500 - - def test_error_cb_true(self, mocker): - mocker.patch("time.sleep") - - bad_response = requests.Response() - bad_response.status_code = 500 - bad_response._content = b"oh noes!" - good_response = requests.Response() - good_response.status_code = 200 - good_response._content = b"yay" - - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.side_effect = (bad_response, good_response) - - readurl("http://some/path", retries=1, exception_cb=lambda _: True) - assert m_request.call_count == 2 - - def test_error_cb_false(self, mocker): - mocker.patch("time.sleep") - - bad_response = requests.Response() - bad_response.status_code = 500 - bad_response._content = b"oh noes!" - - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.return_value = bad_response - - with pytest.raises(UrlError): - readurl( - "http://some/path", retries=1, exception_cb=lambda _: False - ) - assert m_request.call_count == 1 - - def test_exception_503(self, mocker): - mocker.patch("time.sleep") - - retry_response = requests.Response() - retry_response.status_code = 503 - retry_response._content = b"try again" - good_response = requests.Response() - good_response.status_code = 200 - good_response._content = b"good" - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.side_effect = (retry_response, retry_response, good_response) - - readurl("http://some/path") - assert m_request.call_count == 3 - - -event = Event() - - -class TestDualStack: - """Async testing suggestions welcome - these all rely on time-bounded - assertions (via threading.Event) to prove ordering - """ - - @pytest.mark.parametrize( - ["func", "addresses", "stagger_delay", "timeout", "expected_val"], - [ - # Assert order based on timeout - (lambda x, _: x, ("one", "two"), 1, 1, "one"), - # Assert timeout results in (None, None) - (lambda _a, _b: event.wait(1), ("one", "two"), 1, 0, None), - ( - lambda a, _b: 1 / 0 if a == "one" else a, - ("one", "two"), - 0, - 1, - "two", - ), - # Assert that exception in func is only raised - # if neither thread gets a valid result - ( - lambda a, _b: 1 / 0 if a == "two" else a, - ("one", "two"), - 0, - 1, - "one", - ), - # simulate a slow response to verify correct order - ( - lambda x, _: event.wait(1) if x != "two" else x, - ("one", "two"), - 0, - 1, - "two", - ), - # simulate a slow response to verify correct order - ( - lambda x, _: event.wait(1) if x != "tri" else x, - ("one", "two", "tri"), - 0, - 1, - "tri", - ), - ], - ) - def test_dual_stack( - self, - func, - addresses, - stagger_delay, - timeout, - expected_val, - ): - """Assert various failure modes behave as expected""" - event.clear() - - gen = partial( - dual_stack, - func, - addresses, - stagger_delay=stagger_delay, - timeout=timeout, - ) - _, result = assert_time(gen) - assert expected_val == result - - event.set() - - @pytest.mark.parametrize( - [ - "func", - "addresses", - "stagger_delay", - "timeout", - "message", - "expected_exc", - ], - [ - ( - lambda _a, _b: 1 / 0, - ("¯\\_(ツ)_/¯", "(╯°□°）╯︵ ┻━┻"), - 0, - 1, - "division by zero", - ZeroDivisionError, - ), - ( - lambda _a, _b: 1 / 0, - ("it", "really", "doesn't"), - 0, - 1, - "division by zero", - ZeroDivisionError, - ), - ( - lambda _a, _b: [][0], # pylint: disable=E0643 - ("matter", "these"), - 0, - 1, - "list index out of range", - IndexError, - ), - ( - lambda _a, _b: (_ for _ in ()).throw( - Exception("soapstone is not effective soap") - ), - ("are", "ignored"), - 0, - 1, - "soapstone is not effective soap", - Exception, - ), - ], - ) - def test_dual_stack_exceptions( - self, - func, - addresses, - stagger_delay, - timeout, - message, - expected_exc, - caplog, - ): - # Context: - # - # currently if all threads experience exception - # dual_stack() logs an error containing all exceptions - # but only raises the last exception to occur - # Verify "best effort behavior" - # dual_stack will temporarily ignore an exception in any of the - # request threads in hopes that a later thread will succeed - # this behavior is intended to allow a requests.ConnectionError - # exception from on endpoint to occur without preventing another - # thread from succeeding - event.clear() - - # Note: python3.6 repr(Exception("test")) produces different output - # than later versions, so we cannot match exact message without - # some ugly manual exception repr() function, which I'd rather not do - # in dual_stack(), so we recreate expected messages manually here - # in a version-independant way for testing, the extra comma on old - # versions won't hurt anything - exc_list = str([expected_exc(message) for _ in addresses]) - expected_msg = f"Exception(s) {exc_list} during request" - gen = partial( - dual_stack, - func, - addresses, - stagger_delay=stagger_delay, - timeout=timeout, - ) - with pytest.raises(expected_exc): - gen() # 1 - with caplog.at_level(logging.DEBUG): - try: - gen() # 2 - except expected_exc: - pass - finally: - assert 2 == len(caplog.records) - assert 2 == caplog.text.count(expected_msg) - event.set() - - def test_dual_stack_staggered(self): - """Assert expected call intervals occur""" - stagger = 0.1 - with mock.patch(M_PATH + "_run_func_with_delay") as delay_func: - - def identity_of_first_arg(x, _): - return x - - dual_stack( - identity_of_first_arg, - ["you", "and", "me", "and", "dog"], - stagger_delay=stagger, - timeout=1, - ) - - # ensure that stagger delay for each call is made with args: - # [ 0 * N, 1 * N, 2 * N, 3 * N, 4 * N, 5 * N] where N = stagger - # it appears that without an explicit wait/join we can't assert - # number of calls - calls = [ - call( - func=identity_of_first_arg, - addr="you", - timeout=1, - event=ANY, - delay=stagger * 0, - ), - call( - func=identity_of_first_arg, - addr="and", - timeout=1, - event=ANY, - delay=stagger * 1, - ), - call( - func=identity_of_first_arg, - addr="me", - timeout=1, - event=ANY, - delay=stagger * 2, - ), - call( - func=identity_of_first_arg, - addr="and", - timeout=1, - event=ANY, - delay=stagger * 3, - ), - call( - func=identity_of_first_arg, - addr="dog", - timeout=1, - event=ANY, - delay=stagger * 4, - ), - ] - num_calls = 0 - for call_instance in calls: - if call_instance in delay_func.call_args_list: - num_calls += 1 - - # we can't know the order of the submitted functions' execution - # we can't know how many of the submitted functions get called - # in advance - # - # we _do_ know what the possible arg combinations are - # we _do_ know from the mocked function how many got called - # assert that all calls that occurred had known valid arguments - # by checking for the correct number of matches - assert num_calls == len(delay_func.call_args_list) - - -ADDR1 = "https://addr1/" -SLEEP1 = "https://sleep1/" -SLEEP2 = "https://sleep2/" - - -class TestWaitForUrl: - success = "SUCCESS" - fail = "FAIL" - event = Event() - - @pytest.fixture - def retry_mocks(self, mocker): - self.mock_time_value = 0 - m_readurl = mocker.patch( - f"{M_PATH}readurl", side_effect=self.readurl_side_effect - ) - m_sleep = mocker.patch( - f"{M_PATH}time.sleep", side_effect=self.sleep_side_effect - ) - mocker.patch( - f"{M_PATH}time.monotonic", side_effect=self.time_side_effect - ) - - yield m_readurl, m_sleep - - self.mock_time_value = 0 - - @classmethod - def response_wait(cls, _request): - cls.event.wait(0.1) - return (500, {"request-id": "1"}, cls.fail) - - @classmethod - def response_nowait(cls, _request): - return (200, {"request-id": "0"}, cls.success) - - @pytest.mark.parametrize( - ["addresses", "expected_address_index", "response"], - [ - # Use timeout to test ordering happens as expected - ((ADDR1, SLEEP1), 0, "SUCCESS"), - ((SLEEP1, ADDR1), 1, "SUCCESS"), - ((SLEEP1, SLEEP2, ADDR1), 2, "SUCCESS"), - ((ADDR1, SLEEP1, SLEEP2), 0, "SUCCESS"), - ], - ) - @responses.activate - def test_order(self, addresses, expected_address_index, response): - """Check that the first response gets returned. Simulate a - non-responding endpoint with a response that has a one second wait. - - If this test proves flaky, increase wait time. Since it is async, - increasing wait time for the non-responding endpoint should not - increase total test time, assuming async_delay=0 is used and at least - one non-waiting endpoint is registered with responses. - Subsequent tests will continue execution after the first response is - received. - """ - self.event.clear() - for address in set(addresses): - responses.add_callback( - responses.GET, - address, - callback=( - self.response_wait - if "sleep" in address - else self.response_nowait - ), - content_type="application/json", - ) - - # Use async_delay=0.0 to avoid adding unnecessary time to tests - # In practice a value such as 0.150 is used - url, response_contents = wait_for_url( - urls=addresses, - max_wait=2, - timeout=0.3, - connect_synchronously=False, - async_delay=0.0, - ) - self.event.set() - - # Test for timeout (no responding endpoint) - assert addresses[expected_address_index] == url - assert response.encode() == response_contents - - @responses.activate - def test_timeout(self): - """If no endpoint responds in time, expect no response""" - - self.event.clear() - addresses = [SLEEP1, SLEEP2] - for address in set(addresses): - responses.add_callback( - responses.GET, - address, - callback=( - requests.ConnectTimeout - if "sleep" in address - else self.response_nowait - ), - content_type="application/json", - ) - - # Use async_delay=0.0 to avoid adding unnecessary time to tests - url, response_contents = wait_for_url( - urls=addresses, - max_wait=1, - timeout=1, - connect_synchronously=False, - async_delay=0, - ) - self.event.set() - assert not url - assert not response_contents - - def test_explicit_arguments(self, retry_mocks): - """Ensure that explicit arguments are respected""" - m_readurl, m_sleep = retry_mocks - wait_for_url( - urls=["http://localhost/"], - max_wait=23, - timeout=5, - sleep_time=3, - ) - - assert len(m_readurl.call_args_list) == 3 - assert len(m_sleep.call_args_list) == 2 - - for readurl_call in m_readurl.call_args_list: - assert readurl_call[1]["timeout"] == 5 - for sleep_call in m_sleep.call_args_list: - assert sleep_call[0][0] == 3 - - # Call 1 starts 0 - # Call 2 starts at 8-ish after 5 second timeout and 3 second sleep - # Call 3 starts at 16-ish for same reasons - # The 5 second timeout puts us at 21-ish and now we break - # because 21-ish + the sleep time puts us over max wait of 23 - assert pytest.approx(self.mock_time_value) == 21 - - def test_shortened_timeout(self, retry_mocks): - """Test that we shorten the last timeout to align with max_wait""" - m_readurl, _m_sleep = retry_mocks - wait_for_url( - urls=["http://localhost/"], max_wait=10, timeout=9, sleep_time=0 - ) - - assert len(m_readurl.call_args_list) == 2 - assert m_readurl.call_args_list[-1][1]["timeout"] == pytest.approx(1) - - def test_default_sleep_time(self, retry_mocks): - """Test default sleep behavior when not specified""" - _m_readurl, m_sleep = retry_mocks - wait_for_url( - urls=["http://localhost/"], - max_wait=50, - timeout=1, - ) - - expected_sleep_times = [1] * 5 + [2] * 5 + [3] * 5 - actual_sleep_times = [ - m_sleep.call_args_list[i][0][0] - for i in range(len(m_sleep.call_args_list)) - ] - assert actual_sleep_times == expected_sleep_times - - @responses.activate - def test_503(self, mocker): - mocker.patch("time.sleep") - - for _ in range(10): - responses.add( - method=responses.GET, - url="http://hi/", - status=503, - body=b"try again", - ) - responses.add( - method=responses.GET, - url="http://hi/", - status=200, - body=b"good", - ) - - assert wait_for_url(urls=["http://hi/"], max_wait=0.0001)[1] == b"good" - - @responses.activate - def test_503_async(self, mocker): - mocker.patch("time.sleep") - - for _ in range(10): - responses.add( - method=responses.GET, - url="http://hi/", - status=503, - body=b"try again", - ) - responses.add( - method=responses.GET, - url="http://hi2/", - status=503, - body="try again", - ) - responses.add( - method=responses.GET, - url="http://hi/", - status=200, - body=b"good", - ) - responses.add( - method=responses.GET, - url="http://hi2/", - status=200, - body=b"good", - ) - - assert ( - wait_for_url( - urls=["http://hi/", "http://hi2/"], - max_wait=0.0001, - async_delay=0, - connect_synchronously=False, - )[1] - == b"good" - ) - - # These side effect methods are a way of having a somewhat predictable - # output for time.monotonic(). Otherwise, we have to track too many calls - # to time.monotonic() and unrelated changes to code being called could - # cause these tests to fail. - # 0.0000001 is added to simulate additional execution time but keep it - # small enough for pytest.approx() to work - def sleep_side_effect(self, sleep_time): - self.mock_time_value += sleep_time + 0.0000001 - - def time_side_effect(self): - return self.mock_time_value - - def readurl_side_effect(self, *args, **kwargs): - if "timeout" in kwargs: - self.mock_time_value += kwargs["timeout"] + 0.0000001 - raise UrlError("test") - - -class TestHandleError: - def test_handle_error_no_cb(self): - """Test no callback.""" - assert _handle_error(UrlError("test")) is None - - def test_handle_error_cb_false(self): - """Test callback returning False.""" - with pytest.raises(UrlError) as e: - _handle_error(UrlError("test"), exception_cb=lambda _: False) - assert str(e.value) == "test" - - def test_handle_error_cb_true(self): - """Test callback returning True.""" - assert ( - _handle_error(UrlError("test"), exception_cb=lambda _: True) - ) is None - - def test_handle_503(self, caplog): - """Test 503 with no callback.""" - assert _handle_error(UrlError("test", code=503)) == 1 - assert "Unable to introspect response header" in caplog.text - - def test_handle_503_with_retry_header(self): - """Test 503 with a retry integer value.""" - assert ( - _handle_error( - UrlError("test", code=503, headers={"Retry-After": 5}) - ) - == 5 - ) - - def test_handle_503_with_retry_header_in_past(self, caplog): - """Test 503 with date in the past.""" - assert ( - _handle_error( - UrlError( - "test", - code=503, - headers={"Retry-After": "Fri, 31 Dec 1999 23:59:59 GMT"}, - ) - ) - == 1 - ) - assert "Retry-After header value is in the past" in caplog.text - - def test_handle_503_cb_true(self): - """Test 503 with a callback returning True.""" - assert ( - _handle_error( - UrlError("test", code=503), - exception_cb=lambda _: True, - ) - is None - ) - - def test_handle_503_cb_false(self): - """Test 503 with a callback returning False.""" - assert ( - _handle_error( - UrlError("test", code=503), - exception_cb=lambda _: False, - ) - == 1 - ) diff --git a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/cloudinit/cmd/main.py b/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/cloudinit/cmd/main.py deleted file mode 100644 index 7ad713eb..00000000 --- a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/cloudinit/cmd/main.py +++ /dev/null @@ -1,1403 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# Copyright (C) 2017 Amazon.com, Inc. or its affiliates -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# Author: Andrew Jorgensen -# -# This file is part of cloud-init. See LICENSE file for license information. - -import argparse -import json -import os -import sys -import traceback -import logging -import yaml -from typing import Optional, Tuple, Callable, Union - -from cloudinit import netinfo -from cloudinit import signal_handler -from cloudinit import sources -from cloudinit import socket -from cloudinit import stages -from cloudinit import url_helper -from cloudinit import util -from cloudinit import performance -from cloudinit import version -from cloudinit import warnings -from cloudinit import reporting -from cloudinit import atomic_helper -from cloudinit import lifecycle -from cloudinit import handlers -from cloudinit.log import log_util, loggers -from cloudinit.cmd.devel import read_cfg_paths -from cloudinit.config import cc_set_hostname -from cloudinit.config.modules import Modules -from cloudinit.config.schema import validate_cloudconfig_schema -from cloudinit.lifecycle import log_with_downgradable_level -from cloudinit.reporting import events -from cloudinit.settings import ( - PER_INSTANCE, - PER_ALWAYS, - PER_ONCE, - CLOUD_CONFIG, -) - -Reason = str - -# Welcome message template -WELCOME_MSG_TPL = ( - "Cloud-init v. {version} running '{action}' at " - "{timestamp}. Up {uptime} seconds." -) - -# Module section template -MOD_SECTION_TPL = "cloud_%s_modules" - -# Frequency shortname to full name -# (so users don't have to remember the full name...) -FREQ_SHORT_NAMES = { - "instance": PER_INSTANCE, - "always": PER_ALWAYS, - "once": PER_ONCE, -} - -# https://docs.cloud-init.io/en/latest/explanation/boot.html -STAGE_NAME = { - "init-local": "Local Stage", - "init": "Network Stage", - "modules-config": "Config Stage", - "modules-final": "Final Stage", -} - -LOG = logging.getLogger(__name__) - - -# Used for when a logger may not be active -# and we still want to print exceptions... -def print_exc(msg=""): - if msg: - sys.stderr.write("%s\n" % (msg)) - sys.stderr.write("-" * 60) - sys.stderr.write("\n") - traceback.print_exc(file=sys.stderr) - sys.stderr.write("-" * 60) - sys.stderr.write("\n") - - -def welcome(action, msg=None): - if not msg: - msg = welcome_format(action) - log_util.multi_log("%s\n" % (msg), console=False, stderr=True, log=LOG) - return msg - - -def welcome_format(action): - return WELCOME_MSG_TPL.format( - version=version.version_string(), - uptime=util.uptime(), - timestamp=util.time_rfc2822(), - action=action, - ) - - -@performance.timed("Closing stdin") -def close_stdin(logger: Callable[[str], None] = LOG.debug): - """ - reopen stdin as /dev/null to ensure no side effects - - logger: a function for logging messages - """ - if not os.isatty(sys.stdin.fileno()): - logger("Closing stdin") - with open(os.devnull) as fp: - os.dup2(fp.fileno(), sys.stdin.fileno()) - else: - logger("Not closing stdin, stdin is a tty.") - - -def extract_fns(args): - # Files are already opened so lets just pass that along - # since it would of broke if it couldn't have - # read that file already... - fn_cfgs = [] - if args.files: - for fh in args.files: - # The realpath is more useful in logging - # so lets resolve to that... - fn_cfgs.append(os.path.realpath(fh.name)) - return fn_cfgs - - -def run_module_section(mods: Modules, action_name, section): - full_section_name = MOD_SECTION_TPL % (section) - (which_ran, failures) = mods.run_section(full_section_name) - total_attempted = len(which_ran) + len(failures) - if total_attempted == 0: - msg = "No '%s' modules to run under section '%s'" % ( - action_name, - full_section_name, - ) - sys.stderr.write("%s\n" % (msg)) - LOG.debug(msg) - return [] - else: - LOG.debug( - "Ran %s modules with %s failures", len(which_ran), len(failures) - ) - return failures - - -def apply_reporting_cfg(cfg): - if cfg.get("reporting"): - reporting.update_configuration(cfg.get("reporting")) - - -def parse_cmdline_url(cmdline, names=("cloud-config-url", "url")): - data = util.keyval_str_to_dict(cmdline) - for key in names: - if key in data: - return key, data[key] - raise KeyError("No keys (%s) found in string '%s'" % (cmdline, names)) - - -def attempt_cmdline_url(path, network=True, cmdline=None) -> Tuple[int, str]: - """Write data from url referenced in command line to path. - - path: a file to write content to if downloaded. - network: should network access be assumed. - cmdline: the cmdline to parse for cloud-config-url. - - This is used in MAAS datasource, in "ephemeral" (read-only root) - environment where the instance netboots to iscsi ro root. - and the entity that controls the pxe config has to configure - the maas datasource. - - An attempt is made on network urls even in local datasource - for case of network set up in initramfs. - - Return value is a tuple of a logger function (logging.DEBUG) - and a message indicating what happened. - """ - - if cmdline is None: - cmdline = util.get_cmdline() - - try: - cmdline_name, url = parse_cmdline_url(cmdline) - except KeyError: - return (logging.DEBUG, "No kernel command line url found.") - - path_is_local = url.startswith(("file://", "/")) - - if path_is_local and os.path.exists(path): - if network: - m = ( - "file '%s' existed, possibly from local stage download" - " of command line url '%s'. Not re-writing." % (path, url) - ) - level = logging.INFO - if path_is_local: - level = logging.DEBUG - else: - m = ( - "file '%s' existed, possibly from previous boot download" - " of command line url '%s'. Not re-writing." % (path, url) - ) - level = logging.WARN - - return (level, m) - - kwargs = {"url": url, "timeout": 10, "retries": 2, "stream": True} - if network or path_is_local: - level = logging.WARN - kwargs["sec_between"] = 1 - else: - level = logging.DEBUG - kwargs["sec_between"] = 0.1 - - data = None - header = b"#cloud-config" - try: - resp = url_helper.read_file_or_url(**kwargs) - sniffed_content = b"" - if resp.ok(): - is_cloud_cfg = True - if isinstance(resp, url_helper.UrlResponse): - try: - sniffed_content += next( - resp.iter_content(chunk_size=len(header)) - ) - except StopIteration: - pass - if not sniffed_content.startswith(header): - is_cloud_cfg = False - elif not resp.contents.startswith(header): - is_cloud_cfg = False - if is_cloud_cfg: - if cmdline_name == "url": - return lifecycle.deprecate( - deprecated="The kernel command line key `url`", - deprecated_version="22.3", - extra_message=" Please use `cloud-config-url` " - "kernel command line parameter instead", - skip_log=True, - ) - else: - if cmdline_name == "cloud-config-url": - level = logging.WARN - else: - level = logging.INFO - return ( - level, - f"contents of '{url}' did not start with {str(header)}", - ) - else: - return ( - level, - "url '%s' returned code %s. Ignoring." % (url, resp.code), - ) - data = sniffed_content + resp.contents - - except url_helper.UrlError as e: - return (level, "retrieving url '%s' failed: %s" % (url, e)) - - util.write_file(path, data, mode=0o600) - return ( - logging.INFO, - "wrote cloud-config data from %s='%s' to %s" - % (cmdline_name, url, path), - ) - - -def purge_cache_on_python_version_change(init): - """Purge the cache if python version changed on us. - - There could be changes not represented in our cache (obj.pkl) after we - upgrade to a new version of python, so at that point clear the cache - """ - current_python_version = "%d.%d" % ( - sys.version_info.major, - sys.version_info.minor, - ) - python_version_path = os.path.join( - init.paths.get_cpath("data"), "python-version" - ) - if os.path.exists(python_version_path): - cached_python_version = util.load_text_file(python_version_path) - # The Python version has changed out from under us, anything that was - # pickled previously is likely useless due to API changes. - if cached_python_version != current_python_version: - LOG.debug("Python version change detected. Purging cache") - init.purge_cache(True) - util.write_file(python_version_path, current_python_version) - else: - if os.path.exists(init.paths.get_ipath_cur("obj_pkl")): - LOG.info( - "Writing python-version file. " - "Cache compatibility status is currently unknown." - ) - util.write_file(python_version_path, current_python_version) - - -def _should_bring_up_interfaces(init, args): - if util.get_cfg_option_bool(init.cfg, "disable_network_activation"): - return False - return not args.local - - -def _should_wait_via_user_data( - raw_config: Optional[Union[str, bytes]] -) -> Tuple[bool, Reason]: - """Determine if our cloud-config requires us to wait - - User data requires us to wait during cloud-init network phase if: - - We have user data that is anything other than cloud-config - - This can likely be further optimized in the future to include - other user data types - - cloud-config contains: - - bootcmd - - random_seed command - - mounts - - write_files with source - """ - if not raw_config: - return False, "no configuration found" - - # Since this could be some arbitrarily large blob of binary data, - # such as a gzipped file, only grab enough to inspect the header. - # Since we can get a header like #cloud-config-archive, make sure - # we grab enough to not be incorrectly identified as cloud-config. - if ( - handlers.type_from_starts_with(raw_config.strip()[:42]) - != "text/cloud-config" - ): - return True, "non-cloud-config user data found" - - try: - parsed_yaml = yaml.safe_load(raw_config) - except Exception as e: - log_with_downgradable_level( - logger=LOG, - version="24.4", - requested_level=logging.WARNING, - msg="Unexpected failure parsing userdata: %s", - args=(e,), - ) - return True, "failed to parse user data as yaml" - - if not isinstance(parsed_yaml, dict): - return True, "parsed config not in cloud-config format" - - # These all have the potential to require network access, so we should wait - if "write_files" in parsed_yaml: - for item in parsed_yaml["write_files"]: - source_dict = item.get("source") or {} - source_uri = source_dict.get("uri", "") - if source_uri and not (source_uri.startswith(("/", "file:"))): - return True, "write_files with source uri found" - return False, "write_files without source uri found" - if parsed_yaml.get("bootcmd"): - return True, "bootcmd found" - if parsed_yaml.get("random_seed", {}).get("command"): - return True, "random_seed command found" - if parsed_yaml.get("mounts"): - return True, "mounts found" - return False, "cloud-config does not contain network requiring elements" - - -def _should_wait_on_network( - datasource: Optional[sources.DataSource], -) -> Tuple[bool, Reason]: - """Determine if we should wait on network connectivity for cloud-init. - - We need to wait during the cloud-init network phase if: - - We have no datasource - - We have user data that may require network access - """ - if not datasource: - return True, "no datasource found" - user_should_wait, user_reason = _should_wait_via_user_data( - datasource.get_userdata_raw() - ) - if user_should_wait: - return True, f"{user_reason} in user data" - vendor_should_wait, vendor_reason = _should_wait_via_user_data( - datasource.get_vendordata_raw() - ) - if vendor_should_wait: - return True, f"{vendor_reason} in vendor data" - vendor2_should_wait, vendor2_reason = _should_wait_via_user_data( - datasource.get_vendordata2_raw() - ) - if vendor2_should_wait: - return True, f"{vendor2_reason} in vendor data2" - - return ( - False, - ( - f"user data: {user_reason}, " - f"vendor data: {vendor_reason}, " - f"vendor data2: {vendor2_reason}" - ), - ) - - -def main_init(name, args): - deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] - if args.local: - deps = [sources.DEP_FILESYSTEM] - - early_logs = [ - attempt_cmdline_url( - path=os.path.join( - "%s.d" % CLOUD_CONFIG, "91_kernel_cmdline_url.cfg" - ), - network=not args.local, - ) - ] - - # Cloud-init 'init' stage is broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Setup logging/output redirections with resultant config (if any) - # 3. Initialize the cloud-init filesystem - # 4. Check if we can stop early by looking for various files - # 5. Fetch the datasource - # 6. Connect to the current instance location + update the cache - # 7. Consume the userdata (handlers get activated here) - # 8. Construct the modules object - # 9. Adjust any subsequent logging/output redirections using the modules - # objects config as it may be different from init object - # 10. Run the modules for the 'init' stage - # 11. Done! - bootstage_name = "init-local" if args.local else "init" - w_msg = welcome_format(bootstage_name) - init = stages.Init(ds_deps=deps, reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - outfmt = None - errfmt = None - try: - if not args.skip_log_setup: - close_stdin(lambda msg: early_logs.append((logging.DEBUG, msg))) - outfmt, errfmt = util.fixup_output(init.cfg, name) - else: - outfmt, errfmt = util.get_output_cfg(init.cfg, name) - except Exception: - msg = "Failed to setup output redirection!" - util.logexc(LOG, msg) - print_exc(msg) - early_logs.append((logging.WARN, msg)) - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug( - "Logging being reset, this logger may no longer be active shortly" - ) - loggers.reset_logging() - if not args.skip_log_setup: - loggers.setup_logging(init.cfg) - apply_reporting_cfg(init.cfg) - - # Any log usage prior to setup_logging above did not have local user log - # config applied. We send the welcome message now, as stderr/out have - # been redirected and log now configured. - welcome(name, msg=w_msg) - LOG.info("PID [%s] started cloud-init '%s'.", os.getppid(), bootstage_name) - - # re-play early log messages before logging was setup - for lvl, msg in early_logs: - LOG.log(lvl, msg) - - # Stage 3 - try: - init.initialize() - except Exception: - util.logexc(LOG, "Failed to initialize, likely bad things to come!") - # Stage 4 - path_helper = init.paths - purge_cache_on_python_version_change(init) - mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK - - if mode == sources.DSMODE_NETWORK: - if not os.path.exists(init.paths.get_runpath(".skip-network")): - LOG.debug("Will wait for network connectivity before continuing") - init.distro.wait_for_network() - existing = "trust" - sys.stderr.write("%s\n" % (netinfo.debug_info())) - else: - existing = "check" - mcfg = util.get_cfg_option_bool(init.cfg, "manual_cache_clean", False) - if mcfg: - LOG.debug("manual cache clean set from config") - existing = "trust" - else: - mfile = path_helper.get_ipath_cur("manual_clean_marker") - if os.path.exists(mfile): - LOG.debug("manual cache clean found from marker: %s", mfile) - existing = "trust" - - init.purge_cache() - - # Stage 5 - bring_up_interfaces = _should_bring_up_interfaces(init, args) - try: - init.fetch(existing=existing) - # if in network mode, and the datasource is local - # then work was done at that stage. - if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: - LOG.debug( - "[%s] Exiting. datasource %s in local mode", - mode, - init.datasource, - ) - return (None, []) - except sources.DataSourceNotFoundException: - # In the case of 'cloud-init init' without '--local' it is a bit - # more likely that the user would consider it failure if nothing was - # found. - if mode == sources.DSMODE_LOCAL: - LOG.debug("No local datasource found") - else: - util.logexc( - LOG, "No instance datasource found! Likely bad things to come!" - ) - if not args.force: - init.apply_network_config(bring_up=bring_up_interfaces) - LOG.debug("[%s] Exiting without datasource", mode) - if mode == sources.DSMODE_LOCAL: - return (None, []) - else: - return (None, ["No instance datasource found."]) - else: - LOG.debug( - "[%s] barreling on in force mode without datasource", mode - ) - - _maybe_persist_instance_data(init) - # Stage 6 - iid = init.instancify() - LOG.debug( - "[%s] %s will now be targeting instance id: %s. new=%s", - mode, - name, - iid, - init.is_new_instance(), - ) - - if mode == sources.DSMODE_LOCAL: - # Before network comes up, set any configured hostname to allow - # dhcp clients to advertize this hostname to any DDNS services - # LP: #1746455. - _maybe_set_hostname(init, stage="local", retry_stage="network") - - init.apply_network_config(bring_up=bring_up_interfaces) - - if mode == sources.DSMODE_LOCAL: - should_wait, reason = _should_wait_on_network(init.datasource) - if should_wait: - LOG.debug( - "Network connectivity determined necessary for " - "cloud-init's network stage. Reason: %s", - reason, - ) - else: - LOG.debug( - "Network connectivity determined unnecessary for " - "cloud-init's network stage. Reason: %s", - reason, - ) - util.write_file(init.paths.get_runpath(".skip-network"), "") - - if init.datasource.dsmode != mode: - LOG.debug( - "[%s] Exiting. datasource %s not in local mode.", - mode, - init.datasource, - ) - return (init.datasource, []) - else: - LOG.debug( - "[%s] %s is in local mode, will apply init modules now.", - mode, - init.datasource, - ) - - # Give the datasource a chance to use network resources. - # This is used on Azure to communicate with the fabric over network. - init.setup_datasource() - # update fully realizes user-data (pulling in #include if necessary) - init.update() - _maybe_set_hostname(init, stage="init-net", retry_stage="modules:config") - # Stage 7 - try: - # Attempt to consume the data per instance. - # This may run user-data handlers and/or perform - # url downloads and such as needed. - (ran, _results) = init.cloudify().run( - "consume_data", - init.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - if not ran: - # Just consume anything that is set to run per-always - # if nothing ran in the per-instance code - # - # See: https://bugs.launchpad.net/bugs/819507 for a little - # reason behind this... - init.consume_data(PER_ALWAYS) - except Exception: - util.logexc(LOG, "Consuming user data failed!") - return (init.datasource, ["Consuming user data failed!"]) - - # Validate user-data adheres to schema definition - cloud_cfg_path = init.paths.get_ipath_cur("cloud_config") - if os.path.exists(cloud_cfg_path) and os.stat(cloud_cfg_path).st_size != 0: - validate_cloudconfig_schema( - config=yaml.safe_load(util.load_text_file(cloud_cfg_path)), - strict=False, - log_details=False, - log_deprecations=True, - ) - else: - LOG.debug("Skipping user-data validation. No user-data found.") - - apply_reporting_cfg(init.cfg) - - # Stage 8 - re-read and apply relevant cloud-config to include user-data - mods = Modules(init, extract_fns(args), reporter=args.reporter) - # Stage 9 - try: - outfmt_orig = outfmt - errfmt_orig = errfmt - (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) - if outfmt_orig != outfmt or errfmt_orig != errfmt: - LOG.warning("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) - (outfmt, errfmt) = util.fixup_output(mods.cfg, name) - except Exception: - util.logexc(LOG, "Failed to re-adjust output redirection!") - loggers.setup_logging(mods.cfg) - - # give the activated datasource a chance to adjust - init.activate_datasource() - - di_report_warn(datasource=init.datasource, cfg=init.cfg) - - # Stage 10 - return (init.datasource, run_module_section(mods, name, name)) - - -def di_report_warn(datasource, cfg): - if "di_report" not in cfg: - LOG.debug("no di_report found in config.") - return - - dicfg = cfg["di_report"] - if dicfg is None: - # ds-identify may write 'di_report:\n #comment\n' - # which reads as {'di_report': None} - LOG.debug("di_report was None.") - return - - if not isinstance(dicfg, dict): - LOG.warning("di_report config not a dictionary: %s", dicfg) - return - - dslist = dicfg.get("datasource_list") - if dslist is None: - LOG.warning("no 'datasource_list' found in di_report.") - return - elif not isinstance(dslist, list): - LOG.warning("di_report/datasource_list not a list: %s", dslist) - return - - # ds.__module__ is like cloudinit.sources.DataSourceName - # where Name is the thing that shows up in datasource_list. - modname = datasource.__module__.rpartition(".")[2] - if modname.startswith(sources.DS_PREFIX): - modname = modname[len(sources.DS_PREFIX) :] - else: - LOG.warning( - "Datasource '%s' came from unexpected module '%s'.", - datasource, - modname, - ) - - if modname in dslist: - LOG.debug( - "used datasource '%s' from '%s' was in di_report's list: %s", - datasource, - modname, - dslist, - ) - return - - warnings.show_warning( - "dsid_missing_source", cfg, source=modname, dslist=str(dslist) - ) - - -def main_modules(action_name, args): - name = args.mode - # Cloud-init 'modules' stages are broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Get the datasource from the init object, if it does - # not exist then that means the main_init stage never - # worked, and thus this stage can not run. - # 3. Construct the modules object - # 4. Adjust any subsequent logging/output redirections using - # the modules objects configuration - # 5. Run the modules for the given stage name - # 6. Done! - bootstage_name = "%s:%s" % (action_name, name) - w_msg = welcome_format(bootstage_name) - init = stages.Init(ds_deps=[], reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - try: - init.fetch(existing="trust") - except sources.DataSourceNotFoundException: - # There was no datasource found, theres nothing to do - msg = ( - "Can not apply stage %s, no datasource found! Likely bad " - "things to come!" % name - ) - util.logexc(LOG, msg) - print_exc(msg) - if not args.force: - return [(msg)] - _maybe_persist_instance_data(init) - # Stage 3 - mods = Modules(init, extract_fns(args), reporter=args.reporter) - # Stage 4 - try: - if not args.skip_log_setup: - close_stdin() - util.fixup_output(mods.cfg, name) - except Exception: - util.logexc(LOG, "Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug( - "Logging being reset, this logger may no longer be active shortly" - ) - loggers.reset_logging() - if not args.skip_log_setup: - loggers.setup_logging(mods.cfg) - apply_reporting_cfg(init.cfg) - - # now that logging is setup and stdout redirected, send welcome - welcome(name, msg=w_msg) - LOG.info("PID [%s] started cloud-init '%s'.", os.getppid(), bootstage_name) - - if name == "init": - lifecycle.deprecate( - deprecated="`--mode init`", - deprecated_version="24.1", - extra_message="Use `cloud-init init` instead.", - ) - - # Stage 5 - return run_module_section(mods, name, name) - - -def main_single(name, args): - # Cloud-init single stage is broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Attempt to fetch the datasource (warn if it doesn't work) - # 3. Construct the modules object - # 4. Adjust any subsequent logging/output redirections using - # the modules objects configuration - # 5. Run the single module - # 6. Done! - mod_name = args.name - w_msg = welcome_format(name) - init = stages.Init(ds_deps=[], reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - try: - init.fetch(existing="trust") - except sources.DataSourceNotFoundException: - # There was no datasource found, - # that might be bad (or ok) depending on - # the module being ran (so continue on) - util.logexc( - LOG, "Failed to fetch your datasource, likely bad things to come!" - ) - print_exc( - "Failed to fetch your datasource, likely bad things to come!" - ) - if not args.force: - return 1 - _maybe_persist_instance_data(init) - # Stage 3 - mods = Modules(init, extract_fns(args), reporter=args.reporter) - mod_args = args.module_args - if mod_args: - LOG.debug("Using passed in arguments %s", mod_args) - mod_freq = args.frequency - if mod_freq: - LOG.debug("Using passed in frequency %s", mod_freq) - mod_freq = FREQ_SHORT_NAMES.get(mod_freq) - # Stage 4 - try: - close_stdin() - util.fixup_output(mods.cfg, None) - except Exception: - util.logexc(LOG, "Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug( - "Logging being reset, this logger may no longer be active shortly" - ) - loggers.reset_logging() - loggers.setup_logging(mods.cfg) - apply_reporting_cfg(init.cfg) - - # now that logging is setup and stdout redirected, send welcome - welcome(name, msg=w_msg) - - # Stage 5 - (which_ran, failures) = mods.run_single(mod_name, mod_args, mod_freq) - if failures: - LOG.warning("Ran %s but it failed!", mod_name) - return 1 - elif not which_ran: - LOG.warning("Did not run %s, does it exist?", mod_name) - return 1 - else: - # Guess it worked - return 0 - - -def status_wrapper(name, args): - paths = read_cfg_paths() - data_d = paths.get_cpath("data") - link_d = os.path.normpath(paths.run_dir) - - status_path = os.path.join(data_d, "status.json") - status_link = os.path.join(link_d, "status.json") - result_path = os.path.join(data_d, "result.json") - result_link = os.path.join(link_d, "result.json") - root_logger = logging.getLogger() - - util.ensure_dirs( - ( - data_d, - link_d, - ) - ) - - (_name, functor) = args.action - - if name == "init": - if args.local: - mode = "init-local" - else: - mode = "init" - elif name == "modules": - mode = "modules-%s" % args.mode - else: - raise ValueError("unknown name: %s" % name) - - if mode not in STAGE_NAME: - raise ValueError( - "Invalid cloud init mode specified '{0}'".format(mode) - ) - - nullstatus = { - "errors": [], - "recoverable_errors": {}, - "start": None, - "finished": None, - } - status = { - "v1": { - "datasource": None, - "init": nullstatus.copy(), - "init-local": nullstatus.copy(), - "modules-config": nullstatus.copy(), - "modules-final": nullstatus.copy(), - } - } - if mode == "init-local": - for f in (status_link, result_link, status_path, result_path): - util.del_file(f) - else: - try: - status = json.loads(util.load_text_file(status_path)) - except Exception: - pass - - if mode not in status["v1"]: - # this should never happen, but leave it just to be safe - status["v1"][mode] = nullstatus.copy() - - v1 = status["v1"] - v1["stage"] = mode - if v1[mode]["start"] and not v1[mode]["finished"]: - # This stage was restarted, which isn't expected. - LOG.warning( - "Unexpected start time found for %s. Was this stage restarted?", - STAGE_NAME[mode], - ) - - v1[mode]["start"] = float(util.uptime()) - handler = next( - filter( - lambda h: isinstance(h, loggers.LogExporter), root_logger.handlers - ) - ) - preexisting_recoverable_errors = handler.export_logs() - - # Write status.json prior to running init / module code - atomic_helper.write_json(status_path, status) - util.sym_link( - os.path.relpath(status_path, link_d), status_link, force=True - ) - - try: - ret = functor(name, args) - if mode in ("init", "init-local"): - (datasource, errors) = ret - if datasource is not None: - v1["datasource"] = str(datasource) - else: - errors = ret - - v1[mode]["errors"].extend([str(e) for e in errors]) - except Exception as e: - LOG.exception("failed stage %s", mode) - print_exc("failed run of stage %s" % mode) - v1[mode]["errors"].append(str(e)) - except SystemExit as e: - # All calls to sys.exit() resume running here. - # silence a pylint false positive - # https://github.com/pylint-dev/pylint/issues/9556 - if e.code: # pylint: disable=using-constant-test - # Only log errors when sys.exit() is called with a non-zero - # exit code - LOG.exception("failed stage %s", mode) - print_exc("failed run of stage %s" % mode) - v1[mode]["errors"].append(f"sys.exit({str(e.code)}) called") - finally: - # Before it exits, cloud-init will: - # 1) Write status.json (and result.json if in Final stage). - # 2) Write the final log message containing module run time. - # 3) Flush any queued reporting event handlers. - v1[mode]["finished"] = float(util.uptime()) - v1["stage"] = None - - # merge new recoverable errors into existing recoverable error list - new_recoverable_errors = handler.export_logs() - handler.clean_logs() - for key in new_recoverable_errors.keys(): - if key in preexisting_recoverable_errors: - v1[mode]["recoverable_errors"][key] = list( - set( - preexisting_recoverable_errors[key] - + new_recoverable_errors[key] - ) - ) - else: - v1[mode]["recoverable_errors"][key] = new_recoverable_errors[ - key - ] - - # Write status.json after running init / module code - atomic_helper.write_json(status_path, status) - - if mode == "modules-final": - # write the 'finished' file - errors = [] - for m in v1.keys(): - if isinstance(v1[m], dict) and v1[m].get("errors"): - errors.extend(v1[m].get("errors", [])) - - atomic_helper.write_json( - result_path, - {"v1": {"datasource": v1["datasource"], "errors": errors}}, - ) - util.sym_link( - os.path.relpath(result_path, link_d), result_link, force=True - ) - - return len(v1[mode]["errors"]) - - -def _maybe_persist_instance_data(init: stages.Init): - """Write instance-data.json file if absent and datasource is restored.""" - if init.datasource and init.ds_restored: - instance_data_file = init.paths.get_runpath("instance_data") - if not os.path.exists(instance_data_file): - init.datasource.persist_instance_data(write_cache=False) - - -def _maybe_set_hostname(init, stage, retry_stage): - """Call set_hostname if metadata, vendordata or userdata provides it. - - @param stage: String representing current stage in which we are running. - @param retry_stage: String represented logs upon error setting hostname. - """ - cloud = init.cloudify() - (hostname, _fqdn, _) = util.get_hostname_fqdn( - init.cfg, cloud, metadata_only=True - ) - if hostname: # meta-data or user-data hostname content - try: - cc_set_hostname.handle("set_hostname", init.cfg, cloud, None) - except cc_set_hostname.SetHostnameError as e: - LOG.debug( - "Failed setting hostname in %s stage. Will" - " retry in %s stage. Error: %s.", - stage, - retry_stage, - str(e), - ) - - -def main_features(name, args): - sys.stdout.write("\n".join(sorted(version.FEATURES)) + "\n") - - -def main(sysv_args=None): - loggers.configure_root_logger() - if not sysv_args: - sysv_args = sys.argv - parser = argparse.ArgumentParser(prog=sysv_args.pop(0)) - - # Top level args - parser.add_argument( - "--version", - "-v", - action="version", - version="%(prog)s " + (version.version_string()), - help="Show program's version number and exit.", - ) - parser.add_argument( - "--debug", - "-d", - action="store_true", - help="Show additional pre-action logging (default: %(default)s).", - default=False, - ) - parser.add_argument( - "--force", - action="store_true", - help=( - "Force running even if no datasource is" - " found (use at your own risk)." - ), - dest="force", - default=False, - ) - - parser.add_argument( - "--all-stages", - dest="all_stages", - action="store_true", - help=( - "Run cloud-init's stages under a single process using a " - "syncronization protocol. This is not intended for CLI usage." - ), - default=False, - ) - - parser.set_defaults(reporter=None) - subparsers = parser.add_subparsers(title="Subcommands", dest="subcommand") - - # Each action and its sub-options (if any) - parser_init = subparsers.add_parser( - "init", help="Initialize cloud-init and perform initial modules." - ) - parser_init.add_argument( - "--local", - "-l", - action="store_true", - help="Start in local mode (default: %(default)s).", - default=False, - ) - parser_init.add_argument( - "--file", - "-f", - action="append", - dest="files", - help="Use additional yaml configuration files.", - type=argparse.FileType("rb"), - ) - # This is used so that we can know which action is selected + - # the functor to use to run this subcommand - parser_init.set_defaults(action=("init", main_init)) - - # These settings are used for the 'config' and 'final' stages - parser_mod = subparsers.add_parser( - "modules", help="Activate modules using a given configuration key." - ) - extra_help = lifecycle.deprecate( - deprecated="`init`", - deprecated_version="24.1", - extra_message="Use `cloud-init init` instead.", - skip_log=True, - ).message - parser_mod.add_argument( - "--mode", - "-m", - action="store", - help=( - f"Module configuration name to use (default: %(default)s)." - f" {extra_help}" - ), - default="config", - choices=("init", "config", "final"), - ) - parser_mod.add_argument( - "--file", - "-f", - action="append", - dest="files", - help="Use additional yaml configuration files.", - type=argparse.FileType("rb"), - ) - parser_mod.set_defaults(action=("modules", main_modules)) - - # This subcommand allows you to run a single module - parser_single = subparsers.add_parser( - "single", help="Run a single module." - ) - parser_single.add_argument( - "--name", - "-n", - action="store", - help="Module name to run.", - required=True, - ) - parser_single.add_argument( - "--frequency", - action="store", - help="Module frequency for this run.", - required=False, - choices=list(FREQ_SHORT_NAMES.keys()), - ) - parser_single.add_argument( - "--report", - action="store_true", - help="Enable reporting.", - required=False, - ) - parser_single.add_argument( - "module_args", - nargs="*", - metavar="argument", - help="Any additional arguments to pass to this module.", - ) - parser_single.add_argument( - "--file", - "-f", - action="append", - dest="files", - help="Use additional yaml configuration files.", - type=argparse.FileType("rb"), - ) - parser_single.set_defaults(action=("single", main_single)) - - parser_query = subparsers.add_parser( - "query", - help="Query standardized instance metadata from the command line.", - ) - - parser_features = subparsers.add_parser( - "features", help="List defined features." - ) - parser_features.set_defaults(action=("features", main_features)) - - parser_analyze = subparsers.add_parser( - "analyze", help="Devel tool: Analyze cloud-init logs and data." - ) - - parser_devel = subparsers.add_parser( - "devel", help="Run development tools." - ) - - parser_collect_logs = subparsers.add_parser( - "collect-logs", help="Collect and tar all cloud-init debug info." - ) - - parser_clean = subparsers.add_parser( - "clean", help="Remove logs and artifacts so cloud-init can re-run." - ) - - parser_status = subparsers.add_parser( - "status", help="Report cloud-init status or wait on completion." - ) - - parser_schema = subparsers.add_parser( - "schema", help="Validate cloud-config files using jsonschema." - ) - - if sysv_args: - # Only load subparsers if subcommand is specified to avoid load cost - subcommand = next( - (posarg for posarg in sysv_args if not posarg.startswith("-")), - None, - ) - if subcommand == "analyze": - from cloudinit.analyze import get_parser as analyze_parser - - # Construct analyze subcommand parser - analyze_parser(parser_analyze) - elif subcommand == "devel": - from cloudinit.cmd.devel.parser import get_parser as devel_parser - - # Construct devel subcommand parser - devel_parser(parser_devel) - elif subcommand == "collect-logs": - from cloudinit.cmd.devel.logs import ( - get_parser as logs_parser, - handle_collect_logs_args, - ) - - logs_parser(parser=parser_collect_logs) - parser_collect_logs.set_defaults( - action=("collect-logs", handle_collect_logs_args) - ) - elif subcommand == "clean": - from cloudinit.cmd.clean import ( - get_parser as clean_parser, - handle_clean_args, - ) - - clean_parser(parser_clean) - parser_clean.set_defaults(action=("clean", handle_clean_args)) - elif subcommand == "query": - from cloudinit.cmd.query import ( - get_parser as query_parser, - handle_args as handle_query_args, - ) - - query_parser(parser_query) - parser_query.set_defaults(action=("render", handle_query_args)) - elif subcommand == "schema": - from cloudinit.config.schema import ( - get_parser as schema_parser, - handle_schema_args, - ) - - schema_parser(parser_schema) - parser_schema.set_defaults(action=("schema", handle_schema_args)) - elif subcommand == "status": - from cloudinit.cmd.status import ( - get_parser as status_parser, - handle_status_args, - ) - - status_parser(parser_status) - parser_status.set_defaults(action=("status", handle_status_args)) - else: - parser.error("a subcommand is required") - - args = parser.parse_args(args=sysv_args) - setattr(args, "skip_log_setup", False) - if not args.all_stages: - return sub_main(args) - return all_stages(parser) - - -def all_stages(parser): - """Run all stages in a single process using an ordering protocol.""" - LOG.info("Running cloud-init in single process mode.") - - # this _must_ be called before sd_notify is called otherwise netcat may - # attempt to send "start" before a socket exists - sync = socket.SocketSync("local", "network", "config", "final") - - # notify systemd that this stage has completed - socket.sd_notify("READY=1") - # wait for cloud-init-local.service to start - with sync("local"): - # set up logger - args = parser.parse_args(args=["init", "--local"]) - args.skip_log_setup = False - # run local stage - sync.systemd_exit_code = sub_main(args) - - # wait for cloud-init-network.service to start - with sync("network"): - # skip re-setting up logger - args = parser.parse_args(args=["init"]) - args.skip_log_setup = True - # run init stage - sync.systemd_exit_code = sub_main(args) - - # wait for cloud-config.service to start - with sync("config"): - # skip re-setting up logger - args = parser.parse_args(args=["modules", "--mode=config"]) - args.skip_log_setup = True - # run config stage - sync.systemd_exit_code = sub_main(args) - - # wait for cloud-final.service to start - with sync("final"): - # skip re-setting up logger - args = parser.parse_args(args=["modules", "--mode=final"]) - args.skip_log_setup = True - # run final stage - sync.systemd_exit_code = sub_main(args) - - # signal completion to cloud-init-main.service - if sync.experienced_any_error: - message = "a stage of cloud-init exited non-zero" - if sync.first_exception: - message = f"first exception received: {sync.first_exception}" - socket.sd_notify( - f"STATUS=Completed with failure, {message}. Run 'cloud-init status" - " --long' for more details." - ) - socket.sd_notify("STOPPING=1") - # exit 1 for a fatal failure in any stage - return 1 - else: - socket.sd_notify("STATUS=Completed") - socket.sd_notify("STOPPING=1") - - -def sub_main(args): - - # Subparsers.required = True and each subparser sets action=(name, functor) - (name, functor) = args.action - - # Setup basic logging for cloud-init: - # - for cloud-init stages if --debug - # - for all other subcommands: - # - if --debug is passed, logging.DEBUG - # - if --debug is not passed, logging.WARNING - if name not in ("init", "modules"): - loggers.setup_basic_logging( - logging.DEBUG if args.debug else logging.WARNING - ) - elif args.debug: - loggers.setup_basic_logging() - - # Setup signal handlers before running - signal_handler.attach_handlers() - - # Write boot stage data to write status.json and result.json - # Exclude modules --mode=init, since it is not a real boot stage and - # should not be written into status.json - if "init" == name or ("modules" == name and "init" != args.mode): - functor = status_wrapper - - rname = None - report_on = True - if name == "init": - if args.local: - rname, rdesc = ("init-local", "searching for local datasources") - else: - rname, rdesc = ( - "init-network", - "searching for network datasources", - ) - elif name == "modules": - rname, rdesc = ( - "modules-%s" % args.mode, - "running modules for %s" % args.mode, - ) - elif name == "single": - rname, rdesc = ( - "single/%s" % args.name, - "running single module %s" % args.name, - ) - report_on = args.report - else: - rname = name - rdesc = "running 'cloud-init %s'" % name - report_on = False - - args.reporter = events.ReportEventStack( - rname, rdesc, reporting_enabled=report_on - ) - - with args.reporter: - with performance.Timed(f"cloud-init stage: '{rname}'"): - retval = functor(name, args) - reporting.flush_events() - - # handle return code for main_modules, as it is not wrapped by - # status_wrapped when mode == init - if "modules" == name and "init" == args.mode: - retval = len(retval) - - return retval - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/cloudinit/features.py b/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/cloudinit/features.py deleted file mode 100644 index 5f03324a..00000000 --- a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/cloudinit/features.py +++ /dev/null @@ -1,131 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. -""" -Feature flags are used as a way to easily toggle configuration -**at build time**. They are provided to accommodate feature deprecation and -downstream configuration changes. - -Currently used upstream values for feature flags are set in -``cloudinit/features.py``. Overrides to these values should be -patched directly (e.g., via quilt patch) by downstreams. - -Each flag should include a short comment regarding the reason for -the flag and intended lifetime. - -Tests are required for new feature flags, and tests must verify -all valid states of a flag, not just the default state. -""" -import re -import sys -from typing import Dict - -ERROR_ON_USER_DATA_FAILURE = True -""" -If there is a failure in obtaining user data (i.e., #include or -decompress fails) and ``ERROR_ON_USER_DATA_FAILURE`` is ``False``, -cloud-init will log a warning and proceed. If it is ``True``, -cloud-init will instead raise an exception. - -As of 20.3, ``ERROR_ON_USER_DATA_FAILURE`` is ``True``. - -(This flag can be removed after Focal is no longer supported.) -""" - - -ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES = False -""" -When configuring apt mirrors, if -``ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES`` is ``True`` cloud-init -will detect that a datasource's ``availability_zone`` property looks -like an EC2 availability zone and set the ``ec2_region`` variable when -generating mirror URLs; this can lead to incorrect mirrors being -configured in clouds whose AZs follow EC2's naming pattern. - -As of 20.3, ``ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES`` is ``False`` -so we no longer include ``ec2_region`` in mirror determination on -non-AWS cloud platforms. - -If the old behavior is desired, users can provide the appropriate -mirrors via :py:mod:`apt: ` -directives in cloud-config. -""" - - -EXPIRE_APPLIES_TO_HASHED_USERS = True -""" -If ``EXPIRE_APPLIES_TO_HASHED_USERS`` is True, then when expire is set true -in cc_set_passwords, hashed passwords will be expired. Previous to 22.3, -only non-hashed passwords were expired. - -(This flag can be removed after Jammy is no longer supported.) -""" - -NETPLAN_CONFIG_ROOT_READ_ONLY = True -""" -If ``NETPLAN_CONFIG_ROOT_READ_ONLY`` is True, then netplan configuration will -be written as a single root read-only file /etc/netplan/50-cloud-init.yaml. -This prevents wifi passwords in network v2 configuration from being -world-readable. Prior to 23.1, netplan configuration is world-readable. - -(This flag can be removed after Jammy is no longer supported.) -""" - - -NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH = True -""" -Append a forward slash '/' if NoCloud seedurl does not end with either -a querystring or forward slash. Prior to 23.1, nocloud seedurl would be used -unaltered, appending meta-data, user-data and vendor-data to without URL path -separators. - -(This flag can be removed when Jammy is no longer supported.) -""" - -APT_DEB822_SOURCE_LIST_FILE = True -""" -On Debian and Ubuntu systems, cc_apt_configure will write a deb822 compatible -/etc/apt/sources.list.d/(debian|ubuntu).sources file. When set False, continue -to write /etc/apt/sources.list directly. -""" - -DEPRECATION_INFO_BOUNDARY = "24.1" -""" -DEPRECATION_INFO_BOUNDARY is used by distros to configure at which upstream -version to start logging deprecations at a level higher than INFO. - -The default value "devel" tells cloud-init to log all deprecations higher -than INFO. This value may be overriden by downstreams in order to maintain -stable behavior across releases. - -Jsonschema key deprecations and inline logger deprecations include a -deprecated_version key. When the variable below is set to a version, -cloud-init will use that version as a demarcation point. Deprecations which -are added after this version will be logged as at an INFO level. Deprecations -which predate this version will be logged at the higher DEPRECATED level. -Downstreams that want stable log behavior may set the variable below to the -first version released in their stable distro. By doing this, they can expect -that newly added deprecations will be logged at INFO level. The implication of -the different log levels is that logs at DEPRECATED level result in a return -code of 2 from `cloud-init status`. - -This may may also be used in some limited cases where new error messages may be -logged which increase the risk of regression in stable downstreams where the -error was previously unreported yet downstream users expected stable behavior -across new cloud-init releases. - -format: - - :: = | - ::= "devel" - ::= "." ["." ] - -where , , and are positive integers -""" - - -def get_features() -> Dict[str, bool]: - """Return a dict of applicable features/overrides and their values.""" - return { - k: getattr(sys.modules["cloudinit.features"], k) - for k in sys.modules["cloudinit.features"].__dict__.keys() - if re.match(r"^[_A-Z0-9]+$", k) - } diff --git a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/integration_tests/datasources/test_nocloud.py b/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/integration_tests/datasources/test_nocloud.py deleted file mode 100644 index 66645ce7..00000000 --- a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/integration_tests/datasources/test_nocloud.py +++ /dev/null @@ -1,463 +0,0 @@ -"""NoCloud datasource integration tests.""" - -from textwrap import dedent - -import pytest -from pycloudlib.lxd.instance import LXDInstance - -from cloudinit import lifecycle -from cloudinit.subp import subp -from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.integration_settings import PLATFORM -from tests.integration_tests.releases import CURRENT_RELEASE, FOCAL -from tests.integration_tests.util import ( - get_feature_flag_value, - network_wait_logged, - override_kernel_command_line, - verify_clean_boot, - verify_clean_log, -) - -VENDOR_DATA = """\ -#cloud-config -bootcmd: - - touch /var/tmp/seeded_vendordata_test_file -""" - - -LXD_METADATA_NOCLOUD_SEED = """\ - /var/lib/cloud/seed/nocloud-net/meta-data: - when: - - create - - copy - create_only: false - template: emptycfg.tpl - properties: - default: | - #cloud-config - {} - /var/lib/cloud/seed/nocloud-net/user-data: - when: - - create - - copy - create_only: false - template: emptycfg.tpl - properties: - default: | - #cloud-config - {} -""" - - -def setup_nocloud(instance: LXDInstance): - # On Jammy and above, LXD no longer uses NoCloud, so we need to set - # it up manually - lxd_image_metadata = subp( - ["lxc", "config", "metadata", "show", instance.name] - ) - if "/var/lib/cloud/seed/nocloud-net" in lxd_image_metadata.stdout: - return - subp( - ["lxc", "config", "template", "create", instance.name, "emptycfg.tpl"], - ) - subp( - ["lxc", "config", "template", "edit", instance.name, "emptycfg.tpl"], - data="#cloud-config\n{}\n", - ) - subp( - ["lxc", "config", "metadata", "edit", instance.name], - data=f"{lxd_image_metadata.stdout}{LXD_METADATA_NOCLOUD_SEED}", - ) - - -@pytest.mark.lxd_setup.with_args(setup_nocloud) -@pytest.mark.lxd_use_exec -@pytest.mark.skipif( - PLATFORM != "lxd_container", - reason="Requires NoCloud with custom setup", -) -def test_nocloud_seedfrom_vendordata(client: IntegrationInstance): - """Integration test for #570. - - Test that we can add optional vendor-data to the seedfrom file in a - NoCloud environment - """ - seed_dir = "/var/tmp/test_seed_dir" - result = client.execute( - "mkdir {seed_dir} && " - "touch {seed_dir}/user-data && " - "touch {seed_dir}/meta-data && " - "echo 'seedfrom: {seed_dir}/' > " - "/var/lib/cloud/seed/nocloud-net/meta-data".format(seed_dir=seed_dir) - ) - assert result.return_code == 0 - - client.write_to_file( - "{}/vendor-data".format(seed_dir), - VENDOR_DATA, - ) - client.execute("cloud-init clean --logs") - client.restart() - assert client.execute("cloud-init status").ok - assert "seeded_vendordata_test_file" in client.execute("ls /var/tmp") - assert network_wait_logged(client.execute("cat /var/log/cloud-init.log")) - - -SMBIOS_USERDATA = """\ -#cloud-config -runcmd: - - touch /var/tmp/smbios_test_file -""" -SMBIOS_SEED_DIR = "/smbios_seed" - - -def setup_nocloud_local_serial(instance: LXDInstance): - subp( - [ - "lxc", - "config", - "set", - instance.name, - "raw.qemu=-smbios " - f"type=1,serial=ds=nocloud;s=file://{SMBIOS_SEED_DIR};h=myhost", - ] - ) - - -def setup_nocloud_network_serial(instance: LXDInstance): - subp( - [ - "lxc", - "config", - "set", - instance.name, - "raw.qemu=-smbios " - "type=1,serial=ds=nocloud-net;s=http://0.0.0.0/;h=myhost", - ] - ) - - -@pytest.mark.lxd_use_exec -@pytest.mark.skipif( - PLATFORM != "lxd_vm", - reason="Requires NoCloud with raw QEMU serial setup", -) -class TestSmbios: - @pytest.mark.lxd_setup.with_args(setup_nocloud_local_serial) - def test_smbios_seed_local(self, client: IntegrationInstance): - """Check that smbios seeds that use local disk work""" - assert client.execute(f"mkdir -p {SMBIOS_SEED_DIR}").ok - client.write_to_file(f"{SMBIOS_SEED_DIR}/user-data", SMBIOS_USERDATA) - client.write_to_file(f"{SMBIOS_SEED_DIR}/meta-data", "") - client.write_to_file(f"{SMBIOS_SEED_DIR}/vendor-data", "") - assert client.execute("cloud-init clean --logs").ok - client.restart() - assert client.execute("test -f /var/tmp/smbios_test_file").ok - - @pytest.mark.lxd_setup.with_args(setup_nocloud_network_serial) - def test_smbios_seed_network(self, client: IntegrationInstance): - """Check that smbios seeds that use network (http/https) work""" - service_file = "/lib/systemd/system/local-server.service" - client.write_to_file( - service_file, - dedent( - """\ - [Unit] - Description=Serve a local webserver - Before=cloud-init-network.service - Wants=cloud-init-local.service - DefaultDependencies=no - After=systemd-networkd-wait-online.service - After=networking.service - - - [Install] - WantedBy=cloud-init.target - - [Service] - """ - f"WorkingDirectory={SMBIOS_SEED_DIR}" - """ - ExecStart=/usr/bin/env python3 -m http.server --bind 0.0.0.0 80 - """ - ), - ) - assert client.execute( - "chmod 644 /lib/systemd/system/local-server.service" - ).ok - assert client.execute("systemctl enable local-server.service").ok - client.write_to_file( - "/etc/cloud/cloud.cfg.d/91_do_not_use_lxd.cfg", - "datasource_list: [ NoCloud, None ]\n", - ) - assert client.execute(f"mkdir -p {SMBIOS_SEED_DIR}").ok - client.write_to_file(f"{SMBIOS_SEED_DIR}/user-data", SMBIOS_USERDATA) - client.write_to_file(f"{SMBIOS_SEED_DIR}/meta-data", "") - client.write_to_file(f"{SMBIOS_SEED_DIR}/vendor-data", "") - assert client.execute("cloud-init clean --logs").ok - client.restart() - assert client.execute("test -f /var/tmp/smbios_test_file").ok - version_boundary = get_feature_flag_value( - client, "DEPRECATION_INFO_BOUNDARY" - ) - message = "The 'nocloud-net' datasource name is deprecated" - # nocloud-net deprecated in version 24.1 - if lifecycle.should_log_deprecation("24.1", version_boundary): - verify_clean_boot(client, require_deprecations=[message]) - else: - client.execute( - rf"grep \"INFO]: {message}\" /var/log/cloud-init.log" - ).ok - - -@pytest.mark.skipif(PLATFORM != "lxd_vm", reason="Modifies grub config") -@pytest.mark.lxd_use_exec -class TestFTP: - """Test nocloud's support for unencrypted FTP and FTP over TLS (ftps). - - These tests work by setting up a local ftp server on the test instance - and then rebooting the instance clean (cloud-init clean --logs --reboot). - - Check for the existence (or non-existence) of specific log messages to - verify functionality. - """ - - # should we really be surfacing this netplan stderr as a warning? - # i.e. how does it affect the users? - expected_warnings = [ - "Falling back to a hard restart of systemd-networkd.service" - ] - - @staticmethod - def _boot_with_cmdline( - cmdline: str, client: IntegrationInstance, encrypted: bool = False - ) -> None: - """configure an ftp server to start prior to network timeframe - optionally install certs and make the server support only FTP over TLS - - cmdline: a string containing the kernel command line set on reboot - client: an instance to configure - encrypted: a boolean which modifies the configured ftp server - """ - - # install the essential bits - assert client.execute( - "apt update && apt install -yq python3-pyftpdlib " - "python3-openssl ca-certificates libnss3-tools" - ).ok - - # How do you reliably run a ftp server for your instance to - # read files from during early boot? In typical production - # environments, the ftp server would be separate from the instance. - # - # For a reliable server that fits with the framework of running tests - # on a single instance, it is easier to just install an ftp server - # that runs on the second boot prior to the cloud-init unit which - # reaches out to the ftp server. This achieves reaching out to an - # ftp(s) server for testing - cloud-init just doesn't have to reach - # very far to get what it needs. - # - # DO NOT use these concepts in a production. - # - # This configuration is neither secure nor production-grade - intended - # only for testing purposes. - client.write_to_file( - "/server.py", - dedent( - """\ - #!/usr/bin/python3 - import logging - - from systemd.daemon import notify - - from pyftpdlib.authorizers import DummyAuthorizer - from pyftpdlib.handlers import FTPHandler, TLS_FTPHandler - from pyftpdlib.servers import FTPServer - from pyftpdlib.filesystems import UnixFilesystem - - encrypted = """ - + str(encrypted) - + """ - - logging.basicConfig(level=logging.DEBUG) - - # yeah, it's not secure but that's not the point - authorizer = DummyAuthorizer() - - # Define a read-only anonymous user - authorizer.add_anonymous("/home/anonymous") - - # Instantiate FTP handler class - if not encrypted: - handler = FTPHandler - logging.info("Running unencrypted ftp server") - else: - handler = TLS_FTPHandler - handler.certfile = "/cert.pem" - handler.keyfile = "/key.pem" - logging.info("Running encrypted ftp server") - - handler.authorizer = authorizer - handler.abstracted_fs = UnixFilesystem - server = FTPServer(("localhost", 2121), handler) - - # tell systemd to proceed - notify("READY=1") - - # start the ftp server - server.serve_forever() - """ - ), - ) - assert client.execute("chmod +x /server.py").ok - - if encrypted: - if CURRENT_RELEASE > FOCAL: - assert client.execute("apt install -yq mkcert").ok - else: - - # install golang - assert client.execute("apt install -yq golang").ok - - # build mkcert from source - # - # we could check out a tag, but the project hasn't - # been updated in 2 years - # - # instructions from https://github.com/FiloSottile/mkcert - assert client.execute( - "git clone https://github.com/FiloSottile/mkcert && " - "cd mkcert && " - "export latest_ver=$(git describe --tags --abbrev=0) && " - 'wget "https://github.com/FiloSottile/mkcert/releases/' - "download/${latest_ver}/mkcert-" - '${latest_ver}-linux-amd64"' - " -O mkcert && " - "chmod 755 mkcert" - ).ok - - # giddyup - assert client.execute( - "ln -s $HOME/mkcert/mkcert /usr/local/bin/mkcert" - ).ok - - # more palatable than openssl commands - assert client.execute( - "mkcert -install -cert-file /cert.pem -key-file /key.pem " - "localhost 127.0.0.1 0.0.0.0 ::1" - ).ok - - client.write_to_file( - "/lib/systemd/system/local-ftp.service", - dedent( - """\ - [Unit] - Description=TESTING USE ONLY ftp server - Wants=cloud-init-local.service - DefaultDependencies=no - - # we want the network up for network operations - # and NoCloud operates in network timeframe - After=systemd-networkd-wait-online.service - After=networking.service - Before=cloud-init-network.service - Before=cloud-init.service - - [Service] - Type=notify - ExecStart=/server.py - - [Install] - WantedBy=cloud-init.target - """ - ), - ) - assert client.execute( - "chmod 644 /lib/systemd/system/local-ftp.service" - ).ok - assert client.execute("systemctl enable local-ftp.service").ok - assert client.execute("mkdir /home/anonymous").ok - - client.write_to_file( - "/user-data", - dedent( - """\ - #cloud-config - - hostname: ftp-bootstrapper - """ - ), - ) - client.write_to_file( - "/meta-data", - dedent( - """\ - instance-id: ftp-instance - """ - ), - ) - client.write_to_file("/vendor-data", "") - - # set the kernel command line, reboot with it - override_kernel_command_line(cmdline, client) - - def test_nocloud_ftp_unencrypted_server_succeeds( - self, client: IntegrationInstance - ): - """check that ftp:// succeeds to unencrypted ftp server - - this mode allows administrators to choose unencrypted ftp, - at their own risk - """ - cmdline = "ds=nocloud;seedfrom=ftp://0.0.0.0:2121" - self._boot_with_cmdline(cmdline, client) - verify_clean_boot(client, ignore_warnings=self.expected_warnings) - assert "ftp-bootstrapper" == client.execute("hostname").rstrip() - verify_clean_log(client.execute("cat /var/log/cloud-init.log").stdout) - - def test_nocloud_ftps_unencrypted_server_fails( - self, client: IntegrationInstance - ): - """check that ftps:// fails to unencrypted ftp server - - this mode allows administrators to enforce TLS encryption - """ - cmdline = "ds=nocloud;seedfrom=ftps://localhost:2121" - self._boot_with_cmdline(cmdline, client) - verify_clean_boot( - client, - ignore_warnings=self.expected_warnings, - require_warnings=[ - "Getting data from failed", - "Used fallback datasource", - "Attempted to connect to an insecure ftp server but used" - " a scheme of ftps://, which is not allowed. Use ftp:// " - "to allow connecting to insecure ftp servers.", - ], - ignore_tracebacks=[ - 'ftplib.error_perm: 500 Command "AUTH" not understood.', - "UrlError: Attempted to connect to an insecure ftp server", - ], - ) - - def test_nocloud_ftps_encrypted_server_succeeds( - self, client: IntegrationInstance - ): - """check that ftps:// encrypted ftp server succeeds - - this mode allows administrators to enforce TLS encryption - """ - cmdline = "ds=nocloud;seedfrom=ftps://localhost:2121" - self._boot_with_cmdline(cmdline, client, encrypted=True) - verify_clean_boot(client, ignore_warnings=self.expected_warnings) - assert "ftp-bootstrapper" == client.execute("hostname").rstrip() - verify_clean_log(client.execute("cat /var/log/cloud-init.log").stdout) - - def test_nocloud_ftp_encrypted_server_fails( - self, client: IntegrationInstance - ): - """check that using ftp:// to encrypted ftp server fails""" - cmdline = "ds=nocloud;seedfrom=ftp://0.0.0.0:2121" - self._boot_with_cmdline(cmdline, client, encrypted=True) - verify_clean_boot(client, ignore_warnings=self.expected_warnings) diff --git a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/integration_tests/modules/test_boothook.py b/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/integration_tests/modules/test_boothook.py deleted file mode 100644 index 55ca1ca6..00000000 --- a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/integration_tests/modules/test_boothook.py +++ /dev/null @@ -1,60 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. -import re - -import pytest - -from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.util import ( - network_wait_logged, - verify_clean_boot, - verify_clean_log, -) - -USER_DATA = """\ -## template: jinja -#cloud-boothook -#!/bin/sh -# Error below will generate stderr -BOOTHOOK/0 -echo BOOTHOOKstdout -echo "BOOTHOOK: {{ v1.instance_id }}: is called every boot." >> /boothook.txt -""" - - -@pytest.mark.user_data(USER_DATA) -class TestBoothook: - def test_boothook_header_runs_part_per_instance( - self, - class_client: IntegrationInstance, - ): - """Test boothook handling creates a script that runs per-boot. - Streams stderr and stdout are directed to - /var/log/cloud-init-output.log. - """ - client = class_client - instance_id = client.instance.execute("cloud-init query instance-id") - RE_BOOTHOOK = f"BOOTHOOK: {instance_id}: is called every boot" - log = client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log) - verify_clean_boot(client) - output = client.read_from_file("/boothook.txt") - assert 1 == len(re.findall(RE_BOOTHOOK, output)) - client.restart() - output = client.read_from_file("/boothook.txt") - assert 2 == len(re.findall(RE_BOOTHOOK, output)) - output_log = client.read_from_file("/var/log/cloud-init-output.log") - expected_msgs = [ - "BOOTHOOKstdout", - "boothooks/part-001: 3: BOOTHOOK/0: not found", - ] - for msg in expected_msgs: - assert msg in output_log - - def test_boothook_waits_for_network( - self, class_client: IntegrationInstance - ): - """Test boothook handling waits for network before running.""" - client = class_client - assert network_wait_logged( - client.read_from_file("/var/log/cloud-init.log") - ) diff --git a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/unittests/cmd/test_main.py b/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/unittests/cmd/test_main.py deleted file mode 100644 index 4df78718..00000000 --- a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/unittests/cmd/test_main.py +++ /dev/null @@ -1,396 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -import copy -import getpass -import os -import textwrap -from collections import namedtuple -from unittest import mock - -import pytest - -from cloudinit import safeyaml, util -from cloudinit.cmd import main -from cloudinit.util import ensure_dir, load_text_file, write_file - -MyArgs = namedtuple( - "MyArgs", "debug files force local reporter subcommand skip_log_setup" -) - - -CLOUD_CONFIG_ARCHIVE = """\ -#cloud-config-archive -- type: "text/cloud-boothook" - content: | - #!/bin/sh - echo "this is from a boothook." > /var/tmp/boothook.txt -- type: "text/cloud-config" - content: | - bootcmd: - - echo "this is from a cloud-config." > /var/tmp/bootcmd.txt -""" - - -EXTRA_CLOUD_CONFIG = """\ -#cloud-config -write_files -- path: {tmpdir}/etc/blah.ini - content: override -""" - - -class TestMain: - @pytest.fixture(autouse=True) - def common_mocks(self, mocker): - mocker.patch("cloudinit.cmd.main.os.getppid", return_value=42) - mocker.patch("cloudinit.cmd.main.close_stdin") - mocker.patch( - "cloudinit.cmd.main.netinfo.debug_info", - return_value="my net debug info", - ) - mocker.patch( - "cloudinit.cmd.main.util.fixup_output", - return_value=("outfmt", "errfmt"), - ) - mocker.patch("cloudinit.cmd.main.util.get_cmdline", return_value="") - mocker.patch("cloudinit.cmd.main.util.uptime", return_value="12345") - os.environ["_CLOUD_INIT_SAVE_STDOUT"] = "true" - yield - os.environ.pop("_CLOUD_INIT_SAVE_STDOUT") - - @pytest.fixture - def cloud_cfg(self, mocker, tmpdir, fake_filesystem): - cloud_dir = os.path.join(tmpdir, "var/lib/cloud/") - log_dir = os.path.join(tmpdir, "var/log/") - ensure_dir(cloud_dir) - ensure_dir(os.path.join(tmpdir, "etc/cloud")) - ensure_dir(log_dir) - cloud_cfg_file = os.path.join(tmpdir, "etc/cloud/cloud.cfg") - - cfg = { - "datasource_list": ["None"], - # "def_log_file": os.path.join(log_dir, "cloud-init.log"), - "def_log_file": "", - "runcmd": ["ls /etc"], # test ALL_DISTROS - "system_info": { - "paths": { - "cloud_dir": cloud_dir, - "run_dir": str(tmpdir), - } - }, - "write_files": [ - { - "path": os.path.join(tmpdir, "etc/blah.ini"), - "content": "blah", - "permissions": 0o755, - "owner": getpass.getuser(), - }, - ], - "cloud_init_modules": ["write_files", "runcmd"], - } - write_file(cloud_cfg_file, safeyaml.dumps(cfg)) - yield copy.deepcopy(cfg), cloud_cfg_file - - @pytest.mark.parametrize( - "provide_file_arg,expected_file_content", - ( - pytest.param(False, "blah", id="write_files_from_base_config"), - pytest.param( - True, - "override", - id="write_files_from_supplemental_file_arg", - ), - ), - ) - def test_main_init_run_net_runs_modules( - self, - provide_file_arg, - expected_file_content, - cloud_cfg, - capsys, - tmpdir, - ): - """Modules like write_files are run in 'net' mode.""" - if provide_file_arg: - supplemental_config_file = tmpdir.join("custom.yaml") - supplemental_config_file.write( - EXTRA_CLOUD_CONFIG.format(tmpdir=tmpdir) - ) - files = [open(supplemental_config_file)] - else: - files = None - cmdargs = MyArgs( - debug=False, - files=files, - force=False, - local=False, - reporter=None, - subcommand="init", - skip_log_setup=False, - ) - _ds, msg = main.main_init("init", cmdargs) - assert msg == [] - # Instancify is called - instance_id_path = "var/lib/cloud/data/instance-id" - assert "iid-datasource-none\n" == os.path.join( - load_text_file(os.path.join(tmpdir, instance_id_path)) - ) - # modules are run (including write_files) - assert "blah" == load_text_file(os.path.join(tmpdir, "etc/blah.ini")) - expected_logs = [ - "network config is disabled by fallback", # apply_network_config - "my net debug info", # netinfo.debug_info - "PID [42] started cloud-init 'init'", - ] - stderr = capsys.readouterr().err - for log in expected_logs: - assert log in stderr - - def test_main_init_run_net_calls_set_hostname_when_metadata_present( - self, cloud_cfg, mocker - ): - """When local-hostname metadata is present, call cc_set_hostname.""" - cfg, cloud_cfg_file = cloud_cfg - cfg["datasource"] = { - "None": {"metadata": {"local-hostname": "md-hostname"}} - } - write_file(cloud_cfg_file, safeyaml.dumps(cfg)) - cmdargs = MyArgs( - debug=False, - files=None, - force=False, - local=False, - reporter=None, - subcommand="init", - skip_log_setup=False, - ) - - def set_hostname(name, cfg, cloud, args): - assert "set_hostname" == name - - m_hostname = mocker.patch( - "cloudinit.cmd.main.cc_set_hostname.handle", - side_effect=set_hostname, - ) - main.main_init("init", cmdargs) - - m_hostname.assert_called_once() - - @mock.patch("cloudinit.cmd.clean.get_parser") - @mock.patch("cloudinit.cmd.clean.handle_clean_args") - @mock.patch("cloudinit.log.loggers.configure_root_logger") - def test_main_sys_argv( - self, - _m_configure_root_logger, - _m_handle_clean_args, - m_clean_get_parser, - ): - with mock.patch("sys.argv", ["cloudinit", "--debug", "clean"]): - main.main() - m_clean_get_parser.assert_called_once() - - @pytest.mark.parametrize( - "ds,userdata,expected", - [ - # If we have no datasource, wait regardless - (None, None, True), - (None, "#!/bin/bash\n - echo hello", True), - # Empty user data shouldn't wait - (mock.Mock(), "", False), - # Bootcmd always wait - (mock.Mock(), "#cloud-config\nbootcmd:\n - echo hello", True), - # Bytes are valid too - (mock.Mock(), b"#cloud-config\nbootcmd:\n - echo hello", True), - # write_files with source uri wait - ( - mock.Mock(), - textwrap.dedent( - """\ - #cloud-config - write_files: - - source: - uri: http://example.com - headers: - Authorization: Basic stuff - User-Agent: me - """ - ), - True, - ), - # write_files with source file don't wait - ( - mock.Mock(), - textwrap.dedent( - """\ - #cloud-config - write_files: - - source: - uri: /tmp/hi - headers: - Authorization: Basic stuff - User-Agent: me - """ - ), - False, - ), - # write_files without 'source' don't wait - ( - mock.Mock(), - textwrap.dedent( - """\ - #cloud-config - write_files: - - content: hello - encoding: b64 - owner: root:root - path: /etc/sysconfig/selinux - permissions: '0644' - """ - ), - False, - ), - # random_seed with 'command' wait - ( - mock.Mock(), - "#cloud-config\nrandom_seed:\n command: true", - True, - ), - # random_seed without 'command' no wait - ( - mock.Mock(), - textwrap.dedent( - """\ - #cloud-config - random_seed: - data: 4 - encoding: raw - file: /dev/urandom - """ - ), - False, - ), - # mounts always wait - ( - mock.Mock(), - "#cloud-config\nmounts:\n - [ /dev/sdb, /mnt, ext4 ]", - True, - ), - # Not parseable as yaml - (mock.Mock(), "#cloud-config\nbootcmd:\necho hello", True), - # Yaml that parses to list - (mock.Mock(), CLOUD_CONFIG_ARCHIVE, True), - # Non-cloud-config - (mock.Mock(), "#!/bin/bash\n - echo hello", True), - # Something that after processing won't decode to utf-8 - (mock.Mock(), "RANDOM100", True), - # Something small that after processing won't decode to utf-8 - (mock.Mock(), "RANDOM5", True), - ], - ) - def test_should_wait_on_network(self, ds, userdata, expected): - # pytest-xdist doesn't like randomness - # https://github.com/pytest-dev/pytest-xdist/issues/432 - # So work around it with a super stupid hack - if userdata == "RANDOM100": - userdata = os.urandom(100) - elif userdata == "RANDOM5": - userdata = os.urandom(5) - - if ds: - ds.get_userdata_raw = mock.Mock(return_value=userdata) - ds.get_vendordata_raw = mock.Mock(return_value=None) - ds.get_vendordata2_raw = mock.Mock(return_value=None) - assert main._should_wait_on_network(ds)[0] is expected - - # Here we rotate our configs to ensure that any of userdata, - # vendordata, or vendordata2 can be the one that causes us to wait. - for _ in range(2): - if ds: - ( - ds.get_userdata_raw, - ds.get_vendordata_raw, - ds.get_vendordata2_raw, - ) = ( - ds.get_vendordata_raw, - ds.get_vendordata2_raw, - ds.get_userdata_raw, - ) - assert main._should_wait_on_network(ds)[0] is expected - - @pytest.mark.parametrize( - "distro,should_wait,expected_add_wait", - [ - ("ubuntu", True, True), - ("ubuntu", False, False), - ("debian", True, False), - ("debian", False, False), - ("centos", True, False), - ("rhel", False, False), - ("fedora", True, False), - ("suse", False, False), - ("gentoo", True, False), - ("arch", False, False), - ("alpine", False, False), - ], - ) - def test_distro_wait_for_network( - self, - distro, - should_wait, - expected_add_wait, - cloud_cfg, - mocker, - fake_filesystem, - ): - mocker.patch("cloudinit.net.netplan.available", return_value=True) - m_nm = mocker.patch( - "cloudinit.net.network_manager.available", return_value=False - ) - m_subp = mocker.patch("cloudinit.subp.subp", return_value=("", "")) - if not should_wait: - util.write_file(".skip-network", "") - - cfg, cloud_cfg_file = cloud_cfg - cfg["system_info"]["distro"] = distro - write_file(cloud_cfg_file, safeyaml.dumps(cfg)) - cmdargs = MyArgs( - debug=False, - files=None, - force=False, - local=False, - reporter=None, - subcommand="init", - skip_log_setup=False, - ) - main.main_init("init", cmdargs) - if expected_add_wait: - m_nm.assert_called_once() - m_subp.assert_called_with( - ["systemctl", "start", "systemd-networkd-wait-online.service"] - ) - else: - m_nm.assert_not_called() - m_subp.assert_not_called() - - -class TestShouldBringUpInterfaces: - @pytest.mark.parametrize( - "cfg_disable,args_local,expected", - [ - (True, True, False), - (True, False, False), - (False, True, False), - (False, False, True), - ], - ) - def test_should_bring_up_interfaces( - self, cfg_disable, args_local, expected - ): - init = mock.Mock() - init.cfg = {"disable_network_activation": cfg_disable} - - args = mock.Mock() - args.local = args_local - - result = main._should_bring_up_interfaces(init, args) - assert result == expected diff --git a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/unittests/test_data.py b/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/unittests/test_data.py deleted file mode 100644 index fa1aedf7..00000000 --- a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/unittests/test_data.py +++ /dev/null @@ -1,914 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Tests for handling of userdata within cloud init.""" - -import gzip -import logging -import os -from email import encoders -from email.mime.application import MIMEApplication -from email.mime.base import MIMEBase -from email.mime.multipart import MIMEMultipart -from io import BytesIO -from pathlib import Path -from unittest import mock - -import pytest -import responses - -from cloudinit import handlers -from cloudinit import helpers as c_helpers -from cloudinit import safeyaml, stages -from cloudinit import user_data as ud -from cloudinit import util -from cloudinit.config.modules import Modules -from cloudinit.settings import DEFAULT_RUN_DIR, PER_INSTANCE -from tests.unittests import helpers -from tests.unittests.util import FakeDataSource - -MPATH = "cloudinit.stages" - - -def count_messages(root): - am = 0 - for m in root.walk(): - if ud.is_skippable(m): - continue - am += 1 - return am - - -def gzip_text(text): - contents = BytesIO() - f = gzip.GzipFile(fileobj=contents, mode="wb") - f.write(util.encode_text(text)) - f.flush() - f.close() - return contents.getvalue() - - -@pytest.fixture(scope="function") -def init_tmp(request, tmpdir): - ci = stages.Init() - cloud_dir = tmpdir.join("cloud") - cloud_dir.mkdir() - run_dir = tmpdir.join("run") - run_dir.mkdir() - ci._cfg = { - "system_info": { - "default_user": {"name": "ubuntu"}, - "distro": "ubuntu", - "paths": { - "cloud_dir": cloud_dir.strpath, - "run_dir": run_dir.strpath, - }, - } - } - run_dir.join("instance-data-sensitive.json").write("{}") - return ci - - -class TestConsumeUserData: - def test_simple_jsonp(self, init_tmp): - user_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/bar", "value": "qux2" } -] -""" - init_tmp.datasource = FakeDataSource(user_blob) - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - cc_contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - cc = util.load_yaml(cc_contents) - assert len(cc) == 2 - assert cc["baz"] == "qux" - assert cc["bar"] == "qux2" - - @pytest.mark.usefixtures("fake_filesystem") - def test_simple_jsonp_vendor_and_vendor2_and_user(self): - # test that user-data wins over vendor - user_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/bar", "value": "qux2" }, - { "op": "add", "path": "/foobar", "value": "qux3" } -] -""" - vendor_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "quxA" }, - { "op": "add", "path": "/bar", "value": "quxB" }, - { "op": "add", "path": "/foo", "value": "quxC" }, - { "op": "add", "path": "/corge", "value": "quxEE" } -] -""" - vendor2_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/corge", "value": "quxD" }, - { "op": "add", "path": "/grault", "value": "quxFF" }, - { "op": "add", "path": "/foobar", "value": "quxGG" } -] -""" - initer = stages.Init() - initer.datasource = FakeDataSource( - user_blob, vendordata=vendor_blob, vendordata2=vendor2_blob - ) - initer.read_cfg() - initer.initialize() - initer.fetch() - initer.instancify() - with mock.patch( - "cloudinit.util.read_conf_from_cmdline", return_value={} - ): - initer.update() - initer.cloudify().run( - "consume_data", - initer.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - mods = Modules(initer) - (_which_ran, _failures) = mods.run_section("cloud_init_modules") - cfg = mods.cfg - assert "vendor_data" in cfg - assert "vendor_data2" in cfg - # Confirm that vendordata2 overrides vendordata, and that - # userdata overrides both - assert cfg["baz"] == "qux" - assert cfg["bar"] == "qux2" - assert cfg["foobar"] == "qux3" - assert cfg["foo"] == "quxC" - assert cfg["corge"] == "quxD" - assert cfg["grault"] == "quxFF" - - @pytest.mark.usefixtures("fake_filesystem") - def test_simple_jsonp_no_vendor_consumed(self): - # make sure that vendor data is not consumed - user_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/bar", "value": "qux2" }, - { "op": "add", "path": "/vendor_data", "value": {"enabled": "false"}} -] -""" - vendor_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "quxA" }, - { "op": "add", "path": "/bar", "value": "quxB" }, - { "op": "add", "path": "/foo", "value": "quxC" } -] -""" - initer = stages.Init() - initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) - initer.read_cfg() - initer.initialize() - initer.fetch() - initer.instancify() - initer.update() - initer.cloudify().run( - "consume_data", - initer.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - mods = Modules(initer) - (_which_ran, _failures) = mods.run_section("cloud_init_modules") - cfg = mods.cfg - assert cfg["baz"] == "qux" - assert cfg["bar"] == "qux2" - assert "foo" not in cfg - - def test_mixed_cloud_config(self, init_tmp): - blob_cc = """ -#cloud-config -a: b -c: d -""" - message_cc = MIMEBase("text", "cloud-config") - message_cc.set_payload(blob_cc) - - blob_jp = """ -#cloud-config-jsonp -[ - { "op": "replace", "path": "/a", "value": "c" }, - { "op": "remove", "path": "/c" } -] -""" - - message_jp = MIMEBase("text", "cloud-config-jsonp") - message_jp.set_payload(blob_jp) - - message = MIMEMultipart() - message.attach(message_cc) - message.attach(message_jp) - - init_tmp.datasource = FakeDataSource(str(message)) - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - cc_contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - cc = util.load_yaml(cc_contents) - assert len(cc) == 1 - assert cc["a"] == "c" - - def test_cloud_config_as_x_shell_script(self, init_tmp): - blob_cc = """ -#cloud-config -a: b -c: d -""" - message_cc = MIMEBase("text", "x-shellscript") - message_cc.set_payload(blob_cc) - - blob_jp = """ -#cloud-config-jsonp -[ - { "op": "replace", "path": "/a", "value": "c" }, - { "op": "remove", "path": "/c" } -] -""" - - message_jp = MIMEBase("text", "cloud-config-jsonp") - message_jp.set_payload(blob_jp) - - message = MIMEMultipart() - message.attach(message_cc) - message.attach(message_jp) - - init_tmp.datasource = FakeDataSource(str(message)) - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - cc_contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - cc = util.load_yaml(cc_contents) - assert len(cc) == 1 - assert cc["a"] == "c" - - @pytest.mark.usefixtures("fake_filesystem") - def test_vendor_user_yaml_cloud_config(self): - vendor_blob = """ -#cloud-config -a: b -name: vendor -run: - - x - - y -""" - - user_blob = """ -#cloud-config -a: c -vendor_data: - enabled: true - prefix: /bin/true -name: user -run: - - z -""" - initer = stages.Init() - initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) - initer.read_cfg() - initer.initialize() - initer.fetch() - initer.instancify() - initer.update() - initer.cloudify().run( - "consume_data", - initer.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - mods = Modules(initer) - (_which_ran, _failures) = mods.run_section("cloud_init_modules") - cfg = mods.cfg - assert "vendor_data" in cfg - assert cfg["a"] == "c" - assert cfg["name"] == "user" - assert "x" not in cfg["run"] - assert "y" not in cfg["run"] - assert "z" in cfg["run"] - - @pytest.mark.usefixtures("fake_filesystem") - def test_vendordata_script(self): - vendor_blob = """ -#!/bin/bash -echo "test" -""" - vendor2_blob = """ -#!/bin/bash -echo "dynamic test" -""" - - user_blob = """ -#cloud-config -vendor_data: - enabled: true - prefix: /bin/true -""" - initer = stages.Init() - initer.datasource = FakeDataSource( - user_blob, vendordata=vendor_blob, vendordata2=vendor2_blob - ) - initer.read_cfg() - initer.initialize() - initer.fetch() - initer.instancify() - initer.update() - initer.cloudify().run( - "consume_data", - initer.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - mods = Modules(initer) - (_which_ran, _failures) = mods.run_section("cloud_init_modules") - vendor_script = initer.paths.get_ipath_cur("vendor_scripts") - vendor_script_fns = "%s/part-001" % vendor_script - assert os.path.exists(vendor_script_fns) is True - - def test_merging_cloud_config(self, tmpdir): - blob = """ -#cloud-config -a: b -e: f -run: - - b - - c -""" - message1 = MIMEBase("text", "cloud-config") - message1.set_payload(blob) - - blob2 = """ -#cloud-config -a: e -e: g -run: - - stuff - - morestuff -""" - message2 = MIMEBase("text", "cloud-config") - message2["X-Merge-Type"] = ( - "dict(recurse_array,recurse_str)+list(append)+str(append)" - ) - message2.set_payload(blob2) - - blob3 = """ -#cloud-config -e: - - 1 - - 2 - - 3 -p: 1 -""" - message3 = MIMEBase("text", "cloud-config") - message3.set_payload(blob3) - - messages = [message1, message2, message3] - - paths = c_helpers.Paths( - {"cloud_dir": tmpdir, "run_dir": tmpdir}, ds=FakeDataSource("") - ) - cloud_cfg = handlers.cloud_config.CloudConfigPartHandler(paths) - - cloud_cfg.handle_part( - None, handlers.CONTENT_START, None, None, None, None - ) - for i, m in enumerate(messages): - headers = dict(m) - fn = "part-%s" % (i + 1) - payload = m.get_payload(decode=True) - cloud_cfg.handle_part( - None, headers["Content-Type"], fn, payload, None, headers - ) - cloud_cfg.handle_part( - None, handlers.CONTENT_END, None, None, None, None - ) - contents = util.load_text_file(paths.get_ipath("cloud_config")) - contents = util.load_yaml(contents) - assert contents["run"], ["b", "c", "stuff", "morestuff"] - assert contents["a"] == "be" - assert contents["e"] == [1, 2, 3] - assert contents["p"] == 1 - - def test_unhandled_type_warning(self, init_tmp, caplog): - """Raw text without magic is ignored but shows warning.""" - data = "arbitrary text\n" - init_tmp.datasource = FakeDataSource(data) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert ( - "Unhandled non-multipart (text/x-not-multipart) userdata:" - in caplog.text - ) - mockobj.assert_called_once_with( - init_tmp.paths.get_ipath("cloud_config"), "", 0o600 - ) - - def test_mime_gzip_compressed(self, init_tmp): - """Tests that individual message gzip encoding works.""" - - def gzip_part(text): - return MIMEApplication(gzip_text(text), "gzip") - - base_content1 = """ -#cloud-config -a: 2 -""" - - base_content2 = """ -#cloud-config -b: 3 -c: 4 -""" - - message = MIMEMultipart("test") - message.attach(gzip_part(base_content1)) - message.attach(gzip_part(base_content2)) - init_tmp.datasource = FakeDataSource(str(message)) - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - contents = util.load_yaml(contents) - assert isinstance(contents, dict) is True - assert len(contents) == 3 - assert contents["a"] == 2 - assert contents["b"] == 3 - assert contents["c"] == 4 - - def test_mime_text_plain(self, init_tmp, caplog): - """Mime message of type text/plain is ignored but shows warning.""" - message = MIMEBase("text", "plain") - message.set_payload("Just text") - init_tmp.datasource = FakeDataSource(message.as_string().encode()) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert "Unhandled unknown content-type (text/plain)" in caplog.text - mockobj.assert_called_once_with( - init_tmp.paths.get_ipath("cloud_config"), "", 0o600 - ) - - # Since features are intended to be overridden downstream, mock them - # all here so new feature flags don't require a new change to this - # unit test. - @mock.patch.multiple( - "cloudinit.features", - ERROR_ON_USER_DATA_FAILURE=True, - ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES=True, - EXPIRE_APPLIES_TO_HASHED_USERS=False, - NETPLAN_CONFIG_ROOT_READ_ONLY=True, - DEPRECATION_INFO_BOUNDARY="devel", - NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH=False, - APT_DEB822_SOURCE_LIST_FILE=True, - ) - def test_shellscript(self, init_tmp, tmpdir, caplog): - """Raw text starting #!/bin/sh is treated as script.""" - script = "#!/bin/sh\necho hello\n" - init_tmp.datasource = FakeDataSource(script) - - outpath = os.path.join( - init_tmp.paths.get_ipath_cur("scripts"), "part-001" - ) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert caplog.records == [] # No warnings - - mockobj.assert_has_calls( - [ - mock.call(outpath, script, 0o700), - mock.call(init_tmp.paths.get_ipath("cloud_config"), "", 0o600), - ] - ) - expected = { - "features": { - "ERROR_ON_USER_DATA_FAILURE": True, - "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES": True, - "EXPIRE_APPLIES_TO_HASHED_USERS": False, - "NETPLAN_CONFIG_ROOT_READ_ONLY": True, - "DEPRECATION_INFO_BOUNDARY": "devel", - "NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH": False, - "APT_DEB822_SOURCE_LIST_FILE": True, - }, - "system_info": { - "default_user": {"name": "ubuntu"}, - "distro": "ubuntu", - "paths": { - "cloud_dir": tmpdir.join("cloud").strpath, - "run_dir": tmpdir.join("run").strpath, - }, - }, - } - - loaded_json = util.load_json( - util.load_text_file( - init_tmp.paths.get_runpath("instance_data_sensitive") - ) - ) - assert expected == loaded_json - - expected["_doc"] = stages.COMBINED_CLOUD_CONFIG_DOC - assert expected == util.load_json( - util.load_text_file( - init_tmp.paths.get_runpath("combined_cloud_config") - ) - ) - - def test_mime_text_x_shellscript(self, init_tmp, caplog): - """Mime message of type text/x-shellscript is treated as script.""" - script = "#!/bin/sh\necho hello\n" - message = MIMEBase("text", "x-shellscript") - message.set_payload(script) - init_tmp.datasource = FakeDataSource(message.as_string()) - - outpath = os.path.join( - init_tmp.paths.get_ipath_cur("scripts"), "part-001" - ) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert caplog.records == [] # No warnings - - mockobj.assert_has_calls( - [ - mock.call(outpath, script, 0o700), - mock.call(init_tmp.paths.get_ipath("cloud_config"), "", 0o600), - ] - ) - - def test_mime_text_plain_shell(self, init_tmp, caplog): - """Mime type text/plain starting #!/bin/sh is treated as script.""" - script = "#!/bin/sh\necho hello\n" - message = MIMEBase("text", "plain") - message.set_payload(script) - init_tmp.datasource = FakeDataSource(message.as_string()) - - outpath = os.path.join( - init_tmp.paths.get_ipath_cur("scripts"), "part-001" - ) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert caplog.records == [] # No warnings - - mockobj.assert_has_calls( - [ - mock.call(outpath, script, 0o700), - mock.call(init_tmp.paths.get_ipath("cloud_config"), "", 0o600), - ] - ) - - def test_mime_application_octet_stream(self, init_tmp, caplog): - """Mime type application/octet-stream is ignored but shows warning.""" - message = MIMEBase("application", "octet-stream") - message.set_payload(b"\xbf\xe6\xb2\xc3\xd3\xba\x13\xa4\xd8\xa1\xcc") - encoders.encode_base64(message) - init_tmp.datasource = FakeDataSource(message.as_string().encode()) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert ( - "Unhandled unknown content-type" - " (application/octet-stream)" in caplog.text - ) - mockobj.assert_called_once_with( - init_tmp.paths.get_ipath("cloud_config"), "", 0o600 - ) - - def test_cloud_config_archive(self, init_tmp): - non_decodable = b"\x11\xc9\xb4gTH\xee\x12" - data = [ - {"content": "#cloud-config\npassword: gocubs\n"}, - {"content": "#cloud-config\nlocale: chicago\n"}, - {"content": non_decodable}, - ] - message = b"#cloud-config-archive\n" + safeyaml.dumps(data).encode() - - init_tmp.datasource = FakeDataSource(message) - - fs = {} - - def fsstore(filename, content, mode=0o0644, omode="wb"): - fs[filename] = content - - # consuming the user-data provided should write 'cloud_config' file - # which will have our yaml in it. - with mock.patch("cloudinit.util.write_file") as mockobj: - mockobj.side_effect = fsstore - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - - cfg = util.load_yaml(fs[init_tmp.paths.get_ipath("cloud_config")]) - assert cfg.get("password") == "gocubs" - assert cfg.get("locale") == "chicago" - - @pytest.mark.usefixtures("fake_filesystem") - @mock.patch("cloudinit.util.read_conf_with_confd") - def test_dont_allow_user_data(self, mock_cfg): - mock_cfg.return_value = {"allow_userdata": False} - - # test that user-data is ignored but vendor-data is kept - user_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/bar", "value": "qux2" } -] -""" - vendor_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "quxA" }, - { "op": "add", "path": "/bar", "value": "quxB" }, - { "op": "add", "path": "/foo", "value": "quxC" } -] -""" - init = stages.Init() - init.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) - init.read_cfg() - init.initialize() - init.fetch() - init.instancify() - init.update() - init.cloudify().run( - "consume_data", - init.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - mods = Modules(init) - (_which_ran, _failures) = mods.run_section("cloud_init_modules") - cfg = mods.cfg - assert "vendor_data" in cfg - assert cfg["baz"] == "quxA" - assert cfg["bar"] == "quxB" - assert cfg["foo"] == "quxC" - - -class TestConsumeUserDataHttp: - @responses.activate - @mock.patch("cloudinit.url_helper.time.sleep") - def test_include(self, mock_sleep, init_tmp): - """Test #include.""" - included_url = "http://hostname/path" - included_data = "#cloud-config\nincluded: true\n" - responses.add(responses.GET, included_url, included_data) - - init_tmp.datasource = FakeDataSource("#include\nhttp://hostname/path") - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset") as _reset: - init_tmp.consume_data() - assert _reset.call_count == 1 - cc_contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - cc = util.load_yaml(cc_contents) - assert cc.get("included") is True - - @responses.activate - @mock.patch("cloudinit.url_helper.time.sleep") - def test_include_bad_url(self, mock_sleep, init_tmp): - """Test #include with a bad URL.""" - bad_url = "http://bad/forbidden" - bad_data = "#cloud-config\nbad: true\n" - responses.add(responses.GET, bad_url, bad_data, status=403) - - included_url = "http://hostname/path" - included_data = "#cloud-config\nincluded: true\n" - responses.add(responses.GET, included_url, included_data) - - init_tmp.datasource = FakeDataSource( - "#include\nhttp://bad/forbidden\nhttp://hostname/path" - ) - init_tmp.fetch() - with pytest.raises(Exception, match="403"): - with mock.patch.object(init_tmp, "_reset") as _reset: - init_tmp.consume_data() - assert _reset.call_count == 1 - - with pytest.raises(FileNotFoundError): - util.load_text_file(init_tmp.paths.get_ipath("cloud_config")) - - @responses.activate - @mock.patch("cloudinit.url_helper.time.sleep") - @mock.patch("cloudinit.util.is_container") - @mock.patch( - "cloudinit.user_data.features.ERROR_ON_USER_DATA_FAILURE", False - ) - def test_include_bad_url_no_fail( - self, is_container, mock_sleep, tmpdir, init_tmp, caplog - ): - """Test #include with a bad URL and failure disabled""" - is_container.return_value = True - bad_url = "http://bad/forbidden" - responses.add( - responses.GET, - bad_url, - body="forbidden", - status=403, - ) - - included_url = "http://hostname/path" - included_data = "#cloud-config\nincluded: true\n" - responses.add(responses.GET, included_url, included_data) - - init_tmp.datasource = FakeDataSource( - "#include\nhttp://bad/forbidden\nhttp://hostname/path" - ) - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset") as _reset: - init_tmp.consume_data() - assert _reset.call_count == 1 - - assert ( - "403 Client Error: Forbidden for url: %s" % bad_url in caplog.text - ) - - cc_contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - cc = util.load_yaml(cc_contents) - assert cc.get("bad") is None - assert cc.get("included") is True - - -class TestUDProcess(helpers.ResourceUsingTestCase): - def test_bytes_in_userdata(self): - msg = b"#cloud-config\napt_update: True\n" - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) - message = ud_proc.process(msg) - self.assertTrue(count_messages(message) == 1) - - def test_string_in_userdata(self): - msg = "#cloud-config\napt_update: True\n" - - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) - message = ud_proc.process(msg) - self.assertTrue(count_messages(message) == 1) - - def test_compressed_in_userdata(self): - msg = gzip_text("#cloud-config\napt_update: True\n") - - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) - message = ud_proc.process(msg) - self.assertTrue(count_messages(message) == 1) - - -class TestConvertString(helpers.TestCase): - def test_handles_binary_non_utf8_decodable(self): - """Printable unicode (not utf8-decodable) is safely converted.""" - blob = b"#!/bin/bash\necho \xc3\x84\n" - msg = ud.convert_string(blob) - self.assertEqual(blob, msg.get_payload(decode=True)) - - def test_handles_binary_utf8_decodable(self): - blob = b"\x32\x32" - msg = ud.convert_string(blob) - self.assertEqual(blob, msg.get_payload(decode=True)) - - def test_handle_headers(self): - text = "hi mom" - msg = ud.convert_string(text) - self.assertEqual(text, msg.get_payload(decode=False)) - - def test_handle_mime_parts(self): - """Mime parts are properly returned as a mime message.""" - message = MIMEBase("text", "plain") - message.set_payload("Just text") - msg = ud.convert_string(str(message)) - self.assertEqual("Just text", msg.get_payload(decode=False)) - - -class TestFetchBaseConfig: - @pytest.fixture(autouse=True) - def mocks(self, mocker): - mocker.patch(f"{MPATH}.util.read_conf_from_cmdline") - mocker.patch(f"{MPATH}.read_runtime_config") - - def test_only_builtin_gets_builtin(self, mocker): - mocker.patch(f"{MPATH}.read_runtime_config", return_value={}) - mocker.patch(f"{MPATH}.util.read_conf_with_confd") - config = stages.fetch_base_config(DEFAULT_RUN_DIR) - assert util.get_builtin_cfg() == config - - def test_conf_d_overrides_defaults(self, mocker): - builtin = util.get_builtin_cfg() - test_key = sorted(builtin)[0] - test_value = "test" - - mocker.patch( - f"{MPATH}.util.read_conf_with_confd", - return_value={test_key: test_value}, - ) - mocker.patch(f"{MPATH}.read_runtime_config", return_value={}) - config = stages.fetch_base_config(DEFAULT_RUN_DIR) - assert config.get(test_key) == test_value - builtin[test_key] = test_value - assert config == builtin - - def test_confd_with_template(self, mocker, tmp_path: Path): - instance_data_path = tmp_path / "test_confd_with_template.json" - instance_data_path.write_text('{"template_var": "template_value"}') - cfg_path = tmp_path / "test_conf_with_template.cfg" - cfg_path.write_text('## template:jinja\n{"key": "{{template_var}}"}') - - mocker.patch("cloudinit.stages.CLOUD_CONFIG", cfg_path) - mocker.patch(f"{MPATH}.util.get_builtin_cfg", return_value={}) - config = stages.fetch_base_config( - DEFAULT_RUN_DIR, instance_data_file=instance_data_path - ) - assert config == {"key": "template_value"} - - def test_cmdline_overrides_defaults(self, mocker): - builtin = util.get_builtin_cfg() - test_key = sorted(builtin)[0] - test_value = "test" - cmdline = {test_key: test_value} - - mocker.patch(f"{MPATH}.util.read_conf_with_confd") - mocker.patch( - f"{MPATH}.util.read_conf_from_cmdline", - return_value=cmdline, - ) - mocker.patch(f"{MPATH}.read_runtime_config") - config = stages.fetch_base_config(DEFAULT_RUN_DIR) - assert config.get(test_key) == test_value - builtin[test_key] = test_value - assert config == builtin - - def test_cmdline_overrides_confd_runtime_and_defaults(self, mocker): - builtin = {"key1": "value0", "key3": "other2"} - conf_d = {"key1": "value1", "key2": "other1"} - cmdline = {"key3": "other3", "key2": "other2"} - runtime = {"key3": "runtime3"} - - mocker.patch(f"{MPATH}.util.read_conf_with_confd", return_value=conf_d) - mocker.patch(f"{MPATH}.util.get_builtin_cfg", return_value=builtin) - mocker.patch(f"{MPATH}.read_runtime_config", return_value=runtime) - mocker.patch( - f"{MPATH}.util.read_conf_from_cmdline", - return_value=cmdline, - ) - - config = stages.fetch_base_config(DEFAULT_RUN_DIR) - assert config == {"key1": "value1", "key2": "other2", "key3": "other3"} - - def test_order_precedence_is_builtin_system_runtime_cmdline(self, mocker): - builtin = {"key1": "builtin0", "key3": "builtin3"} - conf_d = {"key1": "confd1", "key2": "confd2", "keyconfd1": "kconfd1"} - runtime = {"key1": "runtime1", "key2": "runtime2"} - cmdline = {"key1": "cmdline1"} - - mocker.patch(f"{MPATH}.util.read_conf_with_confd", return_value=conf_d) - mocker.patch(f"{MPATH}.util.get_builtin_cfg", return_value=builtin) - mocker.patch( - f"{MPATH}.util.read_conf_from_cmdline", - return_value=cmdline, - ) - mocker.patch(f"{MPATH}.read_runtime_config", return_value=runtime) - - config = stages.fetch_base_config(DEFAULT_RUN_DIR) - - assert config == { - "key1": "cmdline1", - "key2": "runtime2", - "key3": "builtin3", - "keyconfd1": "kconfd1", - } diff --git a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/unittests/test_features.py b/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/unittests/test_features.py deleted file mode 100644 index e5e81fbf..00000000 --- a/.pc/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting/tests/unittests/test_features.py +++ /dev/null @@ -1,34 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. -""" -This file is for testing the feature flag functionality itself, -NOT for testing any individual feature flag -""" -from unittest import mock - -from cloudinit import features - - -class TestGetFeatures: - def test_feature_without_override(self): - # Since features are intended to be overridden downstream, mock them - # all here so new feature flags don't require a new change to this - # unit test. - with mock.patch.multiple( - "cloudinit.features", - ERROR_ON_USER_DATA_FAILURE=True, - ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES=True, - EXPIRE_APPLIES_TO_HASHED_USERS=False, - NETPLAN_CONFIG_ROOT_READ_ONLY=True, - DEPRECATION_INFO_BOUNDARY="devel", - NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH=False, - APT_DEB822_SOURCE_LIST_FILE=True, - ): - assert { - "ERROR_ON_USER_DATA_FAILURE": True, - "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES": True, - "EXPIRE_APPLIES_TO_HASHED_USERS": False, - "NETPLAN_CONFIG_ROOT_READ_ONLY": True, - "NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH": False, - "APT_DEB822_SOURCE_LIST_FILE": True, - "DEPRECATION_INFO_BOUNDARY": "devel", - } == features.get_features() diff --git a/.pc/cpick-8810a2dc-test-Remove-CiTestCase-from-test_url_helper.py/tests/unittests/test_url_helper.py b/.pc/cpick-8810a2dc-test-Remove-CiTestCase-from-test_url_helper.py/tests/unittests/test_url_helper.py deleted file mode 100644 index e66b4cae..00000000 --- a/.pc/cpick-8810a2dc-test-Remove-CiTestCase-from-test_url_helper.py/tests/unittests/test_url_helper.py +++ /dev/null @@ -1,951 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. -# pylint: disable=attribute-defined-outside-init - -import logging -from functools import partial -from threading import Event -from time import process_time -from unittest.mock import ANY, call - -import pytest -import requests -import responses - -from cloudinit import util, version -from cloudinit.url_helper import ( - REDACTED, - UrlError, - UrlResponse, - _handle_error, - dual_stack, - oauth_headers, - read_file_or_url, - readurl, - wait_for_url, -) -from tests.unittests.helpers import CiTestCase, mock, skipIf - -try: - import oauthlib - - assert oauthlib # avoid pyflakes error F401: import unused - _missing_oauthlib_dep = False -except ImportError: - _missing_oauthlib_dep = True - - -M_PATH = "cloudinit.url_helper." - - -class TestOAuthHeaders(CiTestCase): - def test_oauth_headers_raises_not_implemented_when_oathlib_missing(self): - """oauth_headers raises a NotImplemented error when oauth absent.""" - with mock.patch.dict("sys.modules", {"oauthlib": None}): - with pytest.raises(NotImplementedError) as context_manager: - oauth_headers(1, 2, 3, 4, 5) - assert "oauth support is not available" == str(context_manager.value) - - @skipIf(_missing_oauthlib_dep, "No python-oauthlib dependency") - @mock.patch("oauthlib.oauth1.Client") - def test_oauth_headers_calls_oathlibclient_when_available(self, m_client): - """oauth_headers calls oaut1.hClient.sign with the provided url.""" - - class fakeclient: - def sign(self, url): - # The first and 3rd item of the client.sign tuple are ignored - return ("junk", url, "junk2") - - m_client.return_value = fakeclient() - - return_value = oauth_headers( - "url", - "consumer_key", - "token_key", - "token_secret", - "consumer_secret", - ) - assert "url" == return_value - - -class TestReadFileOrUrl(CiTestCase): - - with_logs = True - - def test_read_file_or_url_str_from_file(self): - """Test that str(result.contents) on file is text version of contents. - It should not be "b'data'", but just "'data'" """ - tmpf = self.tmp_path("myfile1") - data = b"This is my file content\n" - util.write_file(tmpf, data, omode="wb") - result = read_file_or_url("file://%s" % tmpf) - assert result.contents == data - assert str(result) == data.decode("utf-8") - - @responses.activate - def test_read_file_or_url_str_from_url(self): - """Test that str(result.contents) on url is text version of contents. - It should not be "b'data'", but just "'data'" """ - url = "http://hostname/path" - data = b"This is my url content\n" - responses.add(responses.GET, url, data) - result = read_file_or_url(url) - assert result.contents == data - assert str(result) == data.decode("utf-8") - - @responses.activate - def test_read_file_or_url_str_from_url_streamed(self): - """Test that str(result.contents) on url is text version of contents. - It should not be "b'data'", but just "'data'" """ - url = "http://hostname/path" - data = b"This is my url content\n" - responses.add(responses.GET, url, data) - result = read_file_or_url(url, stream=True) - assert isinstance(result, UrlResponse) - assert result.contents == data - assert str(result) == data.decode("utf-8") - - @responses.activate - def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self): - """Headers are redacted from logs but unredacted in requests.""" - url = "http://hostname/path" - headers = {"sensitive": "sekret", "server": "blah"} - - def _request_callback(request): - for k in headers.keys(): - assert headers[k] == request.headers[k] - return (200, request.headers, "does_not_matter") - - responses.add_callback(responses.GET, url, callback=_request_callback) - - read_file_or_url(url, headers=headers, headers_redact=["sensitive"]) - logs = self.logs.getvalue() - assert REDACTED in logs - assert "sekret" not in logs - - @responses.activate - def test_read_file_or_url_str_from_url_redacts_noheaders(self): - """When no headers_redact, header values are in logs and requests.""" - url = "http://hostname/path" - headers = {"sensitive": "sekret", "server": "blah"} - - def _request_callback(request): - for k in headers.keys(): - assert headers[k] == request.headers[k] - return (200, request.headers, "does_not_matter") - - responses.add_callback(responses.GET, url, callback=_request_callback) - - read_file_or_url(url, headers=headers) - logs = self.logs.getvalue() - assert REDACTED not in logs - assert "sekret" in logs - - def test_wb_read_url_defaults_honored_by_read_file_or_url_callers(self): - """Readurl param defaults used when unspecified by read_file_or_url - - Param defaults tested are as follows: - retries: 0, additional headers None beyond default, method: GET, - data: None, check_status: True and allow_redirects: True - """ - url = "http://hostname/path" - - m_response = mock.MagicMock() - - class FakeSessionRaisesHttpError(requests.Session): - @classmethod - def request(cls, **kwargs): - raise requests.exceptions.RequestException("broke") - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - assert { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": { - "User-Agent": "Cloud-Init/%s" - % (version.version_string()) - }, - "stream": False, - } == kwargs - return m_response - - with mock.patch(M_PATH + "requests.Session") as m_session: - m_session.side_effect = [ - FakeSessionRaisesHttpError(), - FakeSession(), - ] - # assert no retries and check_status == True - with pytest.raises(UrlError) as context_manager: - response = read_file_or_url(url) - assert "broke" == str(context_manager.value) - # assert default headers, method, url and allow_redirects True - # Success on 2nd call with FakeSession - response = read_file_or_url(url) - assert m_response == response._response - - -class TestReadFileOrUrlParameters: - @mock.patch(M_PATH + "readurl") - @pytest.mark.parametrize( - "timeout", [1, 1.2, "1", (1, None), (1, 1), (None, None)] - ) - def test_read_file_or_url_passes_params_to_readurl( - self, m_readurl, timeout - ): - """read_file_or_url passes all params through to readurl.""" - url = "http://hostname/path" - response = "This is my url content\n" - m_readurl.return_value = response - params = { - "url": url, - "timeout": timeout, - "retries": 2, - "headers": {"somehdr": "val"}, - "data": "data", - "sec_between": 1, - "ssl_details": {"cert_file": "/path/cert.pem"}, - "headers_cb": "headers_cb", - "exception_cb": "exception_cb", - "stream": True, - } - - assert response == read_file_or_url(**params) - params.pop("url") # url is passed in as a positional arg - assert m_readurl.call_args_list == [mock.call(url, **params)] - - @pytest.mark.parametrize( - "readurl_timeout,request_timeout", - [ - (-1, 0), - ("-1", 0), - (None, None), - (1, 1.0), - (1.2, 1.2), - ("1", 1.0), - ((1, None), (1, None)), - ((1, 1), (1, 1)), - ((None, None), (None, None)), - ], - ) - def test_readurl_timeout(self, readurl_timeout, request_timeout): - url = "http://hostname/path" - m_response = mock.MagicMock() - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - expected_kwargs = { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": { - "User-Agent": "Cloud-Init/%s" - % (version.version_string()) - }, - "timeout": request_timeout, - "stream": False, - } - if request_timeout is None: - expected_kwargs.pop("timeout") - - assert kwargs == expected_kwargs - return m_response - - with mock.patch( - M_PATH + "requests.Session", side_effect=[FakeSession()] - ): - response = read_file_or_url(url, timeout=readurl_timeout) - - assert response._response == m_response - - -def assert_time(func, max_time=1): - """Assert function time is bounded by a max (default=1s) - - The following async tests should canceled in under 1ms and have stagger - delay and max_ - It is possible that this could yield a false positive, but this should - basically never happen (esp under normal system load). - """ - start = process_time() - try: - out = func() - finally: - diff = process_time() - start - assert diff < max_time - return out - - -class TestReadUrl: - @pytest.mark.parametrize("headers", [{}, {"Metadata": "true"}]) - def test_headers(self, headers): - url = "http://hostname/path" - m_response = mock.MagicMock() - - expected_headers = headers.copy() - expected_headers["User-Agent"] = "Cloud-Init/%s" % ( - version.version_string() - ) - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - expected_kwargs = { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": expected_headers, - "stream": False, - } - - assert kwargs == expected_kwargs - return m_response - - with mock.patch( - M_PATH + "requests.Session", side_effect=[FakeSession()] - ): - response = readurl(url, headers=headers) - - assert response._response == m_response - - @pytest.mark.parametrize("headers", [{}, {"Metadata": "true"}]) - def test_headers_cb(self, headers): - url = "http://hostname/path" - m_response = mock.MagicMock() - - expected_headers = headers.copy() - expected_headers["User-Agent"] = "Cloud-Init/%s" % ( - version.version_string() - ) - headers_cb = lambda _: headers - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - expected_kwargs = { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": expected_headers, - "stream": False, - } - - assert kwargs == expected_kwargs - return m_response - - with mock.patch( - M_PATH + "requests.Session", side_effect=[FakeSession()] - ): - response = readurl(url, headers_cb=headers_cb) - - assert response._response == m_response - - def test_error_no_cb(self, mocker): - response = requests.Response() - response.status_code = 500 - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.return_value = response - - with pytest.raises(UrlError) as e: - readurl("http://some/path") - assert e.value.code == 500 - - def test_error_cb_true(self, mocker): - mocker.patch("time.sleep") - - bad_response = requests.Response() - bad_response.status_code = 500 - bad_response._content = b"oh noes!" - good_response = requests.Response() - good_response.status_code = 200 - good_response._content = b"yay" - - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.side_effect = (bad_response, good_response) - - readurl("http://some/path", retries=1, exception_cb=lambda _: True) - assert m_request.call_count == 2 - - def test_error_cb_false(self, mocker): - mocker.patch("time.sleep") - - bad_response = requests.Response() - bad_response.status_code = 500 - bad_response._content = b"oh noes!" - - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.return_value = bad_response - - with pytest.raises(UrlError): - readurl( - "http://some/path", retries=1, exception_cb=lambda _: False - ) - assert m_request.call_count == 1 - - def test_exception_503(self, mocker): - mocker.patch("time.sleep") - - retry_response = requests.Response() - retry_response.status_code = 503 - retry_response._content = b"try again" - good_response = requests.Response() - good_response.status_code = 200 - good_response._content = b"good" - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.side_effect = (retry_response, retry_response, good_response) - - readurl("http://some/path") - assert m_request.call_count == 3 - - -event = Event() - - -class TestDualStack: - """Async testing suggestions welcome - these all rely on time-bounded - assertions (via threading.Event) to prove ordering - """ - - @pytest.mark.parametrize( - ["func", "addresses", "stagger_delay", "timeout", "expected_val"], - [ - # Assert order based on timeout - (lambda x, _: x, ("one", "two"), 1, 1, "one"), - # Assert timeout results in (None, None) - (lambda _a, _b: event.wait(1), ("one", "two"), 1, 0, None), - ( - lambda a, _b: 1 / 0 if a == "one" else a, - ("one", "two"), - 0, - 1, - "two", - ), - # Assert that exception in func is only raised - # if neither thread gets a valid result - ( - lambda a, _b: 1 / 0 if a == "two" else a, - ("one", "two"), - 0, - 1, - "one", - ), - # simulate a slow response to verify correct order - ( - lambda x, _: event.wait(1) if x != "two" else x, - ("one", "two"), - 0, - 1, - "two", - ), - # simulate a slow response to verify correct order - ( - lambda x, _: event.wait(1) if x != "tri" else x, - ("one", "two", "tri"), - 0, - 1, - "tri", - ), - ], - ) - def test_dual_stack( - self, - func, - addresses, - stagger_delay, - timeout, - expected_val, - ): - """Assert various failure modes behave as expected""" - event.clear() - - gen = partial( - dual_stack, - func, - addresses, - stagger_delay=stagger_delay, - timeout=timeout, - ) - _, result = assert_time(gen) - assert expected_val == result - - event.set() - - @pytest.mark.parametrize( - [ - "func", - "addresses", - "stagger_delay", - "timeout", - "message", - "expected_exc", - ], - [ - ( - lambda _a, _b: 1 / 0, - ("¯\\_(ツ)_/¯", "(╯°□°）╯︵ ┻━┻"), - 0, - 1, - "division by zero", - ZeroDivisionError, - ), - ( - lambda _a, _b: 1 / 0, - ("it", "really", "doesn't"), - 0, - 1, - "division by zero", - ZeroDivisionError, - ), - ( - lambda _a, _b: [][0], # pylint: disable=E0643 - ("matter", "these"), - 0, - 1, - "list index out of range", - IndexError, - ), - ( - lambda _a, _b: (_ for _ in ()).throw( - Exception("soapstone is not effective soap") - ), - ("are", "ignored"), - 0, - 1, - "soapstone is not effective soap", - Exception, - ), - ], - ) - def test_dual_stack_exceptions( - self, - func, - addresses, - stagger_delay, - timeout, - message, - expected_exc, - caplog, - ): - # Context: - # - # currently if all threads experience exception - # dual_stack() logs an error containing all exceptions - # but only raises the last exception to occur - # Verify "best effort behavior" - # dual_stack will temporarily ignore an exception in any of the - # request threads in hopes that a later thread will succeed - # this behavior is intended to allow a requests.ConnectionError - # exception from on endpoint to occur without preventing another - # thread from succeeding - event.clear() - - # Note: python3.6 repr(Exception("test")) produces different output - # than later versions, so we cannot match exact message without - # some ugly manual exception repr() function, which I'd rather not do - # in dual_stack(), so we recreate expected messages manually here - # in a version-independant way for testing, the extra comma on old - # versions won't hurt anything - exc_list = str([expected_exc(message) for _ in addresses]) - expected_msg = f"Exception(s) {exc_list} during request" - gen = partial( - dual_stack, - func, - addresses, - stagger_delay=stagger_delay, - timeout=timeout, - ) - with pytest.raises(expected_exc): - gen() # 1 - with caplog.at_level(logging.DEBUG): - try: - gen() # 2 - except expected_exc: - pass - finally: - assert 2 == len(caplog.records) - assert 2 == caplog.text.count(expected_msg) - event.set() - - def test_dual_stack_staggered(self): - """Assert expected call intervals occur""" - stagger = 0.1 - with mock.patch(M_PATH + "_run_func_with_delay") as delay_func: - - def identity_of_first_arg(x, _): - return x - - dual_stack( - identity_of_first_arg, - ["you", "and", "me", "and", "dog"], - stagger_delay=stagger, - timeout=1, - ) - - # ensure that stagger delay for each call is made with args: - # [ 0 * N, 1 * N, 2 * N, 3 * N, 4 * N, 5 * N] where N = stagger - # it appears that without an explicit wait/join we can't assert - # number of calls - calls = [ - call( - func=identity_of_first_arg, - addr="you", - timeout=1, - event=ANY, - delay=stagger * 0, - ), - call( - func=identity_of_first_arg, - addr="and", - timeout=1, - event=ANY, - delay=stagger * 1, - ), - call( - func=identity_of_first_arg, - addr="me", - timeout=1, - event=ANY, - delay=stagger * 2, - ), - call( - func=identity_of_first_arg, - addr="and", - timeout=1, - event=ANY, - delay=stagger * 3, - ), - call( - func=identity_of_first_arg, - addr="dog", - timeout=1, - event=ANY, - delay=stagger * 4, - ), - ] - num_calls = 0 - for call_instance in calls: - if call_instance in delay_func.call_args_list: - num_calls += 1 - - # we can't know the order of the submitted functions' execution - # we can't know how many of the submitted functions get called - # in advance - # - # we _do_ know what the possible arg combinations are - # we _do_ know from the mocked function how many got called - # assert that all calls that occurred had known valid arguments - # by checking for the correct number of matches - assert num_calls == len(delay_func.call_args_list) - - -ADDR1 = "https://addr1/" -SLEEP1 = "https://sleep1/" -SLEEP2 = "https://sleep2/" - - -class TestWaitForUrl: - success = "SUCCESS" - fail = "FAIL" - event = Event() - - @pytest.fixture - def retry_mocks(self, mocker): - self.mock_time_value = 0 - m_readurl = mocker.patch( - f"{M_PATH}readurl", side_effect=self.readurl_side_effect - ) - m_sleep = mocker.patch( - f"{M_PATH}time.sleep", side_effect=self.sleep_side_effect - ) - mocker.patch( - f"{M_PATH}time.monotonic", side_effect=self.time_side_effect - ) - - yield m_readurl, m_sleep - - self.mock_time_value = 0 - - @classmethod - def response_wait(cls, _request): - cls.event.wait(0.1) - return (500, {"request-id": "1"}, cls.fail) - - @classmethod - def response_nowait(cls, _request): - return (200, {"request-id": "0"}, cls.success) - - @pytest.mark.parametrize( - ["addresses", "expected_address_index", "response"], - [ - # Use timeout to test ordering happens as expected - ((ADDR1, SLEEP1), 0, "SUCCESS"), - ((SLEEP1, ADDR1), 1, "SUCCESS"), - ((SLEEP1, SLEEP2, ADDR1), 2, "SUCCESS"), - ((ADDR1, SLEEP1, SLEEP2), 0, "SUCCESS"), - ], - ) - @responses.activate - def test_order(self, addresses, expected_address_index, response): - """Check that the first response gets returned. Simulate a - non-responding endpoint with a response that has a one second wait. - - If this test proves flaky, increase wait time. Since it is async, - increasing wait time for the non-responding endpoint should not - increase total test time, assuming async_delay=0 is used and at least - one non-waiting endpoint is registered with responses. - Subsequent tests will continue execution after the first response is - received. - """ - self.event.clear() - for address in set(addresses): - responses.add_callback( - responses.GET, - address, - callback=( - self.response_wait - if "sleep" in address - else self.response_nowait - ), - content_type="application/json", - ) - - # Use async_delay=0.0 to avoid adding unnecessary time to tests - # In practice a value such as 0.150 is used - url, response_contents = wait_for_url( - urls=addresses, - max_wait=2, - timeout=0.3, - connect_synchronously=False, - async_delay=0.0, - ) - self.event.set() - - # Test for timeout (no responding endpoint) - assert addresses[expected_address_index] == url - assert response.encode() == response_contents - - @responses.activate - def test_timeout(self): - """If no endpoint responds in time, expect no response""" - - self.event.clear() - addresses = [SLEEP1, SLEEP2] - for address in set(addresses): - responses.add_callback( - responses.GET, - address, - callback=( - requests.ConnectTimeout - if "sleep" in address - else self.response_nowait - ), - content_type="application/json", - ) - - # Use async_delay=0.0 to avoid adding unnecessary time to tests - url, response_contents = wait_for_url( - urls=addresses, - max_wait=1, - timeout=1, - connect_synchronously=False, - async_delay=0, - ) - self.event.set() - assert not url - assert not response_contents - - def test_explicit_arguments(self, retry_mocks): - """Ensure that explicit arguments are respected""" - m_readurl, m_sleep = retry_mocks - wait_for_url( - urls=["http://localhost/"], - max_wait=23, - timeout=5, - sleep_time=3, - ) - - assert len(m_readurl.call_args_list) == 3 - assert len(m_sleep.call_args_list) == 2 - - for readurl_call in m_readurl.call_args_list: - assert readurl_call[1]["timeout"] == 5 - for sleep_call in m_sleep.call_args_list: - assert sleep_call[0][0] == 3 - - # Call 1 starts 0 - # Call 2 starts at 8-ish after 5 second timeout and 3 second sleep - # Call 3 starts at 16-ish for same reasons - # The 5 second timeout puts us at 21-ish and now we break - # because 21-ish + the sleep time puts us over max wait of 23 - assert pytest.approx(self.mock_time_value) == 21 - - def test_shortened_timeout(self, retry_mocks): - """Test that we shorten the last timeout to align with max_wait""" - m_readurl, _m_sleep = retry_mocks - wait_for_url( - urls=["http://localhost/"], max_wait=10, timeout=9, sleep_time=0 - ) - - assert len(m_readurl.call_args_list) == 2 - assert m_readurl.call_args_list[-1][1]["timeout"] == pytest.approx(1) - - def test_default_sleep_time(self, retry_mocks): - """Test default sleep behavior when not specified""" - _m_readurl, m_sleep = retry_mocks - wait_for_url( - urls=["http://localhost/"], - max_wait=50, - timeout=1, - ) - - expected_sleep_times = [1] * 5 + [2] * 5 + [3] * 5 - actual_sleep_times = [ - m_sleep.call_args_list[i][0][0] - for i in range(len(m_sleep.call_args_list)) - ] - assert actual_sleep_times == expected_sleep_times - - @responses.activate - def test_503(self, mocker): - mocker.patch("time.sleep") - - for _ in range(10): - responses.add( - method=responses.GET, - url="http://hi/", - status=503, - body=b"try again", - ) - responses.add( - method=responses.GET, - url="http://hi/", - status=200, - body=b"good", - ) - - assert wait_for_url(urls=["http://hi/"], max_wait=0.0001)[1] == b"good" - - @responses.activate - def test_503_async(self, mocker): - mocker.patch("time.sleep") - - for _ in range(10): - responses.add( - method=responses.GET, - url="http://hi/", - status=503, - body=b"try again", - ) - responses.add( - method=responses.GET, - url="http://hi2/", - status=503, - body="try again", - ) - responses.add( - method=responses.GET, - url="http://hi/", - status=200, - body=b"good", - ) - responses.add( - method=responses.GET, - url="http://hi2/", - status=200, - body=b"good", - ) - - assert ( - wait_for_url( - urls=["http://hi/", "http://hi2/"], - max_wait=0.0001, - async_delay=0, - connect_synchronously=False, - )[1] - == b"good" - ) - - # These side effect methods are a way of having a somewhat predictable - # output for time.monotonic(). Otherwise, we have to track too many calls - # to time.monotonic() and unrelated changes to code being called could - # cause these tests to fail. - # 0.0000001 is added to simulate additional execution time but keep it - # small enough for pytest.approx() to work - def sleep_side_effect(self, sleep_time): - self.mock_time_value += sleep_time + 0.0000001 - - def time_side_effect(self): - return self.mock_time_value - - def readurl_side_effect(self, *args, **kwargs): - if "timeout" in kwargs: - self.mock_time_value += kwargs["timeout"] + 0.0000001 - raise UrlError("test") - - -class TestHandleError: - def test_handle_error_no_cb(self): - """Test no callback.""" - assert _handle_error(UrlError("test")) is None - - def test_handle_error_cb_false(self): - """Test callback returning False.""" - with pytest.raises(UrlError) as e: - _handle_error(UrlError("test"), exception_cb=lambda _: False) - assert str(e.value) == "test" - - def test_handle_error_cb_true(self): - """Test callback returning True.""" - assert ( - _handle_error(UrlError("test"), exception_cb=lambda _: True) - ) is None - - def test_handle_503(self, caplog): - """Test 503 with no callback.""" - assert _handle_error(UrlError("test", code=503)) == 1 - assert "Unable to introspect response header" in caplog.text - - def test_handle_503_with_retry_header(self): - """Test 503 with a retry integer value.""" - assert ( - _handle_error( - UrlError("test", code=503, headers={"Retry-After": 5}) - ) - == 5 - ) - - def test_handle_503_with_retry_header_in_past(self, caplog): - """Test 503 with date in the past.""" - assert ( - _handle_error( - UrlError( - "test", - code=503, - headers={"Retry-After": "Fri, 31 Dec 1999 23:59:59 GMT"}, - ) - ) - == 1 - ) - assert "Retry-After header value is in the past" in caplog.text - - def test_handle_503_cb_true(self): - """Test 503 with a callback returning True.""" - assert ( - _handle_error( - UrlError("test", code=503), - exception_cb=lambda _: True, - ) - is None - ) - - def test_handle_503_cb_false(self): - """Test 503 with a callback returning False.""" - assert ( - _handle_error( - UrlError("test", code=503), - exception_cb=lambda _: False, - ) - == 1 - ) diff --git a/.pc/cpick-9311e066-fix-Update-OauthUrlHelper-to-use-readurl-exception_cb/cloudinit/url_helper.py b/.pc/cpick-9311e066-fix-Update-OauthUrlHelper-to-use-readurl-exception_cb/cloudinit/url_helper.py deleted file mode 100644 index ad2c6383..00000000 --- a/.pc/cpick-9311e066-fix-Update-OauthUrlHelper-to-use-readurl-exception_cb/cloudinit/url_helper.py +++ /dev/null @@ -1,1138 +0,0 @@ -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This file is part of cloud-init. See LICENSE file for license information. - -import copy -import ftplib -import io -import json -import logging -import os -import threading -import time -from concurrent.futures import ThreadPoolExecutor, TimeoutError, as_completed -from email.utils import parsedate -from functools import partial -from http.client import NOT_FOUND -from itertools import count -from ssl import create_default_context -from typing import ( - Any, - Callable, - Iterator, - List, - Mapping, - NamedTuple, - Optional, - Tuple, - Union, -) -from urllib.parse import quote, urlparse, urlsplit, urlunparse - -import requests -from requests import exceptions - -from cloudinit import performance, util, version - -LOG = logging.getLogger(__name__) - -REDACTED = "REDACTED" -ExceptionCallback = Optional[Callable[["UrlError"], bool]] - - -def _cleanurl(url): - parsed_url = list(urlparse(url, scheme="http")) - if not parsed_url[1] and parsed_url[2]: - # Swap these since this seems to be a common - # occurrence when given urls like 'www.google.com' - parsed_url[1] = parsed_url[2] - parsed_url[2] = "" - return urlunparse(parsed_url) - - -def combine_url(base, *add_ons): - def combine_single(url, add_on): - url_parsed = list(urlparse(url)) - path = url_parsed[2] - if path and not path.endswith("/"): - path += "/" - path += quote(str(add_on), safe="/:") - url_parsed[2] = path - return urlunparse(url_parsed) - - url = base - for add_on in add_ons: - url = combine_single(url, add_on) - return url - - -def ftp_get_return_code_from_exception(exc) -> int: - """helper for read_ftps to map return codes to a number""" - # ftplib doesn't expose error codes, so use this lookup table - ftp_error_codes = { - ftplib.error_reply: 300, # unexpected [123]xx reply - ftplib.error_temp: 400, # 4xx errors - ftplib.error_perm: 500, # 5xx errors - ftplib.error_proto: 600, # response does not begin with [1-5] - EOFError: 700, # made up - # OSError is also possible. Use OSError.errno for that. - } - code = ftp_error_codes.get(type(exc)) # pyright: ignore - if not code: - if isinstance(exc, OSError): - code = exc.errno - else: - LOG.warning( - "Unexpected exception type while connecting to ftp server." - ) - code = -99 - return code - - -def read_ftps(url: str, timeout: float = 5.0, **kwargs: dict) -> "FtpResponse": - """connect to URL using ftp over TLS and read a file - - when using strict mode (ftps://), raise exception in event of failure - when not using strict mode (ftp://), fall back to using unencrypted ftp - - url: string containing the desination to read a file from. The url is - parsed with urllib.urlsplit to identify username, password, host, - path, and port in the following format: - ftps://[username:password@]host[:port]/[path] - host is the only required component - timeout: maximum time for the connection to take - kwargs: unused, for compatibility with read_url - returns: UrlResponse - """ - - url_parts = urlsplit(url) - if not url_parts.hostname: - raise UrlError( - cause="Invalid url provided", code=NOT_FOUND, headers=None, url=url - ) - with io.BytesIO() as buffer: - port = url_parts.port or 21 - user = url_parts.username or "anonymous" - if "ftps" == url_parts.scheme: - try: - ftp_tls = ftplib.FTP_TLS( - context=create_default_context(), - ) - LOG.debug( - "Attempting to connect to %s via port [%s] over tls.", - url, - port, - ) - ftp_tls.connect( - host=url_parts.hostname, - port=port, - timeout=timeout or 5.0, # uses float internally - ) - except ftplib.all_errors as e: - code = ftp_get_return_code_from_exception(e) - raise UrlError( - cause=( - "Reading file from server over tls " - f"failed for url {url} [{code}]" - ), - code=code, - headers=None, - url=url, - ) from e - LOG.debug("Attempting to login with user [%s]", user) - try: - ftp_tls.login( - user=user, - passwd=url_parts.password or "", - ) - LOG.debug("Creating a secure connection") - ftp_tls.prot_p() - except ftplib.error_perm as e: - LOG.warning( - "Attempted to connect to an insecure ftp server but used " - "a scheme of ftps://, which is not allowed. Use ftp:// " - "to allow connecting to insecure ftp servers." - ) - raise UrlError( - cause=( - "Attempted to connect to an insecure ftp server but " - "used a scheme of ftps://, which is not allowed. Use " - "ftp:// to allow connecting to insecure ftp servers." - ), - code=500, - headers=None, - url=url, - ) from e - try: - LOG.debug("Reading file: %s", url_parts.path) - ftp_tls.retrbinary( - f"RETR {url_parts.path}", callback=buffer.write - ) - - return FtpResponse(buffer.getvalue(), url) - except ftplib.all_errors as e: - code = ftp_get_return_code_from_exception(e) - raise UrlError( - cause=( - "Reading file from ftp server" - f" failed for url {url} [{code}]" - ), - code=code, - headers=None, - url=url, - ) from e - finally: - LOG.debug("Closing connection") - ftp_tls.close() - else: - try: - ftp = ftplib.FTP() - LOG.debug( - "Attempting to connect to %s via port %s.", url, port - ) - ftp.connect( - host=url_parts.hostname, - port=port, - timeout=timeout or 5.0, # uses float internally - ) - LOG.debug("Attempting to login with user [%s]", user) - ftp.login( - user=user, - passwd=url_parts.password or "", - ) - LOG.debug("Reading file: %s", url_parts.path) - ftp.retrbinary(f"RETR {url_parts.path}", callback=buffer.write) - return FtpResponse(buffer.getvalue(), url) - except ftplib.all_errors as e: - code = ftp_get_return_code_from_exception(e) - raise UrlError( - cause=( - "Reading file from ftp server" - f" failed for url {url} [{code}]" - ), - code=code, - headers=None, - url=url, - ) from e - finally: - LOG.debug("Closing connection") - ftp.close() - - -def _read_file(path: str, **kwargs) -> "FileResponse": - """read a binary file and return a FileResponse - - matches function signature with read_ftps and read_url - """ - if kwargs.get("data"): - LOG.warning("Unable to post data to file resource %s", path) - try: - contents = util.load_binary_file(path) - return FileResponse(contents, path) - except FileNotFoundError as e: - raise UrlError(cause=e, code=NOT_FOUND, headers=None, url=path) from e - except IOError as e: - raise UrlError(cause=e, code=e.errno, headers=None, url=path) from e - - -def read_file_or_url( - url, **kwargs -) -> Union["FileResponse", "UrlResponse", "FtpResponse"]: - """Wrapper function around readurl to allow passing a file path as url. - - When url is not a local file path, passthrough any kwargs to readurl. - - In the case of parameter passthrough to readurl, default values for some - parameters. See: call-signature of readurl in this module for param docs. - """ - url = url.lstrip() - try: - parsed = urlparse(url) - except ValueError as e: - raise UrlError(cause=e, url=url) from e - scheme = parsed.scheme - if scheme == "file" or (url and "/" == url[0]): - return _read_file(parsed.path, **kwargs) - elif scheme in ("ftp", "ftps"): - return read_ftps(url, **kwargs) - elif scheme in ("http", "https"): - return readurl(url, **kwargs) - else: - LOG.warning("Attempting unknown protocol %s", scheme) - return readurl(url, **kwargs) - - -# Made to have same accessors as UrlResponse so that the -# read_file_or_url can return this or that object and the -# 'user' of those objects will not need to know the difference. -class StringResponse: - def __init__(self, contents, url, code=200): - self.code = code - self.headers = {} - self.contents = contents - self.url = url - - def ok(self, *args, **kwargs): - return self.code == 200 - - def __str__(self): - return self.contents.decode("utf-8") - - -class FileResponse(StringResponse): - def __init__(self, contents: bytes, url: str, code=200): - super().__init__(contents, url, code=code) - - -class FtpResponse(StringResponse): - def __init__(self, contents: bytes, url: str): - super().__init__(contents, url) - - -class UrlResponse: - def __init__(self, response: requests.Response): - self._response = response - - @property - def contents(self) -> bytes: - if self._response.content is None: - return b"" - return self._response.content - - @property - def url(self) -> str: - return self._response.url - - def ok(self, redirects_ok=False) -> bool: - upper = 300 - if redirects_ok: - upper = 400 - if 200 <= self.code < upper: - return True - else: - return False - - @property - def headers(self): - return self._response.headers - - @property - def code(self) -> int: - return self._response.status_code - - def __str__(self): - return self._response.text - - def iter_content( - self, chunk_size: Optional[int] = 1, decode_unicode: bool = False - ) -> Iterator[bytes]: - """Iterates over the response data. - - When stream=True is set on the request, this avoids reading the content - at once into memory for large responses. - - :param chunk_size: Number of bytes it should read into memory. - :param decode_unicode: If True, content will be decoded using the best - available encoding based on the response. - """ - yield from self._response.iter_content(chunk_size, decode_unicode) - - -class UrlError(IOError): - def __init__( - self, - cause: Any, # This SHOULD be an exception to wrap, but can be anything - code: Optional[int] = None, - headers: Optional[Mapping] = None, - url: Optional[str] = None, - ): - IOError.__init__(self, str(cause)) - self.cause = cause - self.code = code - self.headers: Mapping = {} if headers is None else headers - self.url = url - - -def _get_ssl_args(url, ssl_details): - ssl_args = {} - scheme = urlparse(url).scheme - if scheme == "https" and ssl_details: - if "ca_certs" in ssl_details and ssl_details["ca_certs"]: - ssl_args["verify"] = ssl_details["ca_certs"] - else: - ssl_args["verify"] = True - if "cert_file" in ssl_details and "key_file" in ssl_details: - ssl_args["cert"] = [ - ssl_details["cert_file"], - ssl_details["key_file"], - ] - elif "cert_file" in ssl_details: - ssl_args["cert"] = str(ssl_details["cert_file"]) - return ssl_args - - -def _get_retry_after(retry_after: str) -> float: - """Parse a Retry-After header value into an integer. - - : param retry_after: The value of the Retry-After header. - https://www.rfc-editor.org/rfc/rfc9110.html#section-10.2.3 - https://www.rfc-editor.org/rfc/rfc2616#section-3.3 - : return: The number of seconds to wait before retrying the request. - """ - try: - to_wait = float(retry_after) - except ValueError: - # Translate a date such as "Fri, 31 Dec 1999 23:59:59 GMT" - # into seconds to wait - try: - time_tuple = parsedate(retry_after) - if not time_tuple: - raise ValueError("Failed to parse Retry-After header value") - to_wait = float(time.mktime(time_tuple) - time.time()) - except ValueError: - LOG.info( - "Failed to parse Retry-After header value: %s. " - "Waiting 1 second instead.", - retry_after, - ) - to_wait = 1 - if to_wait < 0: - LOG.info( - "Retry-After header value is in the past. " - "Waiting 1 second instead." - ) - to_wait = 1 - return to_wait - - -def _handle_error( - error: UrlError, - *, - exception_cb: ExceptionCallback = None, -) -> Optional[float]: - """Handle exceptions raised during request processing. - - If we have no exception callback or the callback handled the error or we - got a 503, return with an optional timeout so the request can be retried. - Otherwise, raise the error. - - :param error: The exception raised during the request. - :param response: The response object. - :param exception_cb: Callable to handle the exception. - - :return: Optional time to wait before retrying the request. - """ - if exception_cb and exception_cb(error): - return None - if error.code and error.code == 503: - LOG.warning( - "Ec2 IMDS endpoint returned a 503 error. " - "HTTP endpoint is overloaded. Retrying." - ) - if error.headers: - return _get_retry_after(error.headers.get("Retry-After", "1")) - LOG.info("Unable to introspect response header. Waiting 1 second.") - return 1 - if not exception_cb: - return None - # If exception_cb returned False and there's no 503 - raise error - - -def readurl( - url, - *, - data=None, - timeout=None, - retries=0, - sec_between=1, - headers=None, - headers_cb=None, - headers_redact=None, - ssl_details=None, - check_status=True, - allow_redirects=True, - exception_cb: ExceptionCallback = None, - session=None, - infinite=False, - log_req_resp=True, - request_method="", - stream: bool = False, -) -> UrlResponse: - """Wrapper around requests.Session to read the url and retry if necessary - - :param url: Mandatory url to request. - :param data: Optional form data to post the URL. Will set request_method - to 'POST' if present. - :param timeout: Timeout in seconds to wait for a response. May be a tuple - if specifying (connection timeout, read timeout). - :param retries: Number of times to retry on exception if exception_cb is - None or exception_cb returns True for the exception caught. Default is - to fail with 0 retries on exception. - :param sec_between: Default 1: amount of seconds passed to time.sleep - between retries. None or -1 means don't sleep. - :param headers: Optional dict of headers to send during request - :param headers_cb: Optional callable returning a dict of values to send as - headers during request - :param headers_redact: Optional list of header names to redact from the log - :param ssl_details: Optional dict providing key_file, ca_certs, and - cert_file keys for use on in ssl connections. - :param check_status: Optional boolean set True to raise when HTTPError - occurs. Default: True. - :param allow_redirects: Optional boolean passed straight to Session.request - as 'allow_redirects'. Default: True. - :param exception_cb: Optional callable to handle exception and returns - True if retries are permitted. - :param session: Optional exiting requests.Session instance to reuse. - :param infinite: Bool, set True to retry indefinitely. Default: False. - :param log_req_resp: Set False to turn off verbose debug messages. - :param request_method: String passed as 'method' to Session.request. - Typically GET, or POST. Default: POST if data is provided, GET - otherwise. - :param stream: if False, the response content will be immediately - downloaded. - """ - url = _cleanurl(url) - req_args = { - "url": url, - "stream": stream, - } - req_args.update(_get_ssl_args(url, ssl_details)) - req_args["allow_redirects"] = allow_redirects - if not request_method: - request_method = "POST" if data else "GET" - req_args["method"] = request_method - if timeout is not None: - if isinstance(timeout, tuple): - req_args["timeout"] = timeout - else: - req_args["timeout"] = max(float(timeout), 0) - if headers_redact is None: - headers_redact = [] - manual_tries = 1 - if retries: - manual_tries = max(int(retries) + 1, 1) - - user_agent = "Cloud-Init/%s" % (version.version_string()) - if headers is not None: - headers = headers.copy() - else: - headers = {} - - if data: - req_args["data"] = data - if sec_between is None: - sec_between = -1 - - if session is None: - session = requests.Session() - - # Handle retrying ourselves since the built-in support - # doesn't handle sleeping between tries... - for i in count(): - if headers_cb: - headers = headers_cb(url) - - if "User-Agent" not in headers: - headers["User-Agent"] = user_agent - - req_args["headers"] = headers - filtered_req_args = {} - for k, v in req_args.items(): - if k == "data": - continue - if k == "headers" and headers_redact: - matched_headers = [k for k in headers_redact if v.get(k)] - if matched_headers: - filtered_req_args[k] = copy.deepcopy(v) - for key in matched_headers: - filtered_req_args[k][key] = REDACTED - else: - filtered_req_args[k] = v - raised_exception: Exception - try: - if log_req_resp: - LOG.debug( - "[%s/%s] open '%s' with %s configuration", - i, - "infinite" if infinite else manual_tries, - url, - filtered_req_args, - ) - - response = session.request(**req_args) - - if check_status: - response.raise_for_status() - LOG.debug( - "Read from %s (%s, %sb) after %s attempts", - url, - response.status_code, - len(response.content), - (i + 1), - ) - # Doesn't seem like we can make it use a different - # subclass for responses, so add our own backward-compat - # attrs - return UrlResponse(response) - except exceptions.SSLError as e: - # ssl exceptions are not going to get fixed by waiting a - # few seconds - raise UrlError(e, url=url) from e - except exceptions.HTTPError as e: - url_error = UrlError( - e, - code=e.response.status_code, - headers=e.response.headers, - url=url, - ) - raised_exception = e - except exceptions.RequestException as e: - url_error = UrlError(e, url=url) - raised_exception = e - response = None - - response_sleep_time = _handle_error( - url_error, - exception_cb=exception_cb, - ) - # If our response tells us to wait, then wait even if we're - # past the max tries - if not response_sleep_time: - will_retry = infinite or (i + 1 < manual_tries) - if not will_retry: - raise url_error from raised_exception - sleep_time = response_sleep_time or sec_between - - if sec_between > 0: - if log_req_resp: - LOG.debug( - "Please wait %s seconds while we wait to try again", - sec_between, - ) - time.sleep(sleep_time) - - raise RuntimeError("This path should be unreachable...") - - -def _run_func_with_delay( - func: Callable[..., Any], - addr: str, - timeout: int, - event: threading.Event, - delay: Optional[float] = None, -) -> Any: - """Execute func with optional delay""" - if delay: - - # event returns True iff the flag is set to true: indicating that - # another thread has already completed successfully, no need to try - # again - exit early - if event.wait(timeout=delay): - return - return func(addr, timeout) - - -def dual_stack( - func: Callable[..., Any], - addresses: List[str], - stagger_delay: float = 0.150, - timeout: int = 10, -) -> Tuple[Optional[str], Optional[UrlResponse]]: - """execute multiple callbacks in parallel - - Run blocking func against two different addresses staggered with a - delay. The first call to return successfully is returned from this - function and remaining unfinished calls are cancelled if they have not - yet started - """ - return_result = None - returned_address = None - last_exception: Optional[BaseException] = None - exceptions = [] - is_done = threading.Event() - - # future work: add cancel_futures to Python stdlib ThreadPoolExecutor - # context manager implementation - # - # for now we don't use this feature since it only supports python >3.8 - # and doesn't provide a context manager and only marginal benefit - executor = ThreadPoolExecutor(max_workers=len(addresses)) - try: - futures = { - executor.submit( - _run_func_with_delay, - func=func, - addr=addr, - timeout=timeout, - event=is_done, - delay=(i * stagger_delay), - ): addr - for i, addr in enumerate(addresses) - } - - # handle returned requests in order of completion - for future in as_completed(futures, timeout=timeout): - - returned_address = futures[future] - return_exception = future.exception() - if return_exception: - last_exception = return_exception - exceptions.append(last_exception) - else: - return_result = future.result() - if return_result: - - # communicate to other threads that they do not need to - # try: this thread has already succeeded - is_done.set() - return (returned_address, return_result) - - # No success, return the last exception but log them all for - # debugging - if last_exception: - LOG.warning( - "Exception(s) %s during request to " - "%s, raising last exception", - exceptions, - returned_address, - ) - raise last_exception - else: - LOG.error("Empty result for address %s", returned_address) - raise ValueError("No result returned") - - # when max_wait expires, log but don't throw (retries happen) - except TimeoutError: - LOG.warning( - "Timed out waiting for addresses: %s, " - "exception(s) raised while waiting: %s", - " ".join(addresses), - " ".join(map(str, exceptions)), - ) - finally: - executor.shutdown(wait=False) - - return (returned_address, return_result) - - -class HandledResponse(NamedTuple): - # Set when we have a response to return - url: Optional[str] - response: Optional[UrlResponse] - - # Possibly set if we need to try again - wait_time: Optional[float] - - -def wait_for_url( - urls, - *, - max_wait: float = float("inf"), - timeout: Optional[float] = None, - status_cb: Callable = LOG.debug, # some sources use different log levels - headers_cb: Optional[Callable] = None, - headers_redact=None, - sleep_time: Optional[float] = None, - exception_cb: ExceptionCallback = None, - sleep_time_cb: Optional[Callable[[Any, float], float]] = None, - request_method: str = "", - connect_synchronously: bool = True, - async_delay: float = 0.150, -): - """Wait for a response from one of the urls provided. - - :param urls: List of urls to try - :param max_wait: Roughly the maximum time to wait before giving up - The max time is *actually* len(urls)*timeout as each url will - be tried once and given the timeout provided. - a number <= 0 will always result in only one try - :param timeout: Timeout provided to urlopen - :param status_cb: Callable with string message when a url is not available - :param headers_cb: Callable with single argument of url to get headers - for request. - :param headers_redact: List of header names to redact from the log - :param sleep_time: Amount of time to sleep between retries. If this and - sleep_time_cb are None, the default sleep time defaults to 1 second - and increases by 1 seconds every 5 tries. Cannot be specified along - with `sleep_time_cb`. - :param exception_cb: Callable to handle exception and returns True if - retries are permitted. - :param sleep_time_cb: Callable with 2 arguments (response, loop_n) that - generates the next sleep time. Cannot be specified - along with 'sleep_time`. - :param request_method: Indicates the type of HTTP request: - GET, PUT, or POST - :param connect_synchronously: If false, enables executing requests - in parallel - :param async_delay: Delay before parallel metadata requests, see RFC 6555 - - :return: tuple of (url, response contents), on failure, (False, None) - - :raises: UrlError on unrecoverable error - """ - - def default_sleep_time(_, loop_number: int) -> float: - return sleep_time if sleep_time is not None else loop_number // 5 + 1 - - def timeup(max_wait: float, start_time: float, sleep_time: float = 0): - """Check if time is up based on start time and max wait""" - if max_wait in (float("inf"), None): - return False - return (max_wait <= 0) or ( - time.monotonic() - start_time + sleep_time > max_wait - ) - - def handle_url_response( - response: Optional[UrlResponse], url: Optional[str] - ) -> Tuple[Optional[UrlError], str]: - """Map requests response code/contents to internal "UrlError" type""" - reason = "" - url_exc = None - if not (response and url): - reason = "Request timed out" - url_exc = UrlError(ValueError(reason)) - return url_exc, reason - try: - # Do this first because it can provide more context for the - # exception than what comes later - response._response.raise_for_status() - except requests.exceptions.HTTPError as e: - url_exc = UrlError( - e, - code=e.response.status_code, - headers=e.response.headers, - url=url, - ) - return url_exc, str(e) - if not response.contents: - reason = "empty response [%s]" % (response.code) - url_exc = UrlError( - ValueError(reason), - code=response.code, - headers=response.headers, - url=url, - ) - elif not response.ok(): - # 3xx "errors" wouldn't be covered by the raise_for_status above - reason = "bad status code [%s]" % (response.code) - url_exc = UrlError( - ValueError(reason), - code=response.code, - headers=response.headers, - url=url, - ) - return (url_exc, reason) - - def read_url_handle_exceptions( - url_reader_cb: Callable[ - [Any], Tuple[Optional[str], Optional[UrlResponse]] - ], - urls: Union[str, List[str]], - start_time: int, - exc_cb: ExceptionCallback, - log_cb: Callable, - ) -> HandledResponse: - """Execute request, handle response, optionally log exception""" - reason = "" - url = None - url_exc: Optional[Exception] - try: - url, response = url_reader_cb(urls) - url_exc, reason = handle_url_response(response, url) - if not url_exc: - return HandledResponse(url, response, wait_time=None) - except UrlError as e: - reason = "request error [%s]" % e - url_exc = e - except Exception as e: - reason = "unexpected error [%s]" % e - url_exc = e - time_taken = int(time.monotonic() - start_time) - max_wait_str = "%ss" % max_wait if max_wait else "unlimited" - status_msg = "Calling '%s' failed [%s/%s]: %s" % ( - url or getattr(url_exc, "url", "url"), - time_taken, - max_wait_str, - reason, - ) - log_cb(status_msg) - - return HandledResponse( - url=None, - response=None, - wait_time=( - _handle_error(url_exc, exception_cb=exc_cb) - if isinstance(url_exc, UrlError) - else None - ), - ) - - def read_url_cb(url: str, timeout: int) -> UrlResponse: - return readurl( - url, - headers={} if headers_cb is None else headers_cb(url), - headers_redact=headers_redact, - timeout=timeout, - check_status=False, - request_method=request_method, - ) - - def read_url_serial( - start_time, timeout, exc_cb, log_cb - ) -> HandledResponse: - """iterate over list of urls, request each one and handle responses - and thrown exceptions individually per url - """ - - def url_reader_serial(url: str): - return (url, read_url_cb(url, timeout)) - - wait_times = [] - for url in urls: - now = time.monotonic() - if loop_n != 0 and not must_try_again: - if timeup(max_wait, start_time): - return HandledResponse( - url=None, response=None, wait_time=None - ) - if ( - max_wait is not None - and timeout - and (now + timeout > (start_time + max_wait)) - ): - # shorten timeout to not run way over max_time - timeout = int((start_time + max_wait) - now) - - out = read_url_handle_exceptions( - url_reader_serial, url, start_time, exc_cb, log_cb - ) - if out.response: - return out - elif out.wait_time: - wait_times.append(out.wait_time) - wait_time = max(wait_times) if wait_times else None - return HandledResponse(url=None, response=None, wait_time=wait_time) - - def read_url_parallel( - start_time, timeout, exc_cb, log_cb - ) -> HandledResponse: - """pass list of urls to dual_stack which sends requests in parallel - handle response and exceptions of the first endpoint to respond - """ - url_reader_parallel = partial( - dual_stack, - read_url_cb, - stagger_delay=async_delay, - timeout=timeout, - ) - return read_url_handle_exceptions( - url_reader_parallel, urls, start_time, exc_cb, log_cb - ) - - start_time = time.monotonic() - if sleep_time and sleep_time_cb: - raise ValueError("sleep_time and sleep_time_cb are mutually exclusive") - - # Dual-stack support factored out serial and parallel execution paths to - # allow the retry loop logic to exist separately from the http calls. - # Serial execution should be fundamentally the same as before, but with a - # layer of indirection so that the parallel dual-stack path may use the - # same max timeout logic. - do_read_url = ( - read_url_serial if connect_synchronously else read_url_parallel - ) - - calculate_sleep_time = sleep_time_cb or default_sleep_time - - loop_n: int = 0 - response = None - while True: - resp = do_read_url(start_time, timeout, exception_cb, status_cb) - must_try_again = False - if resp.response: - return resp.url, resp.response.contents - elif resp.wait_time: - time.sleep(resp.wait_time) - loop_n = loop_n + 1 - must_try_again = True - continue - - current_sleep_time = calculate_sleep_time(response, loop_n) - if timeup(max_wait, start_time, current_sleep_time): - break - - loop_n = loop_n + 1 - LOG.debug( - "Please wait %s seconds while we wait to try again", - current_sleep_time, - ) - time.sleep(current_sleep_time) - - # shorten timeout to not run way over max_time - current_time = time.monotonic() - if timeout and current_time + timeout > start_time + max_wait: - timeout = max_wait - (current_time - start_time) - if timeout <= 0: - # We've already exceeded our max_wait. Time to bail. - break - - return False, None - - -class OauthUrlHelper: - def __init__( - self, - consumer_key=None, - token_key=None, - token_secret=None, - consumer_secret=None, - skew_data_file="/run/oauth_skew.json", - ): - self.consumer_key = consumer_key - self.consumer_secret = consumer_secret or "" - self.token_key = token_key - self.token_secret = token_secret - self.skew_data_file = skew_data_file - self._do_oauth = True - self.skew_change_limit = 5 - required = (self.token_key, self.token_secret, self.consumer_key) - if not any(required): - self._do_oauth = False - elif not all(required): - raise ValueError( - "all or none of token_key, token_secret, or " - "consumer_key can be set" - ) - - old = self.read_skew_file() - self.skew_data = old or {} - - def read_skew_file(self): - if self.skew_data_file and os.path.isfile(self.skew_data_file): - with performance.Timed(f"Reading {self.skew_data_file}"), open( - self.skew_data_file, mode="r" - ) as fp: - return json.load(fp) - return None - - def update_skew_file(self, host, value): - # this is not atomic - if not self.skew_data_file: - return - cur = self.read_skew_file() - if cur is None: - cur = {} - cur[host] = value - with performance.Timed(f"Writing {self.skew_data_file}"), open( - self.skew_data_file, mode="w" - ) as fp: - fp.write(json.dumps(cur)) - - def exception_cb(self, msg, exception): - if not ( - isinstance(exception, UrlError) - and (exception.code == 403 or exception.code == 401) - ): - return - - if "date" not in exception.headers: - LOG.warning("Missing header 'date' in %s response", exception.code) - return - - date = exception.headers["date"] - try: - remote_time = time.mktime(parsedate(date)) - except Exception as e: - LOG.warning("Failed to convert datetime '%s': %s", date, e) - return - - skew = int(remote_time - time.time()) - host = urlparse(exception.url).netloc - old_skew = self.skew_data.get(host, 0) - if abs(old_skew - skew) > self.skew_change_limit: - self.update_skew_file(host, skew) - LOG.warning("Setting oauth clockskew for %s to %d", host, skew) - self.skew_data[host] = skew - - return - - def headers_cb(self, url): - if not self._do_oauth: - return {} - - timestamp = None - host = urlparse(url).netloc - if self.skew_data and host in self.skew_data: - timestamp = int(time.time()) + self.skew_data[host] - - return oauth_headers( - url=url, - consumer_key=self.consumer_key, - token_key=self.token_key, - token_secret=self.token_secret, - consumer_secret=self.consumer_secret, - timestamp=timestamp, - ) - - def _wrapped(self, wrapped_func, args, kwargs): - kwargs["headers_cb"] = partial( - self._headers_cb, kwargs.get("headers_cb") - ) - kwargs["exception_cb"] = partial( - self._exception_cb, kwargs.get("exception_cb") - ) - return wrapped_func(*args, **kwargs) - - def wait_for_url(self, *args, **kwargs): - return self._wrapped(wait_for_url, args, kwargs) - - def readurl(self, *args, **kwargs): - return self._wrapped(readurl, args, kwargs) - - def _exception_cb(self, extra_exception_cb, msg, exception): - ret = None - try: - if extra_exception_cb: - ret = extra_exception_cb(msg, exception) - finally: - self.exception_cb(msg, exception) - return ret - - def _headers_cb(self, extra_headers_cb, url): - headers = {} - if extra_headers_cb: - headers = extra_headers_cb(url) - headers.update(self.headers_cb(url)) - return headers - - -def oauth_headers( - url, consumer_key, token_key, token_secret, consumer_secret, timestamp=None -): - try: - import oauthlib.oauth1 as oauth1 - except ImportError as e: - raise NotImplementedError("oauth support is not available") from e - - if timestamp: - timestamp = str(timestamp) - else: - timestamp = None - - client = oauth1.Client( - consumer_key, - client_secret=consumer_secret, - resource_owner_key=token_key, - resource_owner_secret=token_secret, - signature_method=oauth1.SIGNATURE_PLAINTEXT, - timestamp=timestamp, - ) - _uri, signed_headers, _body = client.sign(url) - return signed_headers diff --git a/.pc/cpick-c60771d8-test-pytestify-test_url_helper.py/tests/unittests/test_url_helper.py b/.pc/cpick-c60771d8-test-pytestify-test_url_helper.py/tests/unittests/test_url_helper.py deleted file mode 100644 index 61857173..00000000 --- a/.pc/cpick-c60771d8-test-pytestify-test_url_helper.py/tests/unittests/test_url_helper.py +++ /dev/null @@ -1,956 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. -# pylint: disable=attribute-defined-outside-init - -import logging -from functools import partial -from threading import Event -from time import process_time -from unittest.mock import ANY, call - -import pytest -import requests -import responses - -from cloudinit import util, version -from cloudinit.url_helper import ( - REDACTED, - UrlError, - UrlResponse, - _handle_error, - dual_stack, - oauth_headers, - read_file_or_url, - readurl, - wait_for_url, -) -from tests.unittests.helpers import CiTestCase, mock, skipIf - -try: - import oauthlib - - assert oauthlib # avoid pyflakes error F401: import unused - _missing_oauthlib_dep = False -except ImportError: - _missing_oauthlib_dep = True - - -M_PATH = "cloudinit.url_helper." - - -class TestOAuthHeaders(CiTestCase): - def test_oauth_headers_raises_not_implemented_when_oathlib_missing(self): - """oauth_headers raises a NotImplemented error when oauth absent.""" - with mock.patch.dict("sys.modules", {"oauthlib": None}): - with self.assertRaises(NotImplementedError) as context_manager: - oauth_headers(1, 2, 3, 4, 5) - self.assertEqual( - "oauth support is not available", str(context_manager.exception) - ) - - @skipIf(_missing_oauthlib_dep, "No python-oauthlib dependency") - @mock.patch("oauthlib.oauth1.Client") - def test_oauth_headers_calls_oathlibclient_when_available(self, m_client): - """oauth_headers calls oaut1.hClient.sign with the provided url.""" - - class fakeclient: - def sign(self, url): - # The first and 3rd item of the client.sign tuple are ignored - return ("junk", url, "junk2") - - m_client.return_value = fakeclient() - - return_value = oauth_headers( - "url", - "consumer_key", - "token_key", - "token_secret", - "consumer_secret", - ) - self.assertEqual("url", return_value) - - -class TestReadFileOrUrl(CiTestCase): - - with_logs = True - - def test_read_file_or_url_str_from_file(self): - """Test that str(result.contents) on file is text version of contents. - It should not be "b'data'", but just "'data'" """ - tmpf = self.tmp_path("myfile1") - data = b"This is my file content\n" - util.write_file(tmpf, data, omode="wb") - result = read_file_or_url("file://%s" % tmpf) - self.assertEqual(result.contents, data) - self.assertEqual(str(result), data.decode("utf-8")) - - @responses.activate - def test_read_file_or_url_str_from_url(self): - """Test that str(result.contents) on url is text version of contents. - It should not be "b'data'", but just "'data'" """ - url = "http://hostname/path" - data = b"This is my url content\n" - responses.add(responses.GET, url, data) - result = read_file_or_url(url) - self.assertEqual(result.contents, data) - self.assertEqual(str(result), data.decode("utf-8")) - - @responses.activate - def test_read_file_or_url_str_from_url_streamed(self): - """Test that str(result.contents) on url is text version of contents. - It should not be "b'data'", but just "'data'" """ - url = "http://hostname/path" - data = b"This is my url content\n" - responses.add(responses.GET, url, data) - result = read_file_or_url(url, stream=True) - assert isinstance(result, UrlResponse) - self.assertEqual(result.contents, data) - self.assertEqual(str(result), data.decode("utf-8")) - - @responses.activate - def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self): - """Headers are redacted from logs but unredacted in requests.""" - url = "http://hostname/path" - headers = {"sensitive": "sekret", "server": "blah"} - - def _request_callback(request): - for k in headers.keys(): - self.assertEqual(headers[k], request.headers[k]) - return (200, request.headers, "does_not_matter") - - responses.add_callback(responses.GET, url, callback=_request_callback) - - read_file_or_url(url, headers=headers, headers_redact=["sensitive"]) - logs = self.logs.getvalue() - self.assertIn(REDACTED, logs) - self.assertNotIn("sekret", logs) - - @responses.activate - def test_read_file_or_url_str_from_url_redacts_noheaders(self): - """When no headers_redact, header values are in logs and requests.""" - url = "http://hostname/path" - headers = {"sensitive": "sekret", "server": "blah"} - - def _request_callback(request): - for k in headers.keys(): - self.assertEqual(headers[k], request.headers[k]) - return (200, request.headers, "does_not_matter") - - responses.add_callback(responses.GET, url, callback=_request_callback) - - read_file_or_url(url, headers=headers) - logs = self.logs.getvalue() - self.assertNotIn(REDACTED, logs) - self.assertIn("sekret", logs) - - def test_wb_read_url_defaults_honored_by_read_file_or_url_callers(self): - """Readurl param defaults used when unspecified by read_file_or_url - - Param defaults tested are as follows: - retries: 0, additional headers None beyond default, method: GET, - data: None, check_status: True and allow_redirects: True - """ - url = "http://hostname/path" - - m_response = mock.MagicMock() - - class FakeSessionRaisesHttpError(requests.Session): - @classmethod - def request(cls, **kwargs): - raise requests.exceptions.RequestException("broke") - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - self.assertEqual( - { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": { - "User-Agent": "Cloud-Init/%s" - % (version.version_string()) - }, - "stream": False, - }, - kwargs, - ) - return m_response - - with mock.patch(M_PATH + "requests.Session") as m_session: - m_session.side_effect = [ - FakeSessionRaisesHttpError(), - FakeSession(), - ] - # assert no retries and check_status == True - with self.assertRaises(UrlError) as context_manager: - response = read_file_or_url(url) - self.assertEqual("broke", str(context_manager.exception)) - # assert default headers, method, url and allow_redirects True - # Success on 2nd call with FakeSession - response = read_file_or_url(url) - self.assertEqual(m_response, response._response) - - -class TestReadFileOrUrlParameters: - @mock.patch(M_PATH + "readurl") - @pytest.mark.parametrize( - "timeout", [1, 1.2, "1", (1, None), (1, 1), (None, None)] - ) - def test_read_file_or_url_passes_params_to_readurl( - self, m_readurl, timeout - ): - """read_file_or_url passes all params through to readurl.""" - url = "http://hostname/path" - response = "This is my url content\n" - m_readurl.return_value = response - params = { - "url": url, - "timeout": timeout, - "retries": 2, - "headers": {"somehdr": "val"}, - "data": "data", - "sec_between": 1, - "ssl_details": {"cert_file": "/path/cert.pem"}, - "headers_cb": "headers_cb", - "exception_cb": "exception_cb", - "stream": True, - } - - assert response == read_file_or_url(**params) - params.pop("url") # url is passed in as a positional arg - assert m_readurl.call_args_list == [mock.call(url, **params)] - - @pytest.mark.parametrize( - "readurl_timeout,request_timeout", - [ - (-1, 0), - ("-1", 0), - (None, None), - (1, 1.0), - (1.2, 1.2), - ("1", 1.0), - ((1, None), (1, None)), - ((1, 1), (1, 1)), - ((None, None), (None, None)), - ], - ) - def test_readurl_timeout(self, readurl_timeout, request_timeout): - url = "http://hostname/path" - m_response = mock.MagicMock() - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - expected_kwargs = { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": { - "User-Agent": "Cloud-Init/%s" - % (version.version_string()) - }, - "timeout": request_timeout, - "stream": False, - } - if request_timeout is None: - expected_kwargs.pop("timeout") - - assert kwargs == expected_kwargs - return m_response - - with mock.patch( - M_PATH + "requests.Session", side_effect=[FakeSession()] - ): - response = read_file_or_url(url, timeout=readurl_timeout) - - assert response._response == m_response - - -def assert_time(func, max_time=1): - """Assert function time is bounded by a max (default=1s) - - The following async tests should canceled in under 1ms and have stagger - delay and max_ - It is possible that this could yield a false positive, but this should - basically never happen (esp under normal system load). - """ - start = process_time() - try: - out = func() - finally: - diff = process_time() - start - assert diff < max_time - return out - - -class TestReadUrl: - @pytest.mark.parametrize("headers", [{}, {"Metadata": "true"}]) - def test_headers(self, headers): - url = "http://hostname/path" - m_response = mock.MagicMock() - - expected_headers = headers.copy() - expected_headers["User-Agent"] = "Cloud-Init/%s" % ( - version.version_string() - ) - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - expected_kwargs = { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": expected_headers, - "stream": False, - } - - assert kwargs == expected_kwargs - return m_response - - with mock.patch( - M_PATH + "requests.Session", side_effect=[FakeSession()] - ): - response = readurl(url, headers=headers) - - assert response._response == m_response - - @pytest.mark.parametrize("headers", [{}, {"Metadata": "true"}]) - def test_headers_cb(self, headers): - url = "http://hostname/path" - m_response = mock.MagicMock() - - expected_headers = headers.copy() - expected_headers["User-Agent"] = "Cloud-Init/%s" % ( - version.version_string() - ) - headers_cb = lambda _: headers - - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): - expected_kwargs = { - "url": url, - "allow_redirects": True, - "method": "GET", - "headers": expected_headers, - "stream": False, - } - - assert kwargs == expected_kwargs - return m_response - - with mock.patch( - M_PATH + "requests.Session", side_effect=[FakeSession()] - ): - response = readurl(url, headers_cb=headers_cb) - - assert response._response == m_response - - def test_error_no_cb(self, mocker): - response = requests.Response() - response.status_code = 500 - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.return_value = response - - with pytest.raises(UrlError) as e: - readurl("http://some/path") - assert e.value.code == 500 - - def test_error_cb_true(self, mocker): - mocker.patch("time.sleep") - - bad_response = requests.Response() - bad_response.status_code = 500 - bad_response._content = b"oh noes!" - good_response = requests.Response() - good_response.status_code = 200 - good_response._content = b"yay" - - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.side_effect = (bad_response, good_response) - - readurl("http://some/path", retries=1, exception_cb=lambda _: True) - assert m_request.call_count == 2 - - def test_error_cb_false(self, mocker): - mocker.patch("time.sleep") - - bad_response = requests.Response() - bad_response.status_code = 500 - bad_response._content = b"oh noes!" - - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.return_value = bad_response - - with pytest.raises(UrlError): - readurl( - "http://some/path", retries=1, exception_cb=lambda _: False - ) - assert m_request.call_count == 1 - - def test_exception_503(self, mocker): - mocker.patch("time.sleep") - - retry_response = requests.Response() - retry_response.status_code = 503 - retry_response._content = b"try again" - good_response = requests.Response() - good_response.status_code = 200 - good_response._content = b"good" - m_request = mocker.patch("requests.Session.request", autospec=True) - m_request.side_effect = (retry_response, retry_response, good_response) - - readurl("http://some/path") - assert m_request.call_count == 3 - - -event = Event() - - -class TestDualStack: - """Async testing suggestions welcome - these all rely on time-bounded - assertions (via threading.Event) to prove ordering - """ - - @pytest.mark.parametrize( - ["func", "addresses", "stagger_delay", "timeout", "expected_val"], - [ - # Assert order based on timeout - (lambda x, _: x, ("one", "two"), 1, 1, "one"), - # Assert timeout results in (None, None) - (lambda _a, _b: event.wait(1), ("one", "two"), 1, 0, None), - ( - lambda a, _b: 1 / 0 if a == "one" else a, - ("one", "two"), - 0, - 1, - "two", - ), - # Assert that exception in func is only raised - # if neither thread gets a valid result - ( - lambda a, _b: 1 / 0 if a == "two" else a, - ("one", "two"), - 0, - 1, - "one", - ), - # simulate a slow response to verify correct order - ( - lambda x, _: event.wait(1) if x != "two" else x, - ("one", "two"), - 0, - 1, - "two", - ), - # simulate a slow response to verify correct order - ( - lambda x, _: event.wait(1) if x != "tri" else x, - ("one", "two", "tri"), - 0, - 1, - "tri", - ), - ], - ) - def test_dual_stack( - self, - func, - addresses, - stagger_delay, - timeout, - expected_val, - ): - """Assert various failure modes behave as expected""" - event.clear() - - gen = partial( - dual_stack, - func, - addresses, - stagger_delay=stagger_delay, - timeout=timeout, - ) - _, result = assert_time(gen) - assert expected_val == result - - event.set() - - @pytest.mark.parametrize( - [ - "func", - "addresses", - "stagger_delay", - "timeout", - "message", - "expected_exc", - ], - [ - ( - lambda _a, _b: 1 / 0, - ("¯\\_(ツ)_/¯", "(╯°□°）╯︵ ┻━┻"), - 0, - 1, - "division by zero", - ZeroDivisionError, - ), - ( - lambda _a, _b: 1 / 0, - ("it", "really", "doesn't"), - 0, - 1, - "division by zero", - ZeroDivisionError, - ), - ( - lambda _a, _b: [][0], # pylint: disable=E0643 - ("matter", "these"), - 0, - 1, - "list index out of range", - IndexError, - ), - ( - lambda _a, _b: (_ for _ in ()).throw( - Exception("soapstone is not effective soap") - ), - ("are", "ignored"), - 0, - 1, - "soapstone is not effective soap", - Exception, - ), - ], - ) - def test_dual_stack_exceptions( - self, - func, - addresses, - stagger_delay, - timeout, - message, - expected_exc, - caplog, - ): - # Context: - # - # currently if all threads experience exception - # dual_stack() logs an error containing all exceptions - # but only raises the last exception to occur - # Verify "best effort behavior" - # dual_stack will temporarily ignore an exception in any of the - # request threads in hopes that a later thread will succeed - # this behavior is intended to allow a requests.ConnectionError - # exception from on endpoint to occur without preventing another - # thread from succeeding - event.clear() - - # Note: python3.6 repr(Exception("test")) produces different output - # than later versions, so we cannot match exact message without - # some ugly manual exception repr() function, which I'd rather not do - # in dual_stack(), so we recreate expected messages manually here - # in a version-independant way for testing, the extra comma on old - # versions won't hurt anything - exc_list = str([expected_exc(message) for _ in addresses]) - expected_msg = f"Exception(s) {exc_list} during request" - gen = partial( - dual_stack, - func, - addresses, - stagger_delay=stagger_delay, - timeout=timeout, - ) - with pytest.raises(expected_exc): - gen() # 1 - with caplog.at_level(logging.DEBUG): - try: - gen() # 2 - except expected_exc: - pass - finally: - assert 2 == len(caplog.records) - assert 2 == caplog.text.count(expected_msg) - event.set() - - def test_dual_stack_staggered(self): - """Assert expected call intervals occur""" - stagger = 0.1 - with mock.patch(M_PATH + "_run_func_with_delay") as delay_func: - - def identity_of_first_arg(x, _): - return x - - dual_stack( - identity_of_first_arg, - ["you", "and", "me", "and", "dog"], - stagger_delay=stagger, - timeout=1, - ) - - # ensure that stagger delay for each call is made with args: - # [ 0 * N, 1 * N, 2 * N, 3 * N, 4 * N, 5 * N] where N = stagger - # it appears that without an explicit wait/join we can't assert - # number of calls - calls = [ - call( - func=identity_of_first_arg, - addr="you", - timeout=1, - event=ANY, - delay=stagger * 0, - ), - call( - func=identity_of_first_arg, - addr="and", - timeout=1, - event=ANY, - delay=stagger * 1, - ), - call( - func=identity_of_first_arg, - addr="me", - timeout=1, - event=ANY, - delay=stagger * 2, - ), - call( - func=identity_of_first_arg, - addr="and", - timeout=1, - event=ANY, - delay=stagger * 3, - ), - call( - func=identity_of_first_arg, - addr="dog", - timeout=1, - event=ANY, - delay=stagger * 4, - ), - ] - num_calls = 0 - for call_instance in calls: - if call_instance in delay_func.call_args_list: - num_calls += 1 - - # we can't know the order of the submitted functions' execution - # we can't know how many of the submitted functions get called - # in advance - # - # we _do_ know what the possible arg combinations are - # we _do_ know from the mocked function how many got called - # assert that all calls that occurred had known valid arguments - # by checking for the correct number of matches - assert num_calls == len(delay_func.call_args_list) - - -ADDR1 = "https://addr1/" -SLEEP1 = "https://sleep1/" -SLEEP2 = "https://sleep2/" - - -class TestWaitForUrl: - success = "SUCCESS" - fail = "FAIL" - event = Event() - - @pytest.fixture - def retry_mocks(self, mocker): - self.mock_time_value = 0 - m_readurl = mocker.patch( - f"{M_PATH}readurl", side_effect=self.readurl_side_effect - ) - m_sleep = mocker.patch( - f"{M_PATH}time.sleep", side_effect=self.sleep_side_effect - ) - mocker.patch( - f"{M_PATH}time.monotonic", side_effect=self.time_side_effect - ) - - yield m_readurl, m_sleep - - self.mock_time_value = 0 - - @classmethod - def response_wait(cls, _request): - cls.event.wait(0.1) - return (500, {"request-id": "1"}, cls.fail) - - @classmethod - def response_nowait(cls, _request): - return (200, {"request-id": "0"}, cls.success) - - @pytest.mark.parametrize( - ["addresses", "expected_address_index", "response"], - [ - # Use timeout to test ordering happens as expected - ((ADDR1, SLEEP1), 0, "SUCCESS"), - ((SLEEP1, ADDR1), 1, "SUCCESS"), - ((SLEEP1, SLEEP2, ADDR1), 2, "SUCCESS"), - ((ADDR1, SLEEP1, SLEEP2), 0, "SUCCESS"), - ], - ) - @responses.activate - def test_order(self, addresses, expected_address_index, response): - """Check that the first response gets returned. Simulate a - non-responding endpoint with a response that has a one second wait. - - If this test proves flaky, increase wait time. Since it is async, - increasing wait time for the non-responding endpoint should not - increase total test time, assuming async_delay=0 is used and at least - one non-waiting endpoint is registered with responses. - Subsequent tests will continue execution after the first response is - received. - """ - self.event.clear() - for address in set(addresses): - responses.add_callback( - responses.GET, - address, - callback=( - self.response_wait - if "sleep" in address - else self.response_nowait - ), - content_type="application/json", - ) - - # Use async_delay=0.0 to avoid adding unnecessary time to tests - # In practice a value such as 0.150 is used - url, response_contents = wait_for_url( - urls=addresses, - max_wait=2, - timeout=0.3, - connect_synchronously=False, - async_delay=0.0, - ) - self.event.set() - - # Test for timeout (no responding endpoint) - assert addresses[expected_address_index] == url - assert response.encode() == response_contents - - @responses.activate - def test_timeout(self): - """If no endpoint responds in time, expect no response""" - - self.event.clear() - addresses = [SLEEP1, SLEEP2] - for address in set(addresses): - responses.add_callback( - responses.GET, - address, - callback=( - requests.ConnectTimeout - if "sleep" in address - else self.response_nowait - ), - content_type="application/json", - ) - - # Use async_delay=0.0 to avoid adding unnecessary time to tests - url, response_contents = wait_for_url( - urls=addresses, - max_wait=1, - timeout=1, - connect_synchronously=False, - async_delay=0, - ) - self.event.set() - assert not url - assert not response_contents - - def test_explicit_arguments(self, retry_mocks): - """Ensure that explicit arguments are respected""" - m_readurl, m_sleep = retry_mocks - wait_for_url( - urls=["http://localhost/"], - max_wait=23, - timeout=5, - sleep_time=3, - ) - - assert len(m_readurl.call_args_list) == 3 - assert len(m_sleep.call_args_list) == 2 - - for readurl_call in m_readurl.call_args_list: - assert readurl_call[1]["timeout"] == 5 - for sleep_call in m_sleep.call_args_list: - assert sleep_call[0][0] == 3 - - # Call 1 starts 0 - # Call 2 starts at 8-ish after 5 second timeout and 3 second sleep - # Call 3 starts at 16-ish for same reasons - # The 5 second timeout puts us at 21-ish and now we break - # because 21-ish + the sleep time puts us over max wait of 23 - assert pytest.approx(self.mock_time_value) == 21 - - def test_shortened_timeout(self, retry_mocks): - """Test that we shorten the last timeout to align with max_wait""" - m_readurl, _m_sleep = retry_mocks - wait_for_url( - urls=["http://localhost/"], max_wait=10, timeout=9, sleep_time=0 - ) - - assert len(m_readurl.call_args_list) == 2 - assert m_readurl.call_args_list[-1][1]["timeout"] == pytest.approx(1) - - def test_default_sleep_time(self, retry_mocks): - """Test default sleep behavior when not specified""" - _m_readurl, m_sleep = retry_mocks - wait_for_url( - urls=["http://localhost/"], - max_wait=50, - timeout=1, - ) - - expected_sleep_times = [1] * 5 + [2] * 5 + [3] * 5 - actual_sleep_times = [ - m_sleep.call_args_list[i][0][0] - for i in range(len(m_sleep.call_args_list)) - ] - assert actual_sleep_times == expected_sleep_times - - @responses.activate - def test_503(self, mocker): - mocker.patch("time.sleep") - - for _ in range(10): - responses.add( - method=responses.GET, - url="http://hi/", - status=503, - body=b"try again", - ) - responses.add( - method=responses.GET, - url="http://hi/", - status=200, - body=b"good", - ) - - assert wait_for_url(urls=["http://hi/"], max_wait=0.0001)[1] == b"good" - - @responses.activate - def test_503_async(self, mocker): - mocker.patch("time.sleep") - - for _ in range(10): - responses.add( - method=responses.GET, - url="http://hi/", - status=503, - body=b"try again", - ) - responses.add( - method=responses.GET, - url="http://hi2/", - status=503, - body="try again", - ) - responses.add( - method=responses.GET, - url="http://hi/", - status=200, - body=b"good", - ) - responses.add( - method=responses.GET, - url="http://hi2/", - status=200, - body=b"good", - ) - - assert ( - wait_for_url( - urls=["http://hi/", "http://hi2/"], - max_wait=0.0001, - async_delay=0, - connect_synchronously=False, - )[1] - == b"good" - ) - - # These side effect methods are a way of having a somewhat predictable - # output for time.monotonic(). Otherwise, we have to track too many calls - # to time.monotonic() and unrelated changes to code being called could - # cause these tests to fail. - # 0.0000001 is added to simulate additional execution time but keep it - # small enough for pytest.approx() to work - def sleep_side_effect(self, sleep_time): - self.mock_time_value += sleep_time + 0.0000001 - - def time_side_effect(self): - return self.mock_time_value - - def readurl_side_effect(self, *args, **kwargs): - if "timeout" in kwargs: - self.mock_time_value += kwargs["timeout"] + 0.0000001 - raise UrlError("test") - - -class TestHandleError: - def test_handle_error_no_cb(self): - """Test no callback.""" - assert _handle_error(UrlError("test")) is None - - def test_handle_error_cb_false(self): - """Test callback returning False.""" - with pytest.raises(UrlError) as e: - _handle_error(UrlError("test"), exception_cb=lambda _: False) - assert str(e.value) == "test" - - def test_handle_error_cb_true(self): - """Test callback returning True.""" - assert ( - _handle_error(UrlError("test"), exception_cb=lambda _: True) - ) is None - - def test_handle_503(self, caplog): - """Test 503 with no callback.""" - assert _handle_error(UrlError("test", code=503)) == 1 - assert "Unable to introspect response header" in caplog.text - - def test_handle_503_with_retry_header(self): - """Test 503 with a retry integer value.""" - assert ( - _handle_error( - UrlError("test", code=503, headers={"Retry-After": 5}) - ) - == 5 - ) - - def test_handle_503_with_retry_header_in_past(self, caplog): - """Test 503 with date in the past.""" - assert ( - _handle_error( - UrlError( - "test", - code=503, - headers={"Retry-After": "Fri, 31 Dec 1999 23:59:59 GMT"}, - ) - ) - == 1 - ) - assert "Retry-After header value is in the past" in caplog.text - - def test_handle_503_cb_true(self): - """Test 503 with a callback returning True.""" - assert ( - _handle_error( - UrlError("test", code=503), - exception_cb=lambda _: True, - ) - is None - ) - - def test_handle_503_cb_false(self): - """Test 503 with a callback returning False.""" - assert ( - _handle_error( - UrlError("test", code=503), - exception_cb=lambda _: False, - ) - == 1 - ) diff --git a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/cloudinit/cmd/devel/hotplug_hook.py b/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/cloudinit/cmd/devel/hotplug_hook.py deleted file mode 100755 index 8e839cb1..00000000 --- a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/cloudinit/cmd/devel/hotplug_hook.py +++ /dev/null @@ -1,353 +0,0 @@ -#!/usr/bin/env python3 - -# This file is part of cloud-init. See LICENSE file for license information. -"""Handle reconfiguration on hotplug events.""" -import abc -import argparse -import json -import logging -import os -import sys -import time - -from cloudinit import reporting, stages, util -from cloudinit.config.cc_install_hotplug import install_hotplug -from cloudinit.event import EventScope, EventType -from cloudinit.log import loggers -from cloudinit.net import read_sys_net_safe -from cloudinit.net.network_state import parse_net_config_data -from cloudinit.reporting import events -from cloudinit.sources import DataSource, DataSourceNotFoundException -from cloudinit.stages import Init - -LOG = logging.getLogger(__name__) -NAME = "hotplug-hook" - - -def get_parser(parser=None): - """Build or extend an arg parser for hotplug-hook utility. - - @param parser: Optional existing ArgumentParser instance representing the - subcommand which will be extended to support the args of this utility. - - @returns: ArgumentParser with proper argument configuration. - """ - if not parser: - parser = argparse.ArgumentParser(prog=NAME, description=__doc__) - - parser.description = __doc__ - parser.add_argument( - "-s", - "--subsystem", - required=True, - help="subsystem to act on", - choices=["net"], - ) - - subparsers = parser.add_subparsers( - title="Hotplug Action", dest="hotplug_action" - ) - subparsers.required = True - - subparsers.add_parser( - "query", help="Query if hotplug is enabled for given subsystem." - ) - - parser_handle = subparsers.add_parser( - "handle", help="Handle the hotplug event." - ) - parser_handle.add_argument( - "-d", - "--devpath", - required=True, - metavar="PATH", - help="Sysfs path to hotplugged device", - ) - parser_handle.add_argument( - "-u", - "--udevaction", - required=True, - help="Specify action to take.", - choices=["add", "remove"], - ) - - subparsers.add_parser( - "enable", help="Enable hotplug for a given subsystem." - ) - - return parser - - -class UeventHandler(abc.ABC): - def __init__(self, id, datasource, devpath, action, success_fn): - self.id = id - self.datasource: DataSource = datasource - self.devpath = devpath - self.action = action - self.success_fn = success_fn - - @abc.abstractmethod - def apply(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def config(self): - raise NotImplementedError() - - @abc.abstractmethod - def device_detected(self) -> bool: - raise NotImplementedError() - - def detect_hotplugged_device(self): - detect_presence = None - if self.action == "add": - detect_presence = True - elif self.action == "remove": - detect_presence = False - else: - raise ValueError("Unknown action: %s" % self.action) - - if detect_presence != self.device_detected(): - raise RuntimeError( - "Failed to detect %s in updated metadata" % self.id - ) - - def success(self): - return self.success_fn() - - def update_metadata(self): - result = self.datasource.update_metadata_if_supported( - [EventType.HOTPLUG] - ) - if not result: - raise RuntimeError( - "Datasource %s not updated for event %s" - % (self.datasource, EventType.HOTPLUG) - ) - return result - - -class NetHandler(UeventHandler): - def __init__(self, datasource, devpath, action, success_fn): - # convert devpath to mac address - id = read_sys_net_safe(os.path.basename(devpath), "address") - super().__init__(id, datasource, devpath, action, success_fn) - - def apply(self): - self.datasource.distro.apply_network_config( - self.config, - bring_up=False, - ) - interface_name = os.path.basename(self.devpath) - activator = self.datasource.distro.network_activator() - if self.action == "add": - if not activator.bring_up_interface(interface_name): - raise RuntimeError( - "Failed to bring up device: {}".format(self.devpath) - ) - elif self.action == "remove": - if not activator.bring_down_interface(interface_name): - raise RuntimeError( - "Failed to bring down device: {}".format(self.devpath) - ) - - @property - def config(self): - return self.datasource.network_config - - def device_detected(self) -> bool: - netstate = parse_net_config_data(self.config) - found = [ - iface - for iface in netstate.iter_interfaces() - if iface.get("mac_address") == self.id - ] - LOG.debug("Ifaces with ID=%s : %s", self.id, found) - return len(found) > 0 - - -SUBSYSTEM_PROPERTIES_MAP = { - "net": (NetHandler, EventScope.NETWORK), -} - - -def is_enabled(hotplug_init, subsystem): - try: - scope = SUBSYSTEM_PROPERTIES_MAP[subsystem][1] - except KeyError as e: - raise RuntimeError( - "hotplug-hook: cannot handle events for subsystem: {}".format( - subsystem - ) - ) from e - - return stages.update_event_enabled( - datasource=hotplug_init.datasource, - cfg=hotplug_init.cfg, - event_source_type=EventType.HOTPLUG, - scope=scope, - ) - - -def initialize_datasource(hotplug_init: Init, subsystem: str): - LOG.debug("Fetching datasource") - datasource = hotplug_init.fetch(existing="trust") - - if not datasource.get_supported_events([EventType.HOTPLUG]): - LOG.debug("hotplug not supported for event of type %s", subsystem) - return - - if not is_enabled(hotplug_init, subsystem): - LOG.debug("hotplug not enabled for event of type %s", subsystem) - return - return datasource - - -def handle_hotplug(hotplug_init: Init, devpath, subsystem, udevaction): - datasource = initialize_datasource(hotplug_init, subsystem) - if not datasource: - return - handler_cls = SUBSYSTEM_PROPERTIES_MAP[subsystem][0] - LOG.debug("Creating %s event handler", subsystem) - event_handler: UeventHandler = handler_cls( - datasource=datasource, - devpath=devpath, - action=udevaction, - success_fn=hotplug_init._write_to_cache, - ) - wait_times = [1, 3, 5, 10, 30] - last_exception = Exception("Bug while processing hotplug event.") - for attempt, wait in enumerate(wait_times): - LOG.debug( - "subsystem=%s update attempt %s/%s", - subsystem, - attempt, - len(wait_times), - ) - try: - LOG.debug("Refreshing metadata") - event_handler.update_metadata() - if not datasource.skip_hotplug_detect: - LOG.debug("Detecting device in updated metadata") - event_handler.detect_hotplugged_device() - LOG.debug("Applying config change") - event_handler.apply() - LOG.debug("Updating cache") - event_handler.success() - break - except Exception as e: - LOG.debug("Exception while processing hotplug event. %s", e) - time.sleep(wait) - last_exception = e - else: - raise last_exception - - -def enable_hotplug(hotplug_init: Init, subsystem) -> bool: - datasource = hotplug_init.fetch(existing="trust") - if not datasource: - return False - scope = SUBSYSTEM_PROPERTIES_MAP[subsystem][1] - hotplug_supported = EventType.HOTPLUG in ( - datasource.get_supported_events([EventType.HOTPLUG]).get(scope, set()) - ) - if not hotplug_supported: - print( - f"hotplug not supported for event of {subsystem}", file=sys.stderr - ) - return False - hotplug_enabled_file = util.read_hotplug_enabled_file(hotplug_init.paths) - if scope.value in hotplug_enabled_file["scopes"]: - print( - f"Not installing hotplug for event of type {subsystem}." - " Reason: Already done.", - file=sys.stderr, - ) - return True - - hotplug_enabled_file["scopes"].append(scope.value) - util.write_file( - hotplug_init.paths.get_cpath("hotplug.enabled"), - json.dumps(hotplug_enabled_file), - omode="w", - mode=0o640, - ) - install_hotplug( - datasource, network_hotplug_enabled=True, cfg=hotplug_init.cfg - ) - return True - - -def handle_args(name, args): - # Note that if an exception happens between now and when logging is - # setup, we'll only see it in the journal - hotplug_reporter = events.ReportEventStack( - name, __doc__, reporting_enabled=True - ) - - hotplug_init = Init(ds_deps=[], reporter=hotplug_reporter) - hotplug_init.read_cfg() - - loggers.setup_logging(hotplug_init.cfg) - if "reporting" in hotplug_init.cfg: - reporting.update_configuration(hotplug_init.cfg.get("reporting")) - # Logging isn't going to be setup until now - LOG.debug( - "%s called with the following arguments: {" - "hotplug_action: %s, subsystem: %s, udevaction: %s, devpath: %s}", - name, - args.hotplug_action, - args.subsystem, - args.udevaction if "udevaction" in args else None, - args.devpath if "devpath" in args else None, - ) - - with hotplug_reporter: - try: - if args.hotplug_action == "query": - try: - datasource = initialize_datasource( - hotplug_init, args.subsystem - ) - except DataSourceNotFoundException: - print( - "Unable to determine hotplug state. No datasource " - "detected" - ) - sys.exit(1) - print("enabled" if datasource else "disabled") - elif args.hotplug_action == "handle": - handle_hotplug( - hotplug_init=hotplug_init, - devpath=args.devpath, - subsystem=args.subsystem, - udevaction=args.udevaction, - ) - else: - if os.getuid() != 0: - sys.stderr.write( - "Root is required. Try prepending your command with" - " sudo.\n" - ) - sys.exit(1) - if not enable_hotplug( - hotplug_init=hotplug_init, subsystem=args.subsystem - ): - sys.exit(1) - print( - f"Enabled cloud-init hotplug for " - f"subsystem={args.subsystem}" - ) - - except Exception: - LOG.exception("Received fatal exception handling hotplug!") - raise - - LOG.debug("Exiting hotplug handler") - reporting.flush_events() - - -if __name__ == "__main__": - args = get_parser().parse_args() - handle_args(NAME, args) diff --git a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/cloudinit/sources/DataSourceEc2.py b/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/cloudinit/sources/DataSourceEc2.py deleted file mode 100644 index 7893f0eb..00000000 --- a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/cloudinit/sources/DataSourceEc2.py +++ /dev/null @@ -1,1215 +0,0 @@ -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# Author: Joshua Harlow -# -# This file is part of cloud-init. See LICENSE file for license information. - -import copy -import logging -import os -import time -import uuid -from contextlib import suppress -from typing import Dict, List, Literal - -from cloudinit import dmi, net, sources -from cloudinit import url_helper as uhelp -from cloudinit import util, warnings -from cloudinit.distros import Distro -from cloudinit.event import EventScope, EventType -from cloudinit.net import netplan -from cloudinit.net.dhcp import NoDHCPLeaseError -from cloudinit.net.ephemeral import EphemeralIPNetwork -from cloudinit.sources import NicOrder -from cloudinit.sources.helpers import ec2 - -LOG = logging.getLogger(__name__) - -STRICT_ID_PATH = ("datasource", "Ec2", "strict_id") -STRICT_ID_DEFAULT = "warn" - - -class CloudNames: - ALIYUN = "aliyun" - AWS = "aws" - BRIGHTBOX = "brightbox" - ZSTACK = "zstack" - E24CLOUD = "e24cloud" - OUTSCALE = "outscale" - # UNKNOWN indicates no positive id. If strict_id is 'warn' or 'false', - # then an attempt at the Ec2 Metadata service will be made. - UNKNOWN = "unknown" - # NO_EC2_METADATA indicates this platform does not have a Ec2 metadata - # service available. No attempt at the Ec2 Metadata service will be made. - NO_EC2_METADATA = "no-ec2-metadata" - - -# Drop when LP: #1988157 tag handling is fixed -def skip_404_tag_errors(exception): - return exception.code == 404 and "meta-data/tags/" in exception.url - - -# Cloud platforms that support IMDSv2 style metadata server -IDMSV2_SUPPORTED_CLOUD_PLATFORMS = [CloudNames.AWS, CloudNames.ALIYUN] - -# Only trigger hook-hotplug on NICs with Ec2 drivers. Avoid triggering -# it on docker virtual NICs and the like. LP: #1946003 -_EXTRA_HOTPLUG_UDEV_RULES = """ -ENV{ID_NET_DRIVER}=="vif|ena|ixgbevf", GOTO="cloudinit_hook" -GOTO="cloudinit_end" -""" - - -class DataSourceEc2(sources.DataSource): - dsname = "Ec2" - # Default metadata urls that will be used if none are provided - # They will be checked for 'resolveability' and some of the - # following may be discarded if they do not resolve - metadata_urls = [ - "http://169.254.169.254", - "http://[fd00:ec2::254]", - "http://instance-data.:8773", - ] - - # The minimum supported metadata_version from the ec2 metadata apis - min_metadata_version = "2009-04-04" - - # Priority ordered list of additional metadata versions which will be tried - # for extended metadata content. IPv6 support comes in 2016-09-02. - # Tags support comes in 2021-03-23. - extended_metadata_versions: List[str] = [ - "2021-03-23", - "2018-09-24", - "2016-09-02", - ] - - # Setup read_url parameters per get_url_params. - url_max_wait = 240 - url_timeout = 50 - - _api_token = None # API token for accessing the metadata service - _network_config = sources.UNSET # Used to cache calculated network cfg v1 - - # Whether we want to get network configuration from the metadata service. - perform_dhcp_setup = False - - supported_update_events = { - EventScope.NETWORK: { - EventType.BOOT_NEW_INSTANCE, - EventType.BOOT, - EventType.BOOT_LEGACY, - EventType.HOTPLUG, - } - } - - default_update_events = { - EventScope.NETWORK: { - EventType.BOOT_NEW_INSTANCE, - EventType.HOTPLUG, - } - } - - extra_hotplug_udev_rules = _EXTRA_HOTPLUG_UDEV_RULES - - def __init__(self, sys_cfg, distro, paths): - super(DataSourceEc2, self).__init__(sys_cfg, distro, paths) - self.metadata_address = None - self.identity = None - self._fallback_nic_order = NicOrder.MAC - - def _unpickle(self, ci_pkl_version: int) -> None: - super()._unpickle(ci_pkl_version) - self.extra_hotplug_udev_rules = _EXTRA_HOTPLUG_UDEV_RULES - self._fallback_nic_order = NicOrder.MAC - - def _get_cloud_name(self): - """Return the cloud name as identified during _get_data.""" - return identify_platform() - - def _get_data(self): - strict_mode, _sleep = read_strict_mode( - util.get_cfg_by_path( - self.sys_cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT - ), - ("warn", None), - ) - - LOG.debug( - "strict_mode: %s, cloud_name=%s cloud_platform=%s", - strict_mode, - self.cloud_name, - self.platform, - ) - if strict_mode == "true" and self.cloud_name == CloudNames.UNKNOWN: - return False - elif self.cloud_name == CloudNames.NO_EC2_METADATA: - return False - - if self.perform_dhcp_setup: # Setup networking in init-local stage. - if util.is_FreeBSD(): - LOG.debug("FreeBSD doesn't support running dhclient with -sf") - return False - try: - with EphemeralIPNetwork( - self.distro, - self.distro.fallback_interface, - ipv4=True, - ipv6=True, - ) as netw: - self._crawled_metadata = self.crawl_metadata() - LOG.debug( - "Crawled metadata service%s", - f" {netw.state_msg}" if netw.state_msg else "", - ) - - except NoDHCPLeaseError: - return False - else: - self._crawled_metadata = self.crawl_metadata() - if not self._crawled_metadata: - return False - self.metadata = self._crawled_metadata.get("meta-data", None) - self.userdata_raw = self._crawled_metadata.get("user-data", None) - self.identity = ( - self._crawled_metadata.get("dynamic", {}) - .get("instance-identity", {}) - .get("document", {}) - ) - return True - - def is_classic_instance(self): - """Report if this instance type is Ec2 Classic (non-vpc).""" - if not self.metadata: - # Can return False on inconclusive as we are also called in - # network_config where metadata will be present. - # Secondary call site is in packaging postinst script. - return False - ifaces_md = self.metadata.get("network", {}).get("interfaces", {}) - for _mac, mac_data in ifaces_md.get("macs", {}).items(): - if "vpc-id" in mac_data: - return False - return True - - @property - def launch_index(self): - if not self.metadata: - return None - return self.metadata.get("ami-launch-index") - - @property - def platform(self): - if not self._platform_type: - self._platform_type = DataSourceEc2.dsname.lower() - return self._platform_type - - # IMDSv2 related parameters from the ec2 metadata api document - @property - def api_token_route(self): - return "latest/api/token" - - @property - def imdsv2_token_ttl_seconds(self): - return "21600" - - @property - def imdsv2_token_put_header(self): - return "X-aws-ec2-metadata-token" - - @property - def imdsv2_token_req_header(self): - return self.imdsv2_token_put_header + "-ttl-seconds" - - @property - def imdsv2_token_redact(self): - return [self.imdsv2_token_put_header, self.imdsv2_token_req_header] - - def get_metadata_api_version(self): - """Get the best supported api version from the metadata service. - - Loop through all extended support metadata versions in order and - return the most-fully featured metadata api version discovered. - - If extended_metadata_versions aren't present, return the datasource's - min_metadata_version. - """ - # Assumes metadata service is already up - url_tmpl = "{0}/{1}/meta-data/instance-id" - headers = self._get_headers() - for api_ver in self.extended_metadata_versions: - url = url_tmpl.format(self.metadata_address, api_ver) - try: - resp = uhelp.readurl( - url=url, - headers=headers, - headers_redact=self.imdsv2_token_redact, - ) - except uhelp.UrlError as e: - LOG.debug("url %s raised exception %s", url, e) - else: - if resp.code == 200: - LOG.debug("Found preferred metadata version %s", api_ver) - return api_ver - elif resp.code == 404: - msg = "Metadata api version %s not present. Headers: %s" - LOG.debug(msg, api_ver, resp.headers) - return self.min_metadata_version - - def get_instance_id(self): - if self.cloud_name == CloudNames.AWS: - # Prefer the ID from the instance identity document, but fall back - if not getattr(self, "identity", None): - # If re-using cached datasource, it's get_data run didn't - # setup self.identity. So we need to do that now. - api_version = self.get_metadata_api_version() - self.identity = ec2.get_instance_identity( - api_version, - self.metadata_address, - headers_cb=self._get_headers, - headers_redact=self.imdsv2_token_redact, - exception_cb=self._refresh_stale_aws_token_cb, - ).get("document", {}) - return self.identity.get( - "instanceId", self.metadata["instance-id"] - ) - else: - return self.metadata["instance-id"] - - def _maybe_fetch_api_token(self, mdurls): - """Get an API token for EC2 Instance Metadata Service. - - On EC2. IMDS will always answer an API token, unless - the instance owner has disabled the IMDS HTTP endpoint or - the network topology conflicts with the configured hop-limit. - """ - if self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS: - return - - urls = [] - url2base = {} - url_path = self.api_token_route - request_method = "PUT" - for url in mdurls: - cur = "{0}/{1}".format(url, url_path) - urls.append(cur) - url2base[cur] = url - - # use the self._imds_exception_cb to check for Read errors - LOG.debug("Fetching Ec2 IMDSv2 API Token") - - response = None - url = None - url_params = self.get_url_params() - try: - url, response = uhelp.wait_for_url( - urls=urls, - max_wait=url_params.max_wait_seconds, - timeout=url_params.timeout_seconds, - status_cb=LOG.warning, - headers_cb=self._get_headers, - exception_cb=self._token_exception_cb, - request_method=request_method, - headers_redact=self.imdsv2_token_redact, - connect_synchronously=False, - ) - except uhelp.UrlError: - # We use the raised exception to interrupt the retry loop. - # Nothing else to do here. - pass - - if url and response: - self._api_token = response - return url2base[url] - - # If we get here, then wait_for_url timed out, waiting for IMDS - # or the IMDS HTTP endpoint is disabled - LOG.error("Unable to get response from urls: %s", urls) - return None - - def wait_for_metadata_service(self): - urls = [] - start_time = 0 - mcfg = self.ds_cfg - - url_params = self.get_url_params() - if url_params.max_wait_seconds <= 0: - return False - - # Remove addresses from the list that wont resolve. - mdurls = mcfg.get("metadata_urls", self.metadata_urls) - filtered = [x for x in mdurls if util.is_resolvable_url(x)] - - if set(filtered) != set(mdurls): - LOG.debug( - "Removed the following from metadata urls: %s", - list((set(mdurls) - set(filtered))), - ) - - if len(filtered): - mdurls = filtered - else: - LOG.warning("Empty metadata url list! using default list") - mdurls = self.metadata_urls - - # try the api token path first - metadata_address = self._maybe_fetch_api_token(mdurls) - # When running on EC2, we always access IMDS with an API token. - # If we could not get an API token, then we assume the IMDS - # endpoint was disabled and we move on without a data source. - # Fallback to IMDSv1 if not running on EC2 - if ( - not metadata_address - and self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS - ): - # if we can't get a token, use instance-id path - url2base = {} - url_path = "{ver}/meta-data/instance-id".format( - ver=self.min_metadata_version - ) - request_method = "GET" - for url in mdurls: - cur = "{0}/{1}".format(url, url_path) - urls.append(cur) - url2base[cur] = url - - start_time = time.monotonic() - url, _ = uhelp.wait_for_url( - urls=urls, - max_wait=url_params.max_wait_seconds, - timeout=url_params.timeout_seconds, - status_cb=LOG.warning, - headers_redact=self.imdsv2_token_redact, - headers_cb=self._get_headers, - request_method=request_method, - ) - - if url: - metadata_address = url2base[url] - - if metadata_address: - self.metadata_address = metadata_address - LOG.debug("Using metadata source: '%s'", self.metadata_address) - elif self.cloud_name in IDMSV2_SUPPORTED_CLOUD_PLATFORMS: - LOG.warning("IMDS's HTTP endpoint is probably disabled") - else: - LOG.critical( - "Giving up on md from %s after %s seconds", - urls, - int(time.monotonic() - start_time), - ) - - return bool(metadata_address) - - def device_name_to_device(self, name): - # Consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - if "block-device-mapping" not in self.metadata: - return None - - # Example: - # 'block-device-mapping': - # {'ami': '/dev/sda1', - # 'ephemeral0': '/dev/sdb', - # 'root': '/dev/sda1'} - found = None - bdm = self.metadata["block-device-mapping"] - if not isinstance(bdm, dict): - LOG.debug("block-device-mapping not a dictionary: '%s'", bdm) - return None - - for entname, device in bdm.items(): - if entname == name: - found = device - break - # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' - if entname == "ephemeral" and name == "ephemeral0": - found = device - - if found is None: - LOG.debug("Unable to convert %s to a device", name) - return None - - ofound = found - if not found.startswith("/"): - found = "/dev/%s" % found - - if os.path.exists(found): - return found - - remapped = self._remap_device(os.path.basename(found)) - if remapped: - LOG.debug("Remapped device name %s => %s", found, remapped) - return remapped - - # On t1.micro, ephemeral0 will appear in block-device-mapping from - # metadata, but it will not exist on disk (and never will) - # at this point, we've verified that the path did not exist - # in the special case of 'ephemeral0' return None to avoid bogus - # fstab entry (LP: #744019) - if name == "ephemeral0": - return None - return ofound - - @property - def availability_zone(self): - try: - if self.cloud_name == CloudNames.AWS: - return self.identity.get( - "availabilityZone", - self.metadata["placement"]["availability-zone"], - ) - else: - return self.metadata["placement"]["availability-zone"] - except KeyError: - return None - - @property - def region(self): - if self.cloud_name == CloudNames.AWS: - region = self.identity.get("region") - # Fallback to trimming the availability zone if region is missing - if self.availability_zone and not region: - region = self.availability_zone[:-1] - return region - else: - az = self.availability_zone - if az is not None: - return az[:-1] - return None - - def activate(self, cfg, is_new_instance): - if not is_new_instance: - return - if self.cloud_name == CloudNames.UNKNOWN: - warn_if_necessary( - util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT), - cfg, - ) - - @property - def network_config(self): - """Return a network config dict for rendering ENI or netplan files.""" - if self._network_config != sources.UNSET: - return self._network_config - - if self.metadata is None: - # this would happen if get_data hadn't been called. leave as UNSET - LOG.warning( - "Unexpected call to network_config when metadata is None." - ) - return None - - result = None - iface = self.distro.fallback_interface - net_md = self.metadata.get("network") - if isinstance(net_md, dict): - # SRU_BLOCKER: xenial, bionic and eoan should default - # apply_full_imds_network_config to False to retain original - # behavior on those releases. - result = convert_ec2_metadata_network_config( - net_md, - self.distro, - fallback_nic=iface, - full_network_config=util.get_cfg_option_bool( - self.ds_cfg, "apply_full_imds_network_config", True - ), - fallback_nic_order=self._fallback_nic_order, - ) - - # Non-VPC (aka Classic) Ec2 instances need to rewrite the - # network config file every boot due to MAC address change. - if self.is_classic_instance(): - self.default_update_events = copy.deepcopy( - self.default_update_events - ) - self.default_update_events[EventScope.NETWORK].add( - EventType.BOOT - ) - self.default_update_events[EventScope.NETWORK].add( - EventType.BOOT_LEGACY - ) - else: - LOG.warning("Metadata 'network' key not valid: %s.", net_md) - self._network_config = result - - return self._network_config - - def crawl_metadata(self): - """Crawl metadata service when available. - - @returns: Dictionary of crawled metadata content containing the keys: - meta-data, user-data and dynamic. - """ - if not self.wait_for_metadata_service(): - return {} - api_version = self.get_metadata_api_version() - redact = self.imdsv2_token_redact - crawled_metadata = {} - if self.cloud_name in IDMSV2_SUPPORTED_CLOUD_PLATFORMS: - exc_cb = self._refresh_stale_aws_token_cb - exc_cb_ud = self._skip_or_refresh_stale_aws_token_cb - skip_cb = None - elif self.cloud_name == CloudNames.OUTSCALE: - exc_cb = exc_cb_ud = None - skip_cb = skip_404_tag_errors - else: - exc_cb = exc_cb_ud = skip_cb = None - try: - raw_userdata = ec2.get_instance_userdata( - api_version, - self.metadata_address, - headers_cb=self._get_headers, - headers_redact=redact, - exception_cb=exc_cb_ud, - ) - crawled_metadata["user-data"] = util.maybe_b64decode(raw_userdata) - crawled_metadata["meta-data"] = ec2.get_instance_metadata( - api_version, - self.metadata_address, - headers_cb=self._get_headers, - headers_redact=redact, - exception_cb=exc_cb, - retrieval_exception_ignore_cb=skip_cb, - ) - if self.cloud_name == CloudNames.AWS: - identity = ec2.get_instance_identity( - api_version, - self.metadata_address, - headers_cb=self._get_headers, - headers_redact=redact, - exception_cb=exc_cb, - ) - crawled_metadata["dynamic"] = {"instance-identity": identity} - except Exception: - util.logexc( - LOG, - "Failed reading from metadata address %s", - self.metadata_address, - ) - return {} - crawled_metadata["_metadata_api_version"] = api_version - return crawled_metadata - - def _refresh_api_token(self, seconds=None): - """Request new metadata API token. - @param seconds: The lifetime of the token in seconds - - @return: The API token or None if unavailable. - """ - if self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS: - return None - - if seconds is None: - seconds = self.imdsv2_token_ttl_seconds - - LOG.debug("Refreshing Ec2 metadata API token") - request_header = {self.imdsv2_token_req_header: seconds} - token_url = "{}/{}".format(self.metadata_address, self.api_token_route) - try: - response = uhelp.readurl( - token_url, - headers=request_header, - headers_redact=self.imdsv2_token_redact, - request_method="PUT", - ) - except uhelp.UrlError as e: - LOG.warning( - "Unable to get API token: %s raised exception %s", token_url, e - ) - return None - return response.contents - - def _skip_or_refresh_stale_aws_token_cb( - self, exception: uhelp.UrlError - ) -> bool: - """Callback will not retry on SKIP_USERDATA_CODES or if no token - is available.""" - retry = ec2.skip_retry_on_codes(ec2.SKIP_USERDATA_CODES, exception) - if not retry: - return False # False raises exception - return self._refresh_stale_aws_token_cb(exception) - - def _refresh_stale_aws_token_cb( - self, exception: uhelp.UrlError - ) -> Literal[True]: - """Exception handler for Ec2 to refresh token if token is stale.""" - if exception.code == 401: - # With _api_token as None, _get_headers will _refresh_api_token. - LOG.debug("Clearing cached Ec2 API token due to expiry") - self._api_token = None - return True # always retry - - def _token_exception_cb(self, exception: uhelp.UrlError) -> bool: - """Fail quickly on proper AWS if IMDSv2 rejects API token request - - Guidance from Amazon is that if IMDSv2 had disabled token requests - by returning a 403, or cloud-init malformed requests resulting in - other 40X errors, we want the datasource detection to fail quickly - without retries as those symptoms will likely not be resolved by - retries. - - Exceptions such as requests.ConnectionError due to IMDS being - temporarily unroutable or unavailable will still retry due to the - callsite wait_for_url. - """ - if exception.code: - # requests.ConnectionError will have exception.code == None - if exception.code == 403: - LOG.warning( - "Ec2 IMDS endpoint returned a 403 error. " - "HTTP endpoint is disabled. Aborting." - ) - return False - elif exception.code == 503: - # Let the global handler deal with it - return False - elif exception.code >= 400: - LOG.warning( - "Fatal error while requesting Ec2 IMDSv2 API tokens" - ) - return False - return True - - def _get_headers(self, url=""): - """Return a dict of headers for accessing a url. - - If _api_token is unset on AWS, attempt to refresh the token via a PUT - and then return the updated token header. - """ - if self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS: - return {} - # Request a 6 hour token if URL is api_token_route - request_token_header = { - self.imdsv2_token_req_header: self.imdsv2_token_ttl_seconds - } - if self.api_token_route in url: - return request_token_header - if not self._api_token: - # If we don't yet have an API token, get one via a PUT against - # api_token_route. This _api_token may get unset by a 403 due - # to an invalid or expired token - self._api_token = self._refresh_api_token() - if not self._api_token: - return {} - return {self.imdsv2_token_put_header: self._api_token} - - -class DataSourceEc2Local(DataSourceEc2): - """Datasource run at init-local which sets up network to query metadata. - - In init-local, no network is available. This subclass sets up minimal - networking with dhclient on a viable nic so that it can talk to the - metadata service. If the metadata service provides network configuration - then render the network configuration for that instance based on metadata. - """ - - perform_dhcp_setup = True # Use dhcp before querying metadata - - def get_data(self): - supported_platforms = (CloudNames.AWS, CloudNames.OUTSCALE) - if self.cloud_name not in supported_platforms: - LOG.debug( - "Local Ec2 mode only supported on %s, not %s", - supported_platforms, - self.cloud_name, - ) - return False - return super(DataSourceEc2Local, self).get_data() - - -def read_strict_mode(cfgval, default): - try: - return parse_strict_mode(cfgval) - except ValueError as e: - LOG.warning(e) - return default - - -def parse_strict_mode(cfgval): - # given a mode like: - # true, false, warn,[sleep] - # return tuple with string mode (true|false|warn) and sleep. - if cfgval is True: - return "true", None - if cfgval is False: - return "false", None - - if not cfgval: - return "warn", 0 - - mode, _, sleep = cfgval.partition(",") - if mode not in ("true", "false", "warn"): - raise ValueError( - "Invalid mode '%s' in strict_id setting '%s': " - "Expected one of 'true', 'false', 'warn'." % (mode, cfgval) - ) - - if sleep: - try: - sleep = int(sleep) - except ValueError as e: - raise ValueError( - "Invalid sleep '%s' in strict_id setting '%s': not an integer" - % (sleep, cfgval) - ) from e - else: - sleep = None - - return mode, sleep - - -def warn_if_necessary(cfgval, cfg): - try: - mode, sleep = parse_strict_mode(cfgval) - except ValueError as e: - LOG.warning(e) - return - - if mode == "false": - return - - warnings.show_warning("non_ec2_md", cfg, mode=True, sleep=sleep) - - -def identify_aliyun(data): - if data["product_name"] == "Alibaba Cloud ECS": - return CloudNames.ALIYUN - - -def identify_aws(data): - # data is a dictionary returned by _collect_platform_data. - uuid_str = data["uuid"] - if uuid_str.startswith("ec2"): - # example same-endian uuid: - # EC2E1916-9099-7CAF-FD21-012345ABCDEF - return CloudNames.AWS - with suppress(ValueError): - if uuid.UUID(uuid_str).bytes_le.hex().startswith("ec2"): - # check for other endianness - # example other-endian uuid: - # 45E12AEC-DCD1-B213-94ED-012345ABCDEF - return CloudNames.AWS - return None - - -def identify_brightbox(data): - if data["serial"].endswith(".brightbox.com"): - return CloudNames.BRIGHTBOX - - -def identify_zstack(data): - if data["asset_tag"].endswith(".zstack.io"): - return CloudNames.ZSTACK - - -def identify_e24cloud(data): - if data["vendor"] == "e24cloud": - return CloudNames.E24CLOUD - - -def identify_outscale(data): - if ( - data["product_name"] == "3DS Outscale VM".lower() - and data["vendor"] == "3DS Outscale".lower() - ): - return CloudNames.OUTSCALE - - -def identify_platform(): - # identify the platform and return an entry in CloudNames. - data = _collect_platform_data() - checks = ( - identify_aws, - identify_brightbox, - identify_zstack, - identify_e24cloud, - identify_outscale, - identify_aliyun, - lambda x: CloudNames.UNKNOWN, - ) - for checker in checks: - try: - result = checker(data) - if result: - return result - except Exception as e: - LOG.warning( - "calling %s with %s raised exception: %s", checker, data, e - ) - - -def _collect_platform_data(): - """Returns a dictionary of platform info from dmi or /sys/hypervisor. - - Keys in the dictionary are as follows: - uuid: system-uuid from dmi or /sys/hypervisor - serial: dmi 'system-serial-number' (/sys/.../product_serial) - asset_tag: 'dmidecode -s chassis-asset-tag' - vendor: dmi 'system-manufacturer' (/sys/.../sys_vendor) - product_name: dmi 'system-product-name' (/sys/.../system-manufacturer) - - On Ec2 instances experimentation is that product_serial is upper case, - and product_uuid is lower case. This returns lower case values for both. - """ - uuid = None - with suppress(OSError, UnicodeDecodeError): - uuid = util.load_text_file("/sys/hypervisor/uuid").strip() - - uuid = uuid or dmi.read_dmi_data("system-uuid") or "" - serial = dmi.read_dmi_data("system-serial-number") or "" - asset_tag = dmi.read_dmi_data("chassis-asset-tag") or "" - vendor = dmi.read_dmi_data("system-manufacturer") or "" - product_name = dmi.read_dmi_data("system-product-name") or "" - - return { - "uuid": uuid.lower(), - "serial": serial.lower(), - "asset_tag": asset_tag.lower(), - "vendor": vendor.lower(), - "product_name": product_name.lower(), - } - - -def _build_nic_order( - macs_metadata: Dict[str, Dict], - macs_to_nics: Dict[str, str], - fallback_nic_order: NicOrder = NicOrder.MAC, -) -> Dict[str, int]: - """ - Builds a dictionary containing macs as keys and nic orders as values, - taking into account `network-card` and `device-number` if present. - - Note that the first NIC will be the primary NIC as it will be the one with - [network-card] == 0 and device-number == 0 if present. - - @param macs_metadata: dictionary with mac address as key and contents like: - {"device-number": "0", "interface-id": "...", "local-ipv4s": ...} - @macs_to_nics: dictionary with mac address as key and nic name as value - - @return: Dictionary with macs as keys and nic orders as values. - """ - nic_order: Dict[str, int] = {} - if len(macs_to_nics) == 0 or len(macs_metadata) == 0: - return nic_order - - valid_macs_metadata = filter( - # filter out nics without metadata (not a physical nic) - lambda mmd: mmd[1] is not None, - # filter by macs - map( - lambda mac: (mac, macs_metadata.get(mac), macs_to_nics[mac]), - macs_to_nics.keys(), - ), - ) - - def _get_key_as_int_or(dikt, key, alt_value): - value = dikt.get(key, None) - if value is not None: - return int(value) - return alt_value - - # Sort by (network_card, device_index) as some instances could have - # multiple network cards with repeated device indexes. - # - # On platforms where network-card and device-number are not present, - # as AliYun, the order will be by mac, as before the introduction of this - # function. - return { - mac: i - for i, (mac, _mac_metadata, _nic_name) in enumerate( - sorted( - valid_macs_metadata, - key=lambda mmd: ( - _get_key_as_int_or( - mmd[1], "network-card", float("infinity") - ), - _get_key_as_int_or( - mmd[1], "device-number", float("infinity") - ), - ( - mmd[2] - if fallback_nic_order == NicOrder.NIC_NAME - else mmd[0] - ), - ), - ) - ) - } - - -def _configure_policy_routing( - dev_config: dict, - *, - nic_name: str, - nic_metadata: dict, - distro: Distro, - is_ipv4: bool, - table: int, -) -> None: - """ - Configure policy-based routing on secondary NICs / secondary IPs to - ensure outgoing packets are routed via the correct interface. - - @param: dev_config: network cfg v2 to be updated inplace. - @param: nic_name: nic name. Only used if ipv4. - @param: nic_metadata: nic metadata from IMDS. - @param: distro: Instance of Distro. Only used if ipv4. - @param: is_ipv4: Boolean indicating if we are acting over ipv4 or not. - @param: table: Routing table id. - """ - if is_ipv4: - subnet_prefix_routes = nic_metadata.get("subnet-ipv4-cidr-block") - ips = nic_metadata.get("local-ipv4s") - else: - subnet_prefix_routes = nic_metadata.get("subnet-ipv6-cidr-blocks") - ips = nic_metadata.get("ipv6s") - if not (subnet_prefix_routes and ips): - LOG.debug( - "Not enough IMDS information to configure policy routing " - "for IPv%s", - "4" if is_ipv4 else "6", - ) - return - - if not dev_config.get("routes"): - dev_config["routes"] = [] - if is_ipv4: - try: - lease = distro.dhcp_client.dhcp_discovery(nic_name, distro=distro) - gateway = lease["routers"] - except NoDHCPLeaseError as e: - LOG.warning( - "Could not perform dhcp discovery on %s to find its " - "gateway. Not adding default route via the gateway. " - "Error: %s", - nic_name, - e, - ) - else: - # Add default route via the NIC's gateway - dev_config["routes"].append( - { - "to": "0.0.0.0/0", - "via": gateway, - "table": table, - }, - ) - - subnet_prefix_routes = ( - [subnet_prefix_routes] - if isinstance(subnet_prefix_routes, str) - else subnet_prefix_routes - ) - for prefix_route in subnet_prefix_routes: - dev_config["routes"].append( - { - "to": prefix_route, - "table": table, - }, - ) - - if not dev_config.get("routing-policy"): - dev_config["routing-policy"] = [] - # Packets coming from any IP associated with the current NIC - # will be routed using `table` routing table - ips = [ips] if isinstance(ips, str) else ips - for ip in ips: - dev_config["routing-policy"].append( - { - "from": ip, - "table": table, - }, - ) - - -def convert_ec2_metadata_network_config( - network_md, - distro, - macs_to_nics=None, - fallback_nic=None, - full_network_config=True, - fallback_nic_order=NicOrder.MAC, -): - """Convert ec2 metadata to network config version 2 data dict. - - @param: network_md: 'network' portion of EC2 metadata. - generally formed as {"interfaces": {"macs": {}} where - 'macs' is a dictionary with mac address as key and contents like: - {"device-number": "0", "interface-id": "...", "local-ipv4s": ...} - @param: distro: instance of Distro. - @param: macs_to_nics: Optional dict of mac addresses and nic names. If - not provided, get_interfaces_by_mac is called to get it from the OS. - @param: fallback_nic: Optionally provide the primary nic interface name. - This nic will be guaranteed to minimally have a dhcp4 configuration. - @param: full_network_config: Boolean set True to configure all networking - presented by IMDS. This includes rendering secondary IPv4 and IPv6 - addresses on all NICs and rendering network config on secondary NICs. - If False, only the primary nic will be configured and only with dhcp - (IPv4/IPv6). - - @return A dict of network config version 2 based on the metadata and macs. - """ - netcfg = {"version": 2, "ethernets": {}} - if not macs_to_nics: - macs_to_nics = net.get_interfaces_by_mac() - macs_metadata = network_md["interfaces"]["macs"] - - if not full_network_config: - for mac, nic_name in macs_to_nics.items(): - if nic_name == fallback_nic: - break - dev_config = { - "dhcp4": True, - "dhcp6": False, - "match": {"macaddress": mac.lower()}, - "set-name": nic_name, - } - nic_metadata = macs_metadata.get(mac) - if nic_metadata.get("ipv6s"): # Any IPv6 addresses configured - dev_config["dhcp6"] = True - netcfg["ethernets"][nic_name] = dev_config - return netcfg - # Apply network config for all nics and any secondary IPv4/v6 addresses - is_netplan = isinstance(distro.network_renderer, netplan.Renderer) - nic_order = _build_nic_order( - macs_metadata, macs_to_nics, fallback_nic_order - ) - macs = sorted(macs_to_nics.keys()) - for mac in macs: - nic_name = macs_to_nics[mac] - nic_metadata = macs_metadata.get(mac) - if not nic_metadata: - continue # Not a physical nic represented in metadata - nic_idx = nic_order[mac] - is_primary_nic = nic_idx == 0 - # nic_idx + 1 to start route_metric at 100 (nic_idx is 0-indexed) - dhcp_override = {"route-metric": (nic_idx + 1) * 100} - dev_config = { - "dhcp4": True, - "dhcp4-overrides": dhcp_override, - "dhcp6": False, - "match": {"macaddress": mac.lower()}, - "set-name": nic_name, - } - # This config only works on systems using Netplan because Networking - # config V2 does not support `routing-policy`, but this config is - # passed through on systems using Netplan. - # See: https://github.com/canonical/cloud-init/issues/4862 - # - # If device-number is not present (AliYun or other ec2-like platforms), - # do not configure source-routing as we cannot determine which is the - # primary NIC. - table = 100 + nic_idx - if ( - is_netplan - and nic_metadata.get("device-number") - and not is_primary_nic - ): - dhcp_override["use-routes"] = True - _configure_policy_routing( - dev_config, - distro=distro, - nic_name=nic_name, - nic_metadata=nic_metadata, - is_ipv4=True, - table=table, - ) - if nic_metadata.get("ipv6s"): # Any IPv6 addresses configured - dev_config["dhcp6"] = True - dev_config["dhcp6-overrides"] = dhcp_override - if ( - is_netplan - and nic_metadata.get("device-number") - and not is_primary_nic - ): - _configure_policy_routing( - dev_config, - distro=distro, - nic_name=nic_name, - nic_metadata=nic_metadata, - is_ipv4=False, - table=table, - ) - dev_config["addresses"] = get_secondary_addresses(nic_metadata, mac) - if not dev_config["addresses"]: - dev_config.pop("addresses") # Since we found none configured - - netcfg["ethernets"][nic_name] = dev_config - # Remove route-metric dhcp overrides and routes / routing-policy if only - # one nic configured - if len(netcfg["ethernets"]) == 1: - for nic_name in netcfg["ethernets"].keys(): - netcfg["ethernets"][nic_name].pop("dhcp4-overrides") - netcfg["ethernets"][nic_name].pop("dhcp6-overrides", None) - netcfg["ethernets"][nic_name].pop("routes", None) - netcfg["ethernets"][nic_name].pop("routing-policy", None) - return netcfg - - -def get_secondary_addresses(nic_metadata, mac): - """Parse interface-specific nic metadata and return any secondary IPs - - :return: List of secondary IPv4 or IPv6 addresses to configure on the - interface - """ - ipv4s = nic_metadata.get("local-ipv4s") - ipv6s = nic_metadata.get("ipv6s") - addresses = [] - # In version < 2018-09-24 local_ipv4s or ipv6s is a str with one IP - if bool(isinstance(ipv4s, list) and len(ipv4s) > 1): - addresses.extend( - _get_secondary_addresses( - nic_metadata, "subnet-ipv4-cidr-block", mac, ipv4s, "24" - ) - ) - if bool(isinstance(ipv6s, list) and len(ipv6s) > 1): - addresses.extend( - _get_secondary_addresses( - nic_metadata, "subnet-ipv6-cidr-block", mac, ipv6s, "128" - ) - ) - return sorted(addresses) - - -def _get_secondary_addresses(nic_metadata, cidr_key, mac, ips, default_prefix): - """Return list of IP addresses as CIDRs for secondary IPs - - The CIDR prefix will be default_prefix if cidr_key is absent or not - parseable in nic_metadata. - """ - addresses = [] - cidr = nic_metadata.get(cidr_key) - prefix = default_prefix - if not cidr or len(cidr.split("/")) != 2: - ip_type = "ipv4" if "ipv4" in cidr_key else "ipv6" - LOG.warning( - "Could not parse %s %s for mac %s. %s network" - " config prefix defaults to /%s", - cidr_key, - cidr, - mac, - ip_type, - prefix, - ) - else: - prefix = cidr.split("/")[1] - # We know we have > 1 ips for in metadata for this IP type - for ip in ips[1:]: - addresses.append("{ip}/{prefix}".format(ip=ip, prefix=prefix)) - return addresses - - -# Used to match classes to dependencies -datasources = [ - (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)), # Run at init-local - (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) diff --git a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/cloudinit/sources/__init__.py b/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/cloudinit/sources/__init__.py deleted file mode 100644 index 973d140f..00000000 --- a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/cloudinit/sources/__init__.py +++ /dev/null @@ -1,1253 +0,0 @@ -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This file is part of cloud-init. See LICENSE file for license information. - -import abc -import copy -import json -import logging -import os -import pickle -import re -from collections import namedtuple -from enum import Enum, unique -from typing import Any, Dict, List, Optional, Tuple, Union - -from cloudinit import ( - atomic_helper, - dmi, - importer, - lifecycle, - net, - performance, - type_utils, -) -from cloudinit import user_data as ud -from cloudinit import util -from cloudinit.atomic_helper import write_json -from cloudinit.distros import Distro -from cloudinit.event import EventScope, EventType -from cloudinit.filters import launch_index -from cloudinit.helpers import Paths -from cloudinit.persistence import CloudInitPickleMixin -from cloudinit.reporting import events - -DSMODE_DISABLED = "disabled" -DSMODE_LOCAL = "local" -DSMODE_NETWORK = "net" -DSMODE_PASS = "pass" - -VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK] - -DEP_FILESYSTEM = "FILESYSTEM" -DEP_NETWORK = "NETWORK" -DS_PREFIX = "DataSource" - -EXPERIMENTAL_TEXT = ( - "EXPERIMENTAL: The structure and format of content scoped under the 'ds'" - " key may change in subsequent releases of cloud-init." -) - - -REDACT_SENSITIVE_VALUE = "redacted for non-root user" - -# Key which can be provide a cloud's official product name to cloud-init -METADATA_CLOUD_NAME_KEY = "cloud-name" - -UNSET = "_unset" -METADATA_UNKNOWN = "unknown" - -LOG = logging.getLogger(__name__) - -# CLOUD_ID_REGION_PREFIX_MAP format is: -# : (: ) -CLOUD_ID_REGION_PREFIX_MAP = { - "cn-": ("aws-china", lambda c: c == "aws"), # only change aws regions - "us-gov-": ("aws-gov", lambda c: c == "aws"), # only change aws regions - "china": ("azure-china", lambda c: c == "azure"), # only change azure -} - - -@unique -class NetworkConfigSource(Enum): - """ - Represents the canonical list of network config sources that cloud-init - knows about. - """ - - CMD_LINE = "cmdline" - DS = "ds" - SYSTEM_CFG = "system_cfg" - FALLBACK = "fallback" - INITRAMFS = "initramfs" - - def __str__(self) -> str: - return self.value - - -class NicOrder(Enum): - """Represents ways to sort NICs""" - - MAC = "mac" - NIC_NAME = "nic_name" - - def __str__(self) -> str: - return self.value - - -class DatasourceUnpickleUserDataError(Exception): - """Raised when userdata is unable to be unpickled due to python upgrades""" - - -class DataSourceNotFoundException(Exception): - pass - - -class InvalidMetaDataException(Exception): - """Raised when metadata is broken, unavailable or disabled.""" - - -def process_instance_metadata(metadata, key_path="", sensitive_keys=()): - """Process all instance metadata cleaning it up for persisting as json. - - Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list - - @return Dict copy of processed metadata. - """ - md_copy = copy.deepcopy(metadata) - base64_encoded_keys = [] - sens_keys = [] - for key, val in metadata.items(): - if key_path: - sub_key_path = key_path + "/" + key - else: - sub_key_path = key - if ( - key.lower() in sensitive_keys - or sub_key_path.lower() in sensitive_keys - ): - sens_keys.append(sub_key_path) - if isinstance(val, str) and val.startswith("ci-b64:"): - base64_encoded_keys.append(sub_key_path) - md_copy[key] = val.replace("ci-b64:", "") - if isinstance(val, dict): - return_val = process_instance_metadata( - val, sub_key_path, sensitive_keys - ) - base64_encoded_keys.extend(return_val.pop("base64_encoded_keys")) - sens_keys.extend(return_val.pop("sensitive_keys")) - md_copy[key] = return_val - md_copy["base64_encoded_keys"] = sorted(base64_encoded_keys) - md_copy["sensitive_keys"] = sorted(sens_keys) - return md_copy - - -def redact_sensitive_keys(metadata, redact_value=REDACT_SENSITIVE_VALUE): - """Redact any sensitive keys from to provided metadata dictionary. - - Replace any keys values listed in 'sensitive_keys' with redact_value. - """ - # While 'sensitive_keys' should already sanitized to only include what - # is in metadata, it is possible keys will overlap. For example, if - # "merged_cfg" and "merged_cfg/ds/userdata" both match, it's possible that - # "merged_cfg" will get replaced first, meaning "merged_cfg/ds/userdata" - # no longer represents a valid key. - # Thus, we still need to do membership checks in this function. - if not metadata.get("sensitive_keys", []): - return metadata - md_copy = copy.deepcopy(metadata) - for key_path in metadata.get("sensitive_keys"): - path_parts = key_path.split("/") - obj = md_copy - for path in path_parts: - if ( - path in obj - and isinstance(obj[path], dict) - and path != path_parts[-1] - ): - obj = obj[path] - if path in obj: - obj[path] = redact_value - return md_copy - - -URLParams = namedtuple( - "URLParams", - [ - "max_wait_seconds", - "timeout_seconds", - "num_retries", - "sec_between_retries", - ], -) - -DataSourceHostname = namedtuple( - "DataSourceHostname", - ["hostname", "is_default"], -) - - -class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): - - dsmode = DSMODE_NETWORK - default_locale = "en_US.UTF-8" - - # Datasource name needs to be set by subclasses to determine which - # cloud-config datasource key is loaded - dsname = "_undef" - - # Cached cloud_name as determined by _get_cloud_name - _cloud_name: Optional[str] = None - - # Cached cloud platform api type: e.g. ec2, openstack, kvm, lxd, azure etc. - _platform_type = None - - # More details about the cloud platform: - # - metadata (http://169.254.169.254/) - # - seed-dir () - _subplatform = None - - _crawled_metadata: Optional[Union[Dict, str]] = None - - # The network configuration sources that should be considered for this data - # source. (The first source in this list that provides network - # configuration will be used without considering any that follow.) This - # should always be a subset of the members of NetworkConfigSource with no - # duplicate entries. - network_config_sources: Tuple[NetworkConfigSource, ...] = ( - NetworkConfigSource.CMD_LINE, - NetworkConfigSource.INITRAMFS, - NetworkConfigSource.SYSTEM_CFG, - NetworkConfigSource.DS, - ) - - # read_url_params - url_max_wait = -1 # max_wait < 0 means do not wait - url_timeout = 10 # timeout for each metadata url read attempt - url_retries = 5 # number of times to retry url upon 404 - url_sec_between_retries = 1 # amount of seconds to wait between retries - - # The datasource defines a set of supported EventTypes during which - # the datasource can react to changes in metadata and regenerate - # network configuration on metadata changes. These are defined in - # `supported_network_events`. - # The datasource also defines a set of default EventTypes that the - # datasource can react to. These are the event types that will be used - # if not overridden by the user. - # - # A datasource requiring to write network config on each system boot - # would either: - # - # 1) Overwrite the class attribute `default_update_events` like: - # - # >>> default_update_events = { - # ... EventScope.NETWORK: { - # ... EventType.BOOT_NEW_INSTANCE, - # ... EventType.BOOT, - # ... } - # ... } - # - # 2) Or, if writing network config on every boot has to be determined at - # runtime, then deepcopy to not overwrite the class attribute on other - # elements of this class hierarchy, like: - # - # >>> self.default_update_events = copy.deepcopy( - # ... self.default_update_events - # ... ) - # >>> self.default_update_events[EventScope.NETWORK].add(EventType.BOOT) - - supported_update_events = { - EventScope.NETWORK: { - EventType.BOOT_NEW_INSTANCE, - EventType.BOOT, - EventType.BOOT_LEGACY, - EventType.HOTPLUG, - } - } - - # Default: generate network config on new instance id (first boot). - default_update_events = { - EventScope.NETWORK: { - EventType.BOOT_NEW_INSTANCE, - } - } - - # N-tuple listing default values for any metadata-related class - # attributes cached on an instance by a process_data runs. These attribute - # values are reset via clear_cached_attrs during any update_metadata call. - cached_attr_defaults: Tuple[Tuple[str, Any], ...] = ( - ("ec2_metadata", UNSET), - ("network_json", UNSET), - ("metadata", {}), - ("userdata", None), - ("userdata_raw", None), - ("vendordata", None), - ("vendordata_raw", None), - ("vendordata2", None), - ("vendordata2_raw", None), - ) - - _dirty_cache = False - - # N-tuple of keypaths or keynames redact from instance-data.json for - # non-root users - sensitive_metadata_keys: Tuple[str, ...] = ( - "combined_cloud_config", - "merged_cfg", - "merged_system_cfg", - "security-credentials", - "userdata", - "user-data", - "user_data", - "vendordata", - "vendor-data", - # Provide ds/vendor_data to avoid redacting top-level - # "vendor_data": {enabled: True} - "ds/vendor_data", - ) - - # True on datasources that may not see hotplugged devices reflected - # in the updated metadata - skip_hotplug_detect = False - - # Extra udev rules for cc_install_hotplug - extra_hotplug_udev_rules: Optional[str] = None - - _ci_pkl_version = 1 - - def __init__(self, sys_cfg, distro: Distro, paths: Paths, ud_proc=None): - self.sys_cfg = sys_cfg - self.distro = distro - self.paths = paths - self.userdata: Optional[Any] = None - self.metadata: dict = {} - self.userdata_raw: Optional[Union[str, bytes]] = None - self.vendordata = None - self.vendordata2 = None - self.vendordata_raw = None - self.vendordata2_raw = None - self.metadata_address: Optional[str] = None - self.network_json: Optional[str] = UNSET - self.ec2_metadata = UNSET - - self.ds_cfg = util.get_cfg_by_path( - self.sys_cfg, ("datasource", self.dsname), {} - ) - if not self.ds_cfg: - self.ds_cfg = {} - - if not ud_proc: - self.ud_proc = ud.UserDataProcessor(self.paths) - else: - self.ud_proc = ud_proc - - def _unpickle(self, ci_pkl_version: int) -> None: - """Perform deserialization fixes for Paths.""" - expected_attrs = { - "_crawled_metadata": None, - "_platform_type": None, - "_subplatform": None, - "ec2_metadata": UNSET, - "extra_hotplug_udev_rules": None, - "metadata_address": None, - "network_json": UNSET, - "skip_hotplug_detect": False, - "vendordata2": None, - "vendordata2_raw": None, - } - for key, value in expected_attrs.items(): - if not hasattr(self, key): - setattr(self, key, value) - - if not hasattr(self, "check_if_fallback_is_allowed"): - setattr(self, "check_if_fallback_is_allowed", lambda: False) - if hasattr(self, "userdata") and self.userdata is not None: - # If userdata stores MIME data, on < python3.6 it will be - # missing the 'policy' attribute that exists on >=python3.6. - # Calling str() on the userdata will attempt to access this - # policy attribute. This will raise an exception, causing - # the pickle load to fail, so cloud-init will discard the cache - try: - str(self.userdata) - except AttributeError as e: - LOG.debug( - "Unable to unpickle datasource: %s." - " Ignoring current cache.", - e, - ) - raise DatasourceUnpickleUserDataError() from e - - def __str__(self): - return type_utils.obj_name(self) - - def ds_detect(self) -> bool: - """Check if running on this datasource""" - return True - - def override_ds_detect(self) -> bool: - """Override if either: - - only a single datasource defined (nothing to fall back to) - - command line argument is used (ci.ds=OpenStack) - - Note: get_cmdline() is required for the general case - when ds-identify - does not run, _something_ needs to detect the kernel command line - definition. - """ - if self.dsname.lower() == parse_cmdline().lower(): - LOG.debug( - "Kernel command line set to use a single datasource %s.", - self, - ) - return True - elif self.sys_cfg.get("datasource_list", []) == [self.dsname]: - LOG.debug( - "Datasource list set to use a single datasource %s.", self - ) - return True - return False - - def _check_and_get_data(self): - """Overrides runtime datasource detection""" - if self.override_ds_detect(): - return self._get_data() - elif self.ds_detect(): - LOG.debug( - "Detected %s", - self, - ) - return self._get_data() - else: - LOG.debug("Did not detect %s", self) - return False - - def _get_standardized_metadata(self, instance_data): - """Return a dictionary of standardized metadata keys.""" - local_hostname = self.get_hostname().hostname - instance_id = self.get_instance_id() - availability_zone = self.availability_zone - # In the event of upgrade from existing cloudinit, pickled datasource - # will not contain these new class attributes. So we need to recrawl - # metadata to discover that content - sysinfo = instance_data["sys_info"] - return { - "v1": { - "_beta_keys": ["subplatform"], - "availability-zone": availability_zone, - "availability_zone": availability_zone, - "cloud_id": canonical_cloud_id( - self.cloud_name, self.region, self.platform_type - ), - "cloud-name": self.cloud_name, - "cloud_name": self.cloud_name, - "distro": sysinfo["dist"][0], - "distro_version": sysinfo["dist"][1], - "distro_release": sysinfo["dist"][2], - "platform": self.platform_type, - "public_ssh_keys": self.get_public_ssh_keys(), - "python_version": sysinfo["python"], - "instance-id": instance_id, - "instance_id": instance_id, - "kernel_release": sysinfo["uname"][2], - "local-hostname": local_hostname, - "local_hostname": local_hostname, - "machine": sysinfo["uname"][4], - "region": self.region, - "subplatform": self.subplatform, - "system_platform": sysinfo["platform"], - "variant": sysinfo["variant"], - } - } - - def clear_cached_attrs(self, attr_defaults=()): - """Reset any cached metadata attributes to datasource defaults. - - @param attr_defaults: Optional tuple of (attr, value) pairs to - set instead of cached_attr_defaults. - """ - if not self._dirty_cache: - return - if attr_defaults: - attr_values = attr_defaults - else: - attr_values = self.cached_attr_defaults - - for attribute, value in attr_values: - if hasattr(self, attribute): - setattr(self, attribute, value) - if not attr_defaults: - self._dirty_cache = False - - @performance.timed("Getting metadata", log_mode="always") - def get_data(self) -> bool: - """Datasources implement _get_data to setup metadata and userdata_raw. - - Minimally, the datasource should return a boolean True on success. - """ - self._dirty_cache = True - return_value = self._check_and_get_data() - # TODO: verify that datasource types are what they are expected to be - # each datasource uses different logic to get userdata, metadata, etc - # and then the rest of the codebase assumes the types of this data - # it would be prudent to have a type check here that warns, when the - # datatype is incorrect, rather than assuming types and throwing - # exceptions later if/when they get used incorrectly. - if not return_value: - return return_value - self.persist_instance_data() - return return_value - - def persist_instance_data(self, write_cache=True): - """Process and write INSTANCE_JSON_FILE with all instance metadata. - - Replace any hyphens with underscores in key names for use in template - processing. - - :param write_cache: boolean set True to persist obj.pkl when - instance_link exists. - - @return True on successful write, False otherwise. - """ - if write_cache and os.path.lexists(self.paths.instance_link): - pkl_store(self, self.paths.get_ipath_cur("obj_pkl")) - if self._crawled_metadata is not None: - # Any datasource with _crawled_metadata will best represent - # most recent, 'raw' metadata - crawled_metadata = copy.deepcopy(self._crawled_metadata) - crawled_metadata.pop("user-data", None) - crawled_metadata.pop("vendor-data", None) - instance_data = {"ds": crawled_metadata} - else: - instance_data = {"ds": {"meta_data": self.metadata}} - if self.network_json != UNSET: - instance_data["ds"]["network_json"] = self.network_json - if self.ec2_metadata != UNSET: - instance_data["ds"]["ec2_metadata"] = self.ec2_metadata - instance_data["ds"]["_doc"] = EXPERIMENTAL_TEXT - # Add merged cloud.cfg and sys info for jinja templates and cli query - instance_data["merged_cfg"] = copy.deepcopy(self.sys_cfg) - instance_data["merged_cfg"][ - "_doc" - ] = "DEPRECATED: Use merged_system_cfg. Will be dropped from 24.1" - # Deprecate merged_cfg to a more specific key name merged_system_cfg - instance_data["merged_system_cfg"] = copy.deepcopy( - instance_data["merged_cfg"] - ) - instance_data["merged_system_cfg"]["_doc"] = ( - "Merged cloud-init system config from /etc/cloud/cloud.cfg and" - " /etc/cloud/cloud.cfg.d/" - ) - instance_data["sys_info"] = util.system_info() - instance_data.update(self._get_standardized_metadata(instance_data)) - try: - # Process content base64encoding unserializable values - content = atomic_helper.json_dumps(instance_data) - # Strip base64: prefix and set base64_encoded_keys list. - processed_data = process_instance_metadata( - json.loads(content), - sensitive_keys=self.sensitive_metadata_keys, - ) - except TypeError as e: - LOG.warning("Error persisting instance-data.json: %s", str(e)) - return False - except UnicodeDecodeError as e: - LOG.warning("Error persisting instance-data.json: %s", str(e)) - return False - json_sensitive_file = self.paths.get_runpath("instance_data_sensitive") - cloud_id = instance_data["v1"].get("cloud_id", "none") - cloud_id_file = os.path.join(self.paths.run_dir, "cloud-id") - util.write_file(f"{cloud_id_file}-{cloud_id}", f"{cloud_id}\n") - # cloud-id not found, then no previous cloud-id file - prev_cloud_id_file = None - new_cloud_id_file = f"{cloud_id_file}-{cloud_id}" - # cloud-id found, then the prev cloud-id file is source of symlink - if os.path.exists(cloud_id_file): - prev_cloud_id_file = os.path.realpath(cloud_id_file) - - util.sym_link(new_cloud_id_file, cloud_id_file, force=True) - if prev_cloud_id_file and prev_cloud_id_file != new_cloud_id_file: - util.del_file(prev_cloud_id_file) - write_json(json_sensitive_file, processed_data, mode=0o600) - json_file = self.paths.get_runpath("instance_data") - # World readable - write_json(json_file, redact_sensitive_keys(processed_data)) - return True - - def _get_data(self) -> bool: - """Walk metadata sources, process crawled data and save attributes.""" - raise NotImplementedError( - "Subclasses of DataSource must implement _get_data which" - " sets self.metadata, vendordata_raw and userdata_raw." - ) - - def get_url_params(self): - """Return the Datasource's preferred url_read parameters. - - Subclasses may override url_max_wait, url_timeout, url_retries. - - @return: A URLParams object with max_wait_seconds, timeout_seconds, - num_retries. - """ - max_wait = self.url_max_wait - try: - max_wait = int(self.ds_cfg.get("max_wait", self.url_max_wait)) - except ValueError: - util.logexc( - LOG, - "Config max_wait '%s' is not an int, using default '%s'", - self.ds_cfg.get("max_wait"), - max_wait, - ) - - timeout = self.url_timeout - try: - timeout = max(0, int(self.ds_cfg.get("timeout", self.url_timeout))) - except ValueError: - timeout = self.url_timeout - util.logexc( - LOG, - "Config timeout '%s' is not an int, using default '%s'", - self.ds_cfg.get("timeout"), - timeout, - ) - - retries = self.url_retries - try: - retries = int(self.ds_cfg.get("retries", self.url_retries)) - except Exception: - util.logexc( - LOG, - "Config retries '%s' is not an int, using default '%s'", - self.ds_cfg.get("retries"), - retries, - ) - - sec_between_retries = self.url_sec_between_retries - try: - sec_between_retries = int( - self.ds_cfg.get( - "sec_between_retries", self.url_sec_between_retries - ) - ) - except Exception: - util.logexc( - LOG, - "Config sec_between_retries '%s' is not an int," - " using default '%s'", - self.ds_cfg.get("sec_between_retries"), - sec_between_retries, - ) - - return URLParams(max_wait, timeout, retries, sec_between_retries) - - def get_userdata(self, apply_filter=False): - if self.userdata is None: - self.userdata = self.ud_proc.process(self.get_userdata_raw()) - if apply_filter: - return self._filter_xdata(self.userdata) - return self.userdata - - def get_vendordata(self): - if self.vendordata is None: - self.vendordata = self.ud_proc.process(self.get_vendordata_raw()) - return self.vendordata - - def get_vendordata2(self): - if self.vendordata2 is None: - self.vendordata2 = self.ud_proc.process(self.get_vendordata2_raw()) - return self.vendordata2 - - @property - def platform_type(self): - if not self._platform_type: - self._platform_type = self.dsname.lower() - return self._platform_type - - @property - def subplatform(self): - """Return a string representing subplatform details for the datasource. - - This should be guidance for where the metadata is sourced. - Examples of this on different clouds: - ec2: metadata (http://169.254.169.254) - openstack: configdrive (/dev/path) - openstack: metadata (http://169.254.169.254) - nocloud: seed-dir (/seed/dir/path) - lxd: nocloud (/seed/dir/path) - """ - if not self._subplatform: - self._subplatform = self._get_subplatform() - return self._subplatform - - def _get_subplatform(self): - """Subclasses should implement to return a "slug (detail)" string.""" - if self.metadata_address: - return f"metadata ({self.metadata_address})" - return METADATA_UNKNOWN - - @property - def cloud_name(self): - """Return lowercase cloud name as determined by the datasource. - - Datasource can determine or define its own cloud product name in - metadata. - """ - if self._cloud_name: - return self._cloud_name - if self.metadata and self.metadata.get(METADATA_CLOUD_NAME_KEY): - cloud_name = self.metadata.get(METADATA_CLOUD_NAME_KEY) - if isinstance(cloud_name, str): - self._cloud_name = cloud_name.lower() - else: - self._cloud_name = self._get_cloud_name().lower() - LOG.debug( - "Ignoring metadata provided key %s: non-string type %s", - METADATA_CLOUD_NAME_KEY, - type(cloud_name), - ) - else: - self._cloud_name = self._get_cloud_name().lower() - return self._cloud_name - - def _get_cloud_name(self): - """Return the datasource name as it frequently matches cloud name. - - Should be overridden in subclasses which can run on multiple - cloud names, such as DatasourceEc2. - """ - return self.dsname - - @property - def launch_index(self): - if not self.metadata: - return None - if "launch-index" in self.metadata: - return self.metadata["launch-index"] - return None - - def _filter_xdata(self, processed_ud): - filters = [ - launch_index.Filter(util.safe_int(self.launch_index)), - ] - new_ud = processed_ud - for f in filters: - new_ud = f.apply(new_ud) - return new_ud - - def get_userdata_raw(self): - return self.userdata_raw - - def get_vendordata_raw(self): - return self.vendordata_raw - - def get_vendordata2_raw(self): - return self.vendordata2_raw - - # the data sources' config_obj is a cloud-config formatted - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return {} - - def get_public_ssh_keys(self): - return normalize_pubkey_data(self.metadata.get("public-keys")) - - def publish_host_keys(self, hostkeys): - """Publish the public SSH host keys (found in /etc/ssh/*.pub). - - @param hostkeys: List of host key tuples (key_type, key_value), - where key_type is the first field in the public key file - (e.g. 'ssh-rsa') and key_value is the key itself - (e.g. 'AAAAB3NzaC1y...'). - """ - - def _remap_device(self, short_name): - # LP: #611137 - # the metadata service may believe that devices are named 'sda' - # when the kernel named them 'vda' or 'xvda' - # we want to return the correct value for what will actually - # exist in this instance - mappings = {"sd": ("vd", "xvd", "vtb")} - for nfrom, tlist in mappings.items(): - if not short_name.startswith(nfrom): - continue - for nto in tlist: - cand = "/dev/%s%s" % (nto, short_name[len(nfrom) :]) - if os.path.exists(cand): - return cand - return None - - def device_name_to_device(self, _name): - # translate a 'name' to a device - # the primary function at this point is on ec2 - # to consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - return None - - def get_locale(self): - """Default locale is en_US.UTF-8, but allow distros to override""" - locale = self.default_locale - try: - locale = self.distro.get_locale() - except NotImplementedError: - pass - return locale - - @property - def availability_zone(self): - top_level_az = self.metadata.get( - "availability-zone", self.metadata.get("availability_zone") - ) - if top_level_az: - return top_level_az - return self.metadata.get("placement", {}).get("availability-zone") - - @property - def region(self): - return self.metadata.get("region") - - def get_instance_id(self): - if not self.metadata or "instance-id" not in self.metadata: - # Return a magic not really instance id string - return "iid-datasource" - return str(self.metadata["instance-id"]) - - def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): - """Get hostname or fqdn from the datasource. Look it up if desired. - - @param fqdn: Boolean, set True to return hostname with domain. - @param resolve_ip: Boolean, set True to attempt to resolve an ipv4 - address provided in local-hostname meta-data. - @param metadata_only: Boolean, set True to avoid looking up hostname - if meta-data doesn't have local-hostname present. - - @return: a DataSourceHostname namedtuple - , (str, bool). - is_default is a bool and - it's true only if hostname is localhost and was - returned by util.get_hostname() as a default. - This is used to differentiate with a user-defined - localhost hostname. - Optionally return (None, False) when - metadata_only is True and local-hostname data is not available. - """ - defdomain = "localdomain" - defhost = "localhost" - domain = defdomain - is_default = False - - if not self.metadata or not self.metadata.get("local-hostname"): - if metadata_only: - return DataSourceHostname(None, is_default) - # this is somewhat questionable really. - # the cloud datasource was asked for a hostname - # and didn't have one. raising error might be more appropriate - # but instead, basically look up the existing hostname - toks = [] - hostname = util.get_hostname() - if hostname == "localhost": - # default hostname provided by socket.gethostname() - is_default = True - hosts_fqdn = util.get_fqdn_from_hosts(hostname) - if hosts_fqdn and hosts_fqdn.find(".") > 0: - toks = str(hosts_fqdn).split(".") - elif hostname and hostname.find(".") > 0: - toks = str(hostname).split(".") - elif hostname: - toks = [hostname, defdomain] - else: - toks = [defhost, defdomain] - else: - # if there is an ipv4 address in 'local-hostname', then - # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx - lhost = self.metadata["local-hostname"] - if net.is_ipv4_address(lhost): - toks = [] - if resolve_ip: - toks = util.gethostbyaddr(lhost) - - if toks: - toks = str(toks).split(".") - else: - toks = ["ip-%s" % lhost.replace(".", "-")] - else: - toks = lhost.split(".") - - if len(toks) > 1: - hostname = toks[0] - domain = ".".join(toks[1:]) - else: - hostname = toks[0] - - if fqdn and domain != defdomain: - hostname = "%s.%s" % (hostname, domain) - - return DataSourceHostname(hostname, is_default) - - def get_package_mirror_info(self): - return self.distro.get_package_mirror_info(data_source=self) - - def get_supported_events(self, source_event_types: List[EventType]): - supported_events: Dict[EventScope, set] = {} - for event in source_event_types: - for ( - update_scope, - update_events, - ) in self.supported_update_events.items(): - if event in update_events: - if not supported_events.get(update_scope): - supported_events[update_scope] = set() - supported_events[update_scope].add(event) - return supported_events - - def update_metadata_if_supported( - self, source_event_types: List[EventType] - ) -> bool: - """Refresh cached metadata if the datasource supports this event. - - The datasource has a list of supported_update_events which - trigger refreshing all cached metadata as well as refreshing the - network configuration. - - @param source_event_types: List of EventTypes which may trigger a - metadata update. - - @return True if the datasource did successfully update cached metadata - due to source_event_type. - """ - supported_events = self.get_supported_events(source_event_types) - for scope, matched_events in supported_events.items(): - LOG.debug( - "Update datasource metadata and %s config due to events: %s", - scope.value, - ", ".join([event.value for event in matched_events]), - ) - # Each datasource has a cached config property which needs clearing - # Once cleared that config property will be regenerated from - # current metadata. - self.clear_cached_attrs((("_%s_config" % scope, UNSET),)) - if supported_events: - self.clear_cached_attrs() - result = self.get_data() - if result: - return True - LOG.debug( - "Datasource %s not updated for events: %s", - self, - ", ".join([event.value for event in source_event_types]), - ) - return False - - def check_instance_id(self, sys_cfg): - # quickly (local check only) if self.instance_id is still - return False - - def check_if_fallback_is_allowed(self): - """check_if_fallback_is_allowed() - Checks if a cached ds is allowed to be restored when no valid ds is - found in local mode by checking instance-id and searching valid data - through ds list. - - @return True if a ds allows fallback, False otherwise. - """ - return False - - @staticmethod - def _determine_dsmode(candidates, default=None, valid=None): - # return the first candidate that is non None, warn if not valid - if default is None: - default = DSMODE_NETWORK - - if valid is None: - valid = VALID_DSMODES - - for candidate in candidates: - if candidate is None: - continue - if candidate in valid: - return candidate - else: - LOG.warning( - "invalid dsmode '%s', using default=%s", candidate, default - ) - return default - - return default - - @property - def network_config(self): - return None - - def setup(self, is_new_instance): - """setup(is_new_instance) - - This is called before user-data and vendor-data have been processed. - - Unless the datasource has set mode to 'local', then networking - per 'fallback' or per 'network_config' will have been written and - brought up the OS at this point. - """ - return - - def activate(self, cfg, is_new_instance): - """activate(cfg, is_new_instance) - - This is called before the init_modules will be called but after - the user-data and vendor-data have been fully processed. - - The cfg is fully up to date config, it contains a merged view of - system config, datasource config, user config, vendor config. - It should be used rather than the sys_cfg passed to __init__. - - is_new_instance is a boolean indicating if this is a new instance. - """ - return - - -def normalize_pubkey_data(pubkey_data): - keys = [] - - if not pubkey_data: - return keys - - if isinstance(pubkey_data, str): - return pubkey_data.splitlines() - - if isinstance(pubkey_data, (list, set)): - return list(pubkey_data) - - if isinstance(pubkey_data, (dict)): - for _keyname, klist in pubkey_data.items(): - # lp:506332 uec metadata service responds with - # data that makes boto populate a string for 'klist' rather - # than a list. - if isinstance(klist, str): - klist = [klist] - if isinstance(klist, (list, set)): - for pkey in klist: - # There is an empty string at - # the end of the keylist, trim it - if pkey: - keys.append(pkey) - - return keys - - -def find_source( - sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter -) -> Tuple[DataSource, str]: - ds_list = list_sources(cfg_list, ds_deps, pkg_list) - ds_names = [type_utils.obj_name(f) for f in ds_list] - mode = "network" if DEP_NETWORK in ds_deps else "local" - LOG.debug("Searching for %s data source in: %s", mode, ds_names) - - for name, cls in zip(ds_names, ds_list): - myrep = events.ReportEventStack( - name="search-%s" % name.replace("DataSource", ""), - description="searching for %s data from %s" % (mode, name), - message="no %s data found from %s" % (mode, name), - parent=reporter, - ) - try: - with myrep: - LOG.debug("Seeing if we can get any data from %s", cls) - s = cls(sys_cfg, distro, paths) - if s.update_metadata_if_supported( - [EventType.BOOT_NEW_INSTANCE] - ): - myrep.message = "found %s data from %s" % (mode, name) - return (s, type_utils.obj_name(cls)) - except Exception: - util.logexc(LOG, "Getting data from %s failed", cls) - - msg = "Did not find any data source, searched classes: (%s)" % ", ".join( - ds_names - ) - raise DataSourceNotFoundException(msg) - - -def list_sources(cfg_list, depends, pkg_list): - """Return a list of classes that have the same depends as 'depends' - iterate through cfg_list, loading "DataSource*" modules - and calling their "get_datasource_list". - Return an ordered list of classes that match (if any) - """ - src_list = [] - LOG.debug( - "Looking for data source in: %s," - " via packages %s that matches dependencies %s", - cfg_list, - pkg_list, - depends, - ) - - for ds in cfg_list: - ds_name = importer.match_case_insensitive_module_name(ds) - m_locs, _looked_locs = importer.find_module( - ds_name, pkg_list, ["get_datasource_list"] - ) - if not m_locs: - LOG.error( - "Could not import %s. Does the DataSource exist and " - "is it importable?", - ds_name, - ) - for m_loc in m_locs: - mod = importer.import_module(m_loc) - lister = getattr(mod, "get_datasource_list") - matches = lister(depends) - if matches: - src_list.extend(matches) - break - return src_list - - -def instance_id_matches_system_uuid( - instance_id, field: str = "system-uuid" -) -> bool: - # quickly (local check only) if self.instance_id is still valid - # we check kernel command line or files. - if not instance_id: - return False - - dmi_value = dmi.read_dmi_data(field) - if not dmi_value: - return False - return instance_id.lower() == dmi_value.lower() - - -def canonical_cloud_id(cloud_name, region, platform): - """Lookup the canonical cloud-id for a given cloud_name and region.""" - if not cloud_name: - cloud_name = METADATA_UNKNOWN - if not region: - region = METADATA_UNKNOWN - if region == METADATA_UNKNOWN: - if cloud_name != METADATA_UNKNOWN: - return cloud_name - return platform - for prefix, cloud_id_test in CLOUD_ID_REGION_PREFIX_MAP.items(): - (cloud_id, valid_cloud) = cloud_id_test - if region.startswith(prefix) and valid_cloud(cloud_name): - return cloud_id - if cloud_name != METADATA_UNKNOWN: - return cloud_name - return platform - - -def convert_vendordata(data, recurse=True): - """data: a loaded object (strings, arrays, dicts). - return something suitable for cloudinit vendordata_raw. - - if data is: - None: return None - string: return string - list: return data - the list is then processed in UserDataProcessor - dict: return convert_vendordata(data.get('cloud-init')) - """ - if not data: - return None - if isinstance(data, str): - return data - if isinstance(data, list): - return copy.deepcopy(data) - if isinstance(data, dict): - if recurse is True: - return convert_vendordata(data.get("cloud-init"), recurse=False) - raise ValueError("vendordata['cloud-init'] cannot be dict") - raise ValueError("Unknown data type for vendordata: %s" % type(data)) - - -class BrokenMetadata(IOError): - pass - - -# 'depends' is a list of dependencies (DEP_FILESYSTEM) -# ds_list is a list of 2 item lists -# ds_list = [ -# ( class, ( depends-that-this-class-needs ) ) -# } -# It returns a list of 'class' that matched these deps exactly -# It mainly is a helper function for DataSourceCollections -def list_from_depends(depends, ds_list): - ret_list = [] - depset = set(depends) - for cls, deps in ds_list: - if depset == set(deps): - ret_list.append(cls) - return ret_list - - -def pkl_store(obj: DataSource, fname: str) -> bool: - """Use pickle to serialize Datasource to a file as a cache. - - :return: True on success - """ - try: - pk_contents = pickle.dumps(obj) - except Exception: - util.logexc(LOG, "Failed pickling datasource %s", obj) - return False - try: - util.write_file(fname, pk_contents, omode="wb", mode=0o400) - except Exception: - util.logexc(LOG, "Failed pickling datasource to %s", fname) - return False - return True - - -def pkl_load(fname: str) -> Optional[DataSource]: - """Use pickle to deserialize a instance Datasource from a cache file.""" - pickle_contents = None - try: - pickle_contents = util.load_binary_file(fname) - except Exception as e: - if os.path.isfile(fname): - LOG.warning("failed loading pickle in %s: %s", fname, e) - - # This is allowed so just return nothing successfully loaded... - if not pickle_contents: - return None - try: - return pickle.loads(pickle_contents) - except DatasourceUnpickleUserDataError: - return None - except Exception: - util.logexc(LOG, "Failed loading pickled blob from %s", fname) - return None - - -def parse_cmdline() -> str: - """Check if command line argument for this datasource was passed - Passing by command line overrides runtime datasource detection - """ - return parse_cmdline_or_dmi(util.get_cmdline()) - - -def parse_cmdline_or_dmi(input: str) -> str: - ds_parse_0 = re.search(r"(?:^|\s)ds=([^\s;]+)", input) - ds_parse_1 = re.search(r"(?:^|\s)ci\.ds=([^\s;]+)", input) - ds_parse_2 = re.search(r"(?:^|\s)ci\.datasource=([^\s;]+)", input) - ds = ds_parse_0 or ds_parse_1 or ds_parse_2 - deprecated = ds_parse_1 or ds_parse_2 - if deprecated: - dsname = deprecated.group(1).strip() - lifecycle.deprecate( - deprecated=( - f"Defining the datasource on the command line using " - f"ci.ds={dsname} or " - f"ci.datasource={dsname}" - ), - deprecated_version="23.2", - extra_message=f"Use ds={dsname} instead", - ) - if ds and ds.group(1): - return ds.group(1) - return "" diff --git a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/doc/module-docs/cc_install_hotplug/data.yaml b/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/doc/module-docs/cc_install_hotplug/data.yaml deleted file mode 100644 index 32369719..00000000 --- a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/doc/module-docs/cc_install_hotplug/data.yaml +++ /dev/null @@ -1,21 +0,0 @@ -cc_install_hotplug: - description: | - This module will install the udev rules to enable hotplug if supported by - the datasource and enabled in the userdata. The udev rules will be - installed as ``/etc/udev/rules.d/90-cloud-init-hook-hotplug.rules``. - - When hotplug is enabled, newly added network devices will be added to the - system by cloud-init. After udev detects the event, cloud-init will - refresh the instance metadata from the datasource, detect the device in - the updated metadata, then apply the updated network configuration. - - Currently supported datasources: Openstack, EC2 - examples: - - comment: | - Example 1: Enable hotplug of network devices - file: cc_install_hotplug/example1.yaml - - comment: | - Example 2: Enable network hotplug alongside boot event - file: cc_install_hotplug/example2.yaml - name: Install Hotplug - title: Install hotplug udev rules if supported and enabled diff --git a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/tests/integration_tests/modules/test_hotplug.py b/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/tests/integration_tests/modules/test_hotplug.py deleted file mode 100644 index 3d09ee09..00000000 --- a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/tests/integration_tests/modules/test_hotplug.py +++ /dev/null @@ -1,557 +0,0 @@ -import time -from collections import namedtuple - -import paramiko -import pytest -import yaml -from pycloudlib.ec2.instance import EC2Instance - -from cloudinit.subp import subp -from tests.integration_tests.clouds import IntegrationCloud -from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.integration_settings import PLATFORM -from tests.integration_tests.releases import ( - CURRENT_RELEASE, - FOCAL, - UBUNTU_STABLE, -) -from tests.integration_tests.util import ( - push_and_enable_systemd_unit, - verify_clean_boot, - verify_clean_log, - wait_for_cloud_init, -) - -USER_DATA = """\ -#cloud-config -updates: - network: - when: ['hotplug'] -""" - -USER_DATA_HOTPLUG_DISABLED = """\ -#cloud-config -updates: - network: - when: ['boot-new-instance'] -""" - -ip_addr = namedtuple("ip_addr", "interface state ip4 ip6") - - -def _wait_till_hotplug_complete(client, expected_runs=1): - for _ in range(60): - if client.execute("command -v systemctl").ok: - if "failed" == client.execute( - "systemctl is-active cloud-init-hotplugd.service" - ): - r = client.execute( - "systemctl status cloud-init-hotplugd.service" - ) - if not r.ok: - raise AssertionError( - "cloud-init-hotplugd.service failed: {r.stdout}" - ) - - log = client.read_from_file("/var/log/cloud-init.log") - if log.count("Exiting hotplug handler") == expected_runs: - return log - time.sleep(1) - raise Exception("Waiting for hotplug handler failed") - - -def _get_ip_addr(client, *, _retries: int = 0): - ips = [] - lines = client.execute("ip --brief addr").split("\n") - for line in lines: - attributes = line.split() - interface, state = attributes[0], attributes[1] - ip4_cidr = attributes[2] if len(attributes) > 2 else None - - # Retry to wait for ipv6_cidr: - # ens6 UP metric 200 - if len(attributes) == 6 and _retries < 3: - time.sleep(1) - return _get_ip_addr(client, _retries=_retries + 1) - - # The output of `ip --brief addr` can contain metric info: - # ens5 UP metric 100 ... - ip6_cidr = None - if len(attributes) > 3: - if attributes[3] != "metric": - ip6_cidr = attributes[3] - elif len(attributes) > 5: - ip6_cidr = attributes[5] - ip4 = ip4_cidr.split("/")[0] if ip4_cidr else None - ip6 = ip6_cidr.split("/")[0] if ip6_cidr else None - ip = ip_addr(interface, state, ip4, ip6) - ips.append(ip) - return ips - - -@pytest.mark.skipif( - PLATFORM != "openstack", - reason=( - "Test was written for openstack but can likely run on other platforms." - ), -) -@pytest.mark.skipif( - CURRENT_RELEASE < FOCAL, - reason="Openstack network metadata support was added in focal.", -) -@pytest.mark.user_data(USER_DATA) -def test_hotplug_add_remove(client: IntegrationInstance): - ips_before = _get_ip_addr(client) - log = client.read_from_file("/var/log/cloud-init.log") - assert "Exiting hotplug handler" not in log - assert client.execute( - "test -f /etc/udev/rules.d/90-cloud-init-hook-hotplug.rules" - ).ok - - # Add new NIC - added_ip = client.instance.add_network_interface() - _wait_till_hotplug_complete(client, expected_runs=1) - ips_after_add = _get_ip_addr(client) - new_addition = [ip for ip in ips_after_add if ip.ip4 == added_ip][0] - - assert len(ips_after_add) == len(ips_before) + 1 - assert added_ip not in [ip.ip4 for ip in ips_before] - assert added_ip in [ip.ip4 for ip in ips_after_add] - assert new_addition.state == "UP" - - netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") - config = yaml.safe_load(netplan_cfg) - assert new_addition.interface in config["network"]["ethernets"] - - # Remove new NIC - client.instance.remove_network_interface(added_ip) - _wait_till_hotplug_complete(client, expected_runs=2) - ips_after_remove = _get_ip_addr(client) - assert len(ips_after_remove) == len(ips_before) - assert added_ip not in [ip.ip4 for ip in ips_after_remove] - - netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") - config = yaml.safe_load(netplan_cfg) - assert new_addition.interface not in config["network"]["ethernets"] - - assert "enabled" == client.execute( - "cloud-init devel hotplug-hook -s net query" - ) - - -@pytest.mark.skipif( - PLATFORM not in ["lxd_container", "lxd_vm", "ec2", "openstack", "azure"], - reason=(f"HOTPLUG is not supported in {PLATFORM}."), -) -def _test_hotplug_enabled_by_cmd(client: IntegrationInstance): - assert "disabled" == client.execute( - "cloud-init devel hotplug-hook -s net query" - ) - ret = client.execute("cloud-init devel hotplug-hook -s net enable") - assert ret.ok, ret.stderr - log = client.read_from_file("/var/log/cloud-init.log") - assert ( - "hotplug-hook called with the following arguments: " - "{hotplug_action: enable" in log - ) - - assert "enabled" == client.execute( - "cloud-init devel hotplug-hook -s net query" - ) - log = client.read_from_file("/var/log/cloud-init.log") - assert ( - "hotplug-hook called with the following arguments: " - "{hotplug_action: query" in log - ) - assert client.execute( - "test -f /etc/udev/rules.d/90-cloud-init-hook-hotplug.rules" - ).ok - - -@pytest.mark.user_data(USER_DATA_HOTPLUG_DISABLED) -def test_hotplug_enable_cmd(client: IntegrationInstance): - _test_hotplug_enabled_by_cmd(client) - - -@pytest.mark.skipif( - PLATFORM != "ec2", - reason=( - f"Test was written for {PLATFORM} but can likely run on " - "other platforms." - ), -) -@pytest.mark.user_data(USER_DATA_HOTPLUG_DISABLED) -def test_hotplug_enable_cmd_ec2(client: IntegrationInstance): - _test_hotplug_enabled_by_cmd(client) - ips_before = _get_ip_addr(client) - - # Add new NIC - added_ip = client.instance.add_network_interface() - _wait_till_hotplug_complete(client, expected_runs=4) - ips_after_add = _get_ip_addr(client) - new_addition = [ip for ip in ips_after_add if ip.ip4 == added_ip][0] - - assert len(ips_after_add) == len(ips_before) + 1 - assert added_ip not in [ip.ip4 for ip in ips_before] - assert added_ip in [ip.ip4 for ip in ips_after_add] - assert new_addition.state == "UP" - - netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") - config = yaml.safe_load(netplan_cfg) - assert new_addition.interface in config["network"]["ethernets"] - - # Remove new NIC - client.instance.remove_network_interface(added_ip) - _wait_till_hotplug_complete(client, expected_runs=5) - ips_after_remove = _get_ip_addr(client) - assert len(ips_after_remove) == len(ips_before) - assert added_ip not in [ip.ip4 for ip in ips_after_remove] - - netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") - config = yaml.safe_load(netplan_cfg) - assert new_addition.interface not in config["network"]["ethernets"] - - -@pytest.mark.skipif( - PLATFORM != "openstack", - reason=( - "Test was written for openstack but can likely run on other platforms." - ), -) -def test_no_hotplug_in_userdata(client: IntegrationInstance): - ips_before = _get_ip_addr(client) - log = client.read_from_file("/var/log/cloud-init.log") - assert "Exiting hotplug handler" not in log - assert client.execute( - "test -f /etc/udev/rules.d/90-cloud-init-hook-hotplug.rules" - ).failed - - # Add new NIC - client.instance.add_network_interface() - log = client.read_from_file("/var/log/cloud-init.log") - assert "hotplug-hook" not in log - - ips_after_add = _get_ip_addr(client) - if len(ips_after_add) == len(ips_before) + 1: - # We can see the device, but it should not have been brought up - new_ip = [ip for ip in ips_after_add if ip not in ips_before][0] - assert new_ip.state == "DOWN" - else: - assert len(ips_after_add) == len(ips_before) - - assert "disabled" == client.execute( - "cloud-init devel hotplug-hook -s net query" - ) - - -@pytest.mark.skipif(PLATFORM != "ec2", reason="test is ec2 specific") -@pytest.mark.user_data(USER_DATA) -def test_multi_nic_hotplug(client: IntegrationInstance): - """Tests that additional secondary NICs are routable from non-local - networks after the hotplug hook is executed when network updates - are configured on the HOTPLUG event.""" - ips_before = _get_ip_addr(client) - secondary_priv_ip = client.instance.add_network_interface( - ipv4_public_ip_count=1, - ) - _wait_till_hotplug_complete(client, expected_runs=1) - - log_content = client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log_content) - verify_clean_boot(client) - - ips_after_add = _get_ip_addr(client) - - netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") - config = yaml.safe_load(netplan_cfg) - new_addition = [ip for ip in ips_after_add if ip.ip4 == secondary_priv_ip][ - 0 - ] - assert new_addition.interface in config["network"]["ethernets"] - new_nic_cfg = config["network"]["ethernets"][new_addition.interface] - assert [{"from": secondary_priv_ip, "table": 101}] == new_nic_cfg[ - "routing-policy" - ] - - assert len(ips_after_add) == len(ips_before) + 1 - - # help mypy realize client.instance is an instance of EC2Instance as - # public_ips is only available on ec2 instances - assert isinstance(client.instance, EC2Instance) - public_ips = client.instance.public_ips - assert len(public_ips) == 2 - - # SSH over all public ips works - for pub_ip in public_ips: - subp("nc -w 10 -zv " + pub_ip + " 22", shell=True) - - # Remove new NIC - client.instance.remove_network_interface(secondary_priv_ip) - _wait_till_hotplug_complete(client, expected_runs=2) - - public_ips = client.instance.public_ips - assert len(public_ips) == 1 - # SSH over primary NIC works - subp("nc -w 10 -zv " + public_ips[0] + " 22", shell=True) - - ips_after_remove = _get_ip_addr(client) - assert len(ips_after_remove) == len(ips_before) - assert secondary_priv_ip not in [ip.ip4 for ip in ips_after_remove] - - netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") - config = yaml.safe_load(netplan_cfg) - assert new_addition.interface not in config["network"]["ethernets"] - - log_content = client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log_content) - verify_clean_boot(client) - - -@pytest.mark.skipif(PLATFORM != "ec2", reason="test is ec2 specific") -@pytest.mark.skip(reason="IMDS race, see GH-5373. Unskip when fixed.") -def test_multi_nic_hotplug_vpc(setup_image, session_cloud: IntegrationCloud): - """Tests that additional secondary NICs are routable from local - networks after the hotplug hook is executed when network updates - are configured on the HOTPLUG event.""" - with session_cloud.launch( - user_data=USER_DATA - ) as client, session_cloud.launch() as bastion: - ips_before = _get_ip_addr(client) - primary_priv_ip4 = ips_before[1].ip4 - primary_priv_ip6 = ips_before[1].ip6 - client.instance.add_network_interface(ipv6_address_count=1) - - _wait_till_hotplug_complete(client) - log_content = client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log_content) - verify_clean_boot(client) - - netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") - config = yaml.safe_load(netplan_cfg) - - ips_after_add = _get_ip_addr(client) - secondary_priv_ip4 = ips_after_add[2].ip4 - secondary_priv_ip6 = ips_after_add[2].ip6 - assert primary_priv_ip4 != secondary_priv_ip4 - - new_addition = [ - ip for ip in ips_after_add if ip.ip4 == secondary_priv_ip4 - ][0] - assert new_addition.interface in config["network"]["ethernets"] - new_nic_cfg = config["network"]["ethernets"][new_addition.interface] - assert "routing-policy" in new_nic_cfg - assert [ - {"from": secondary_priv_ip4, "table": 101}, - {"from": secondary_priv_ip6, "table": 101}, - ] == new_nic_cfg["routing-policy"] - - assert len(ips_after_add) == len(ips_before) + 1 - - # pings to primary and secondary NICs work - r = bastion.execute(f"ping -c1 {primary_priv_ip4}") - assert r.ok, r.stdout - r = bastion.execute(f"ping -c1 {secondary_priv_ip4}") - assert r.ok, r.stdout - r = bastion.execute(f"ping -c1 {primary_priv_ip6}") - assert r.ok, r.stdout - r = bastion.execute(f"ping -c1 {secondary_priv_ip6}") - assert r.ok, r.stdout - - # Check every route has metrics associated. See LP: #2055397 - ip_route_show = client.execute("ip route show") - assert ip_route_show.ok, ip_route_show.stderr - for route in ip_route_show.splitlines(): - assert "metric" in route, "Expected metric to be in the route" - - # Remove new NIC - client.instance.remove_network_interface(secondary_priv_ip4) - _wait_till_hotplug_complete(client, expected_runs=2) - - # ping to primary NIC works - assert bastion.execute(f"ping -c1 {primary_priv_ip4}").ok - assert bastion.execute(f"ping -c1 {primary_priv_ip6}").ok - - log_content = client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log_content) - verify_clean_boot(client) - - -@pytest.mark.skipif(PLATFORM != "ec2", reason="test is ec2 specific") -@pytest.mark.skipif( - CURRENT_RELEASE not in UBUNTU_STABLE, - reason="Docker repo does not contain pkgs for non stable releases.", -) -@pytest.mark.user_data(USER_DATA) -def test_no_hotplug_triggered_by_docker(client: IntegrationInstance): - # Install docker - r = client.execute("curl -fsSL https://get.docker.com | sh") - assert r.ok, r.stderr - - # Start and stop a container - r = client.execute("docker run -dit --name ff ubuntu:focal") - assert r.ok, r.stderr - r = client.execute("docker stop ff") - assert r.ok, r.stderr - - # Verify hotplug-hook was not called - log = client.read_from_file("/var/log/cloud-init.log") - assert "Exiting hotplug handler" not in log - assert "hotplug-hook" not in log - - # Verify hotplug was enabled - assert "enabled" == client.execute( - "cloud-init devel hotplug-hook -s net query" - ) - - -def wait_for_cmd( - client: IntegrationInstance, cmd: str, return_code: int -) -> None: - for _ in range(60): - try: - res = client.execute(cmd) - except paramiko.ssh_exception.SSHException: - pass - else: - if res.return_code == return_code: - return - time.sleep(1) - assert False, f"`{cmd}` never exited with {return_code}" - - -def assert_systemctl_status_code( - client: IntegrationInstance, service: str, return_code: int -): - result = client.execute(f"systemctl status {service}") - assert result.return_code == return_code, ( - f"status of {service} expected to be {return_code} but was" - f" {result.return_code}\nstdout: {result.stdout}\n" - f"stderr {result.stderr}" - ) - - -BLOCK_CLOUD_CONFIG = """\ -[Unit] -Description=Block cloud-config.service -After=cloud-config.target -Before=cloud-config.service - -DefaultDependencies=no -Before=shutdown.target -Conflicts=shutdown.target - -[Service] -Type=oneshot -ExecStart=/usr/bin/sleep 360 -TimeoutSec=0 - -# Output needs to appear in instance console output -StandardOutput=journal+console - -[Install] -WantedBy=cloud-config.service -""" # noqa: E501 - - -BLOCK_CLOUD_FINAL = """\ -[Unit] -Description=Block cloud-final.service -After=cloud-config.target -Before=cloud-final.service - -DefaultDependencies=no -Before=shutdown.target -Conflicts=shutdown.target - -[Service] -Type=oneshot -ExecStart=/usr/bin/sleep 360 -TimeoutSec=0 - -# Output needs to appear in instance console output -StandardOutput=journal+console - -[Install] -WantedBy=cloud-final.service -""" # noqa: E501 - - -def _customize_environment(client: IntegrationInstance): - push_and_enable_systemd_unit( - client, "block-cloud-config.service", BLOCK_CLOUD_CONFIG - ) - push_and_enable_systemd_unit( - client, "block-cloud-final.service", BLOCK_CLOUD_FINAL - ) - - # Disable pam_nologin for 1000(ubuntu) user to allow ssh access early - # during boot. Without this we get: - # - # System is booting up. Unprivileged users are not permitted to log in yet. - # Please come back later. For technical details, see pam_nologin(8). - # - # sshd[xxx]: fatal: Access denied for user ubuntu by PAM account - # configuration [preauth] - # - # See: pam(7), pam_nologin(8), pam_succeed_id(8) - contents = client.read_from_file("/etc/pam.d/sshd") - contents = ( - "account [success=1 default=ignore] pam_succeed_if.so quiet uid eq" - " 1000\n\n" + contents - ) - client.write_to_file("/etc/pam.d/sshd", contents) - - client.instance.shutdown(wait=True) - client.instance.start(wait=False) - - -@pytest.mark.skipif( - PLATFORM != "ec2", - reason="test is ec2 specific but should work on other platforms with the" - " ability to add_network_interface", -) -@pytest.mark.user_data(USER_DATA) -def test_nics_before_config_trigger_hotplug(client: IntegrationInstance): - """ - Test that NICs added/removed after the Network boot stage but before - the rest boot stages do trigger cloud-init-hotplugd. - - Note: Do not test first boot, as cc_install_hotplug runs at - config-final.service time. - """ - _customize_environment(client) - - # wait until we are between cloud-config.target done and - # cloud-config.service - wait_for_cmd(client, "systemctl status cloud-config.target", 0) - wait_for_cmd(client, "systemctl status block-cloud-config.service", 3) - - # socket active but service not - assert_systemctl_status_code(client, "cloud-init-hotplugd.socket", 0) - assert_systemctl_status_code(client, "cloud-init-hotplugd.service", 3) - - assert_systemctl_status_code(client, "cloud-config.service", 3) - assert_systemctl_status_code(client, "cloud-final.service", 3) - - added_ip_0 = client.instance.add_network_interface() - - assert_systemctl_status_code(client, "cloud-config.service", 3) - assert_systemctl_status_code(client, "cloud-final.service", 3) - - # unblock cloud-config.service - assert client.execute("systemctl stop block-cloud-config.service").ok - wait_for_cmd(client, "systemctl status cloud-config.service", 0) - wait_for_cmd(client, "systemctl status block-cloud-final.service", 3) - assert_systemctl_status_code(client, "cloud-final.service", 3) - - # hotplug didn't run before cloud-final.service - _wait_till_hotplug_complete(client, expected_runs=0) - - # unblock cloud-final.service - assert client.execute("systemctl stop block-cloud-final.service").ok - - wait_for_cloud_init(client) - _wait_till_hotplug_complete(client, expected_runs=1) - - client.instance.remove_network_interface(added_ip_0) - _wait_till_hotplug_complete(client, expected_runs=2) diff --git a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/tests/unittests/cmd/devel/test_hotplug_hook.py b/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/tests/unittests/cmd/devel/test_hotplug_hook.py deleted file mode 100644 index 0c8de26b..00000000 --- a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/tests/unittests/cmd/devel/test_hotplug_hook.py +++ /dev/null @@ -1,359 +0,0 @@ -from collections import namedtuple -from typing import Any, NamedTuple -from unittest import mock -from unittest.mock import call - -import pytest - -from cloudinit import settings -from cloudinit.cmd.devel.hotplug_hook import enable_hotplug, handle_hotplug -from cloudinit.distros import Distro -from cloudinit.event import EventScope, EventType -from cloudinit.net.activators import NetworkActivator -from cloudinit.net.network_state import NetworkState -from cloudinit.sources import DataSource -from cloudinit.stages import Init - -M_PATH = "cloudinit.cmd.devel.hotplug_hook." - -hotplug_args = namedtuple("hotplug_args", "udevaction, subsystem, devpath") -FAKE_MAC = "11:22:33:44:55:66" - - -class Mocks(NamedTuple): - m_init: Any - m_network_state: Any - m_activator: Any - m_sleep: Any - - -@pytest.fixture -def mocks(): - m_init = mock.MagicMock(spec=Init) - m_activator = mock.MagicMock(spec=NetworkActivator) - m_distro = mock.MagicMock(spec=Distro) - m_distro.network_activator = mock.PropertyMock(return_value=m_activator) - m_datasource = mock.MagicMock(spec=DataSource) - m_datasource.distro = m_distro - m_datasource.skip_hotplug_detect = False - m_init.datasource = m_datasource - m_init.fetch.return_value = m_datasource - - read_sys_net = mock.patch( - "cloudinit.cmd.devel.hotplug_hook.read_sys_net_safe", - return_value=FAKE_MAC, - ) - - update_event_enabled = mock.patch( - "cloudinit.stages.update_event_enabled", - return_value=True, - ) - - m_network_state = mock.MagicMock(spec=NetworkState) - parse_net = mock.patch( - "cloudinit.cmd.devel.hotplug_hook.parse_net_config_data", - return_value=m_network_state, - ) - - sleep = mock.patch("time.sleep") - - read_sys_net.start() - update_event_enabled.start() - parse_net.start() - m_sleep = sleep.start() - - yield Mocks( - m_init=m_init, - m_network_state=m_network_state, - m_activator=m_activator, - m_sleep=m_sleep, - ) - - read_sys_net.stop() - update_event_enabled.stop() - parse_net.stop() - sleep.stop() - - -class TestUnsupportedActions: - def test_unsupported_subsystem(self, mocks): - with pytest.raises( - Exception, match="cannot handle events for subsystem: not_real" - ): - handle_hotplug( - hotplug_init=mocks.m_init, - devpath="/dev/fake", - subsystem="not_real", - udevaction="add", - ) - - def test_unsupported_udevaction(self, mocks): - with pytest.raises(ValueError, match="Unknown action: not_real"): - handle_hotplug( - hotplug_init=mocks.m_init, - devpath="/dev/fake", - subsystem="net", - udevaction="not_real", - ) - - -class TestHotplug: - def test_succcessful_add(self, mocks): - init = mocks.m_init - mocks.m_network_state.iter_interfaces.return_value = [ - { - "mac_address": FAKE_MAC, - } - ] - handle_hotplug( - hotplug_init=init, - devpath="/dev/fake", - udevaction="add", - subsystem="net", - ) - init.datasource.update_metadata_if_supported.assert_called_once_with( - [EventType.HOTPLUG] - ) - mocks.m_activator.bring_up_interface.assert_called_once_with("fake") - mocks.m_activator.bring_down_interface.assert_not_called() - init._write_to_cache.assert_called_once_with() - - def test_successful_remove(self, mocks): - init = mocks.m_init - mocks.m_network_state.iter_interfaces.return_value = [{}] - handle_hotplug( - hotplug_init=init, - devpath="/dev/fake", - udevaction="remove", - subsystem="net", - ) - init.datasource.update_metadata_if_supported.assert_called_once_with( - [EventType.HOTPLUG] - ) - mocks.m_activator.bring_down_interface.assert_called_once_with("fake") - mocks.m_activator.bring_up_interface.assert_not_called() - init._write_to_cache.assert_called_once_with() - - @mock.patch( - "cloudinit.cmd.devel.hotplug_hook.NetHandler.detect_hotplugged_device" - ) - @pytest.mark.parametrize("skip", [True, False]) - def test_skip_detected(self, m_detect, skip, mocks): - mocks.m_init.datasource.skip_hotplug_detect = skip - expected_call_count = 0 if skip else 1 - handle_hotplug( - hotplug_init=mocks.m_init, - devpath="/dev/fake", - udevaction="add", - subsystem="net", - ) - assert m_detect.call_count == expected_call_count - - def test_update_event_disabled(self, mocks, caplog): - init = mocks.m_init - with mock.patch( - "cloudinit.stages.update_event_enabled", return_value=False - ): - handle_hotplug( - hotplug_init=init, - devpath="/dev/fake", - udevaction="remove", - subsystem="net", - ) - assert "hotplug not enabled for event of type" in caplog.text - init.datasource.update_metadata_if_supported.assert_not_called() - mocks.m_activator.bring_up_interface.assert_not_called() - mocks.m_activator.bring_down_interface.assert_not_called() - init._write_to_cache.assert_not_called() - - def test_update_metadata_failed(self, mocks): - mocks.m_init.datasource.update_metadata_if_supported.return_value = ( - False - ) - with pytest.raises( - RuntimeError, match="Datasource .* not updated for event hotplug" - ): - handle_hotplug( - hotplug_init=mocks.m_init, - devpath="/dev/fake", - udevaction="remove", - subsystem="net", - ) - - def test_detect_hotplugged_device_not_detected_on_add(self, mocks): - mocks.m_network_state.iter_interfaces.return_value = [{}] - with pytest.raises( - RuntimeError, - match="Failed to detect {} in updated metadata".format(FAKE_MAC), - ): - handle_hotplug( - hotplug_init=mocks.m_init, - devpath="/dev/fake", - udevaction="add", - subsystem="net", - ) - - def test_detect_hotplugged_device_detected_on_remove(self, mocks): - mocks.m_network_state.iter_interfaces.return_value = [ - { - "mac_address": FAKE_MAC, - } - ] - with pytest.raises( - RuntimeError, match="Failed to detect .* in updated metadata" - ): - handle_hotplug( - hotplug_init=mocks.m_init, - devpath="/dev/fake", - udevaction="remove", - subsystem="net", - ) - - def test_apply_failed_on_add(self, mocks): - mocks.m_network_state.iter_interfaces.return_value = [ - { - "mac_address": FAKE_MAC, - } - ] - mocks.m_activator.bring_up_interface.return_value = False - with pytest.raises( - RuntimeError, match="Failed to bring up device: /dev/fake" - ): - handle_hotplug( - hotplug_init=mocks.m_init, - devpath="/dev/fake", - udevaction="add", - subsystem="net", - ) - - def test_apply_failed_on_remove(self, mocks): - mocks.m_network_state.iter_interfaces.return_value = [{}] - mocks.m_activator.bring_down_interface.return_value = False - with pytest.raises( - RuntimeError, match="Failed to bring down device: /dev/fake" - ): - handle_hotplug( - hotplug_init=mocks.m_init, - devpath="/dev/fake", - udevaction="remove", - subsystem="net", - ) - - def test_retry(self, mocks): - with pytest.raises(RuntimeError): - handle_hotplug( - hotplug_init=mocks.m_init, - devpath="/dev/fake", - udevaction="add", - subsystem="net", - ) - assert mocks.m_sleep.call_count == 5 - assert mocks.m_sleep.call_args_list == [ - call(1), - call(3), - call(5), - call(10), - call(30), - ] - - -@pytest.mark.usefixtures("fake_filesystem") -class TestEnableHotplug: - @mock.patch(M_PATH + "util.write_file") - @mock.patch( - M_PATH + "util.read_hotplug_enabled_file", - return_value={"scopes": []}, - ) - @mock.patch(M_PATH + "install_hotplug") - def test_enabling( - self, - m_install_hotplug, - m_read_hotplug_enabled_file, - m_write_file, - mocks, - ): - mocks.m_init.datasource.get_supported_events.return_value = { - EventScope.NETWORK: {EventType.HOTPLUG} - } - mocks.m_init.paths.get_cpath.return_value = ( - "/var/lib/cloud/hotplug.enabled" - ) - - enable_hotplug(mocks.m_init, "net") - - assert [ - call([EventType.HOTPLUG]) - ] == mocks.m_init.datasource.get_supported_events.call_args_list - m_read_hotplug_enabled_file.assert_called_once() - assert [ - call( - settings.HOTPLUG_ENABLED_FILE, - '{"scopes": ["network"]}', - omode="w", - mode=0o640, - ) - ] == m_write_file.call_args_list - assert [ - call( - mocks.m_init.datasource, - network_hotplug_enabled=True, - cfg=mocks.m_init.cfg, - ) - ] == m_install_hotplug.call_args_list - - @pytest.mark.parametrize( - ["supported_events"], [({},), ({EventScope.NETWORK: {}},)] - ) - @mock.patch(M_PATH + "util.write_file") - @mock.patch( - M_PATH + "util.read_hotplug_enabled_file", - return_value={"scopes": []}, - ) - @mock.patch(M_PATH + "install_hotplug") - def test_hotplug_not_supported_in_ds( - self, - m_install_hotplug, - m_read_hotplug_enabled_file, - m_write_file, - supported_events, - mocks, - ): - mocks.m_init.datasource.get_supported_events.return_value = ( - supported_events - ) - enable_hotplug(mocks.m_init, "net") - - assert [ - call([EventType.HOTPLUG]) - ] == mocks.m_init.datasource.get_supported_events.call_args_list - assert [] == m_read_hotplug_enabled_file.call_args_list - assert [] == m_write_file.call_args_list - assert [] == m_install_hotplug.call_args_list - - @mock.patch(M_PATH + "util.write_file") - @mock.patch( - M_PATH + "util.read_hotplug_enabled_file", - return_value={"scopes": [EventScope.NETWORK.value]}, - ) - @mock.patch(M_PATH + "install_hotplug") - def test_hotplug_already_enabled_in_file( - self, - m_install_hotplug, - m_read_hotplug_enabled_file, - m_write_file, - mocks, - ): - mocks.m_init.datasource.get_supported_events.return_value = { - EventScope.NETWORK: {EventType.HOTPLUG} - } - mocks.m_init.paths.get_cpath.return_value = ( - "/var/lib/cloud/hotplug.enabled" - ) - enable_hotplug(mocks.m_init, "net") - - assert [ - call([EventType.HOTPLUG]) - ] == mocks.m_init.datasource.get_supported_events.call_args_list - m_read_hotplug_enabled_file.assert_called_once() - assert [] == m_write_file.call_args_list - assert [] == m_install_hotplug.call_args_list diff --git a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/tools/hook-hotplug b/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/tools/hook-hotplug deleted file mode 100755 index e3cd2a1f..00000000 --- a/.pc/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995/tools/hook-hotplug +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# This file is part of cloud-init. See LICENSE file for license information. - -# This script checks if cloud-init has hotplug hooked and if -# cloud-init is ready; if so invoke cloud-init hotplug-hook - -fifo=/run/cloud-init/hook-hotplug-cmd - -should_run() { - if [ -d /run/systemd ]; then - # check that the socket is ready - [ -p $fifo ] - else - # on non-systemd, check cloud-init fully finished. - [ -e /run/cloud-init/result.json ] - fi -} - -if ! should_run; then - exit 0 -fi - -# open cloud-init's hotplug-hook fifo rw -exec 3<>$fifo -env_params=" --subsystem=${SUBSYSTEM} handle --devpath=${DEVPATH} --udevaction=${ACTION}" -# write params to cloud-init's hotplug-hook fifo -echo "${env_params}" >&3 diff --git a/.pc/deprecation-version-boundary.patch/cloudinit/features.py b/.pc/deprecation-version-boundary.patch/cloudinit/features.py index 4f9a59e9..2cc0791e 100644 --- a/.pc/deprecation-version-boundary.patch/cloudinit/features.py +++ b/.pc/deprecation-version-boundary.patch/cloudinit/features.py @@ -87,6 +87,17 @@ to write /etc/apt/sources.list directly. """ +MANUAL_NETWORK_WAIT = True +""" +On Ubuntu systems, cloud-init-network.service will start immediately after +cloud-init-local.service and manually wait for network online when necessary. +If False, rely on systemd ordering to ensure network is available before +starting cloud-init-network.service. + +Note that in addition to this flag, downstream patches are also likely needed +to modify the systemd unit files. +""" + DEPRECATION_INFO_BOUNDARY = "devel" """ DEPRECATION_INFO_BOUNDARY is used by distros to configure at which upstream diff --git a/.pc/grub-dpkg-support.patch/cloudinit/config/schemas/schema-cloud-config-v1.json b/.pc/grub-dpkg-support.patch/cloudinit/config/schemas/schema-cloud-config-v1.json index 7d5c87c6..3edee9c8 100644 --- a/.pc/grub-dpkg-support.patch/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/.pc/grub-dpkg-support.patch/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -444,7 +444,7 @@ "ssh_redirect_user": { "type": "boolean", "default": false, - "description": "Boolean set to true to disable SSH logins for this user. When specified, all cloud meta-data public SSH keys will be set up in a disabled state for this username. Any SSH login as this username will timeout and prompt with a message to login instead as the **default_username** for this instance. Default: ``false``. This key can not be combined with **ssh_import_id** or **ssh_authorized_keys**." + "description": "Boolean set to true to disable SSH logins for this user. When specified, all cloud-provided public SSH keys will be set up in a disabled state for this username. Any SSH login as this username will timeout and prompt with a message to login instead as the **default_username** for this instance. Default: ``false``. This key can not be combined with **ssh_import_id** or **ssh_authorized_keys**." }, "system": { "description": "Optional. Create user as system user with no home directory. Default: ``false``.", @@ -662,7 +662,7 @@ "type": "object", "deprecated": true, "deprecated_version": "24.2", - "deprecated_description": "System and/or distro specific settings. This is not intended to be overridden by user data or vendor data." + "deprecated_description": "System and/or distro specific settings. This is not intended to be overridden by user-data or vendor-data." } } }, @@ -670,7 +670,7 @@ "type": "object", "properties": { "autoinstall": { - "description": "Opaque autoinstall schema definition for Ubuntu autoinstall. Full schema processed by live-installer. See: https://ubuntu.com/server/docs/install/autoinstall-reference.", + "description": "Cloud-init ignores this key and its values. It is used by Subiquity, the Ubuntu Autoinstaller. See: https://ubuntu.com/server/docs/install/autoinstall-reference.", "type": "object", "properties": { "version": { @@ -1219,7 +1219,12 @@ }, "minItems": 1, "uniqueItems": true, - "description": "Create the necessary directories for chef to run. By default, it creates the following directories:\n- ``/etc/chef``\n- ``/var/log/chef``\n- ``/var/lib/chef``\n- ``/var/cache/chef``\n- ``/var/backups/chef``\n- ``/var/run/chef``" + "description": "Create the necessary directories for chef to run. By default, it creates the following directories:\n- ``/etc/chef``\n- ``/var/log/chef``\n- ``/var/lib/chef``\n- ``/var/chef/backup``\n- ``/var/chef/cache``\n- ``/var/run/chef``" + }, + "config_path": { + "type": "string", + "default": "/etc/chef/client.rb", + "description": "Optional path for Chef configuration file. Default: ``/etc/chef/client.rb``" }, "validation_cert": { "type": "string", @@ -1257,13 +1262,13 @@ }, "file_backup_path": { "type": "string", - "default": "/var/backups/chef", - "description": "Specifies the location in which backup files are stored. By default, it uses the ``/var/backups/chef`` location." + "default": "/var/chef/backup", + "description": "Specifies the location in which backup files are stored. By default, it uses the ``/var/chef/backup`` location." }, "file_cache_path": { "type": "string", - "default": "/var/cache/chef", - "description": "Specifies the location in which chef cache files will be saved. By default, it uses the ``/var/cache/chef`` location." + "default": "/var/chef/cache", + "description": "Specifies the location in which chef cache files will be saved. By default, it uses the ``/var/chef/cache`` location." }, "json_attribs": { "type": "string", @@ -2759,7 +2764,7 @@ "additionalProperties": false, "properties": { "enabled": { - "description": "Whether vendor data is enabled or not. Default: ``true``.", + "description": "Whether vendor-data is enabled or not. Default: ``true``.", "oneOf": [ { "type": "boolean", diff --git a/.pc/no-nocloud-network.patch/cloudinit/sources/DataSourceNoCloud.py b/.pc/no-nocloud-network.patch/cloudinit/sources/DataSourceNoCloud.py index 18bf8902..a375106c 100644 --- a/.pc/no-nocloud-network.patch/cloudinit/sources/DataSourceNoCloud.py +++ b/.pc/no-nocloud-network.patch/cloudinit/sources/DataSourceNoCloud.py @@ -167,7 +167,7 @@ def _pp2d_callback(mp, data): # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. - if len(found) == 0: + if not found: return False # The special argument "seedfrom" indicates we should diff --git a/.pc/no-nocloud-network.patch/cloudinit/util.py b/.pc/no-nocloud-network.patch/cloudinit/util.py index 261e138e..bfcc9c8e 100644 --- a/.pc/no-nocloud-network.patch/cloudinit/util.py +++ b/.pc/no-nocloud-network.patch/cloudinit/util.py @@ -34,7 +34,7 @@ import sys import time from base64 import b64decode -from collections import deque, namedtuple +from collections import deque from contextlib import contextmanager, suppress from errno import ENOENT from functools import lru_cache @@ -50,6 +50,7 @@ Generator, List, Mapping, + NamedTuple, Optional, Sequence, Union, @@ -124,7 +125,7 @@ def lsb_release(): if fname in fmap: data[fmap[fname]] = val.strip() missing = [k for k in fmap.values() if k not in data] - if len(missing): + if missing: LOG.warning( "Missing fields in lsb_release --all output: %s", ",".join(missing), @@ -1188,10 +1189,10 @@ def dos2unix(contents): return contents.replace("\r\n", "\n") -HostnameFqdnInfo = namedtuple( - "HostnameFqdnInfo", - ["hostname", "fqdn", "is_default"], -) +class HostnameFqdnInfo(NamedTuple): + hostname: str + fqdn: str + is_default: bool def get_hostname_fqdn(cfg, cloud, metadata_only=False): @@ -1199,7 +1200,7 @@ def get_hostname_fqdn(cfg, cloud, metadata_only=False): @param cfg: Dictionary of merged user-data configuration (from init.cfg). @param cloud: Cloud instance from init.cloudify(). - @param metadata_only: Boolean, set True to only query cloud meta-data, + @param metadata_only: Boolean, set True to only query meta-data, returning None if not present in meta-data. @return: a namedtuple of , , (str, str, bool). @@ -1214,7 +1215,7 @@ def get_hostname_fqdn(cfg, cloud, metadata_only=False): is_default = False if "fqdn" in cfg: # user specified a fqdn. Default hostname then is based off that - fqdn = cfg["fqdn"] + fqdn = str(cfg["fqdn"]) hostname = get_cfg_option_str(cfg, "hostname", fqdn.split(".")[0]) else: if "hostname" in cfg and cfg["hostname"].find(".") > 0: @@ -1626,11 +1627,11 @@ def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None): data = in_fh.read(chunk_size) if len(data) == 0: break - else: - out_fh.write(data) - bytes_piped += len(data) - if chunk_cb: - chunk_cb(bytes_piped) + out_fh.write(data) + bytes_piped += len(data) + if chunk_cb: + chunk_cb(bytes_piped) + out_fh.flush() return bytes_piped @@ -2990,7 +2991,7 @@ def wait_for_files(flist, maxwait, naplen=0.5, log_pre=""): waited = 0 while True: need -= set([f for f in need if os.path.exists(f)]) - if len(need) == 0: + if not need: LOG.debug( "%sAll files appeared after %s seconds: %s", log_pre, diff --git a/.pc/no-nocloud-network.patch/tests/unittests/test_util.py b/.pc/no-nocloud-network.patch/tests/unittests/test_util.py index 8a107191..7d2383f2 100644 --- a/.pc/no-nocloud-network.patch/tests/unittests/test_util.py +++ b/.pc/no-nocloud-network.patch/tests/unittests/test_util.py @@ -799,6 +799,22 @@ def test_get_hostname_fqdn_from_without_fqdn_or_hostname(self): mock.call(metadata_only=False), ] == cloud.get_hostname.call_args_list + def test_get_hostname_fqdn_from_numeric_fqdn(self): + """When cfg fqdn is numeric, ensure it is treated as a string.""" + hostname, fqdn, _ = util.get_hostname_fqdn( + cfg={"fqdn": 12345}, cloud=None + ) + self.assertEqual("12345", hostname) + self.assertEqual("12345", fqdn) + + def test_get_hostname_fqdn_from_numeric_fqdn_with_domain(self): + """When cfg fqdn is numeric with a domain, ensure correct parsing.""" + hostname, fqdn, _ = util.get_hostname_fqdn( + cfg={"fqdn": "12345.example.com"}, cloud=None + ) + self.assertEqual("12345", hostname) + self.assertEqual("12345.example.com", fqdn) + def test_get_hostname_fqdn_from_passes_metadata_only_to_cloud(self): """Calls to cloud.get_hostname pass the metadata_only parameter.""" cloud = mock.MagicMock() diff --git a/.pc/no-single-process.patch/cloudinit/config/cc_mounts.py b/.pc/no-single-process.patch/cloudinit/config/cc_mounts.py index cec09290..bf9d8032 100644 --- a/.pc/no-single-process.patch/cloudinit/config/cc_mounts.py +++ b/.pc/no-single-process.patch/cloudinit/config/cc_mounts.py @@ -31,10 +31,8 @@ # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 DEVICE_NAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" -DEVICE_NAME_RE = re.compile(DEVICE_NAME_FILTER) # Name matches 'server:/path' NETWORK_NAME_FILTER = r"^.+:.*" -NETWORK_NAME_RE = re.compile(NETWORK_NAME_FILTER) FSTAB_PATH = "/etc/fstab" MNT_COMMENT = "comment=cloudconfig" MB = 2**20 @@ -57,7 +55,7 @@ def is_meta_device_name(name): def is_network_device(name): # return true if this is a network device - if NETWORK_NAME_RE.match(name): + if re.match(NETWORK_NAME_FILTER, name): return True return False @@ -114,7 +112,7 @@ def sanitize_devname(startname, transformer, aliases=None): device_path = "/dev/%s" % (device_path,) LOG.debug("Mapped metadata name %s to %s", orig, device_path) else: - if DEVICE_NAME_RE.match(startname): + if re.match(DEVICE_NAME_FILTER, startname): device_path = "/dev/%s" % (device_path,) partition_path = None @@ -550,7 +548,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: if swapfile: updated_cfg.append([swapfile, "none", "swap", "sw", "0", "0"]) - if len(updated_cfg) == 0: + if not updated_cfg: # This will only be true if there is no mount configuration at all # Even if fstab has no functional changes, we'll get past this point # as we remove any 'comment=cloudconfig' lines and then add them back diff --git a/.pc/no-single-process.patch/cloudinit/config/schemas/schema-cloud-config-v1.json b/.pc/no-single-process.patch/cloudinit/config/schemas/schema-cloud-config-v1.json index fffa04b5..c09e8fdd 100644 --- a/.pc/no-single-process.patch/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/.pc/no-single-process.patch/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -444,7 +444,7 @@ "ssh_redirect_user": { "type": "boolean", "default": false, - "description": "Boolean set to true to disable SSH logins for this user. When specified, all cloud meta-data public SSH keys will be set up in a disabled state for this username. Any SSH login as this username will timeout and prompt with a message to login instead as the **default_username** for this instance. Default: ``false``. This key can not be combined with **ssh_import_id** or **ssh_authorized_keys**." + "description": "Boolean set to true to disable SSH logins for this user. When specified, all cloud-provided public SSH keys will be set up in a disabled state for this username. Any SSH login as this username will timeout and prompt with a message to login instead as the **default_username** for this instance. Default: ``false``. This key can not be combined with **ssh_import_id** or **ssh_authorized_keys**." }, "system": { "description": "Optional. Create user as system user with no home directory. Default: ``false``.", @@ -662,7 +662,7 @@ "type": "object", "deprecated": true, "deprecated_version": "24.2", - "deprecated_description": "System and/or distro specific settings. This is not intended to be overridden by user data or vendor data." + "deprecated_description": "System and/or distro specific settings. This is not intended to be overridden by user-data or vendor-data." } } }, @@ -670,7 +670,7 @@ "type": "object", "properties": { "autoinstall": { - "description": "Opaque autoinstall schema definition for Ubuntu autoinstall. Full schema processed by live-installer. See: https://ubuntu.com/server/docs/install/autoinstall-reference.", + "description": "Cloud-init ignores this key and its values. It is used by Subiquity, the Ubuntu Autoinstaller. See: https://ubuntu.com/server/docs/install/autoinstall-reference.", "type": "object", "properties": { "version": { @@ -1219,7 +1219,12 @@ }, "minItems": 1, "uniqueItems": true, - "description": "Create the necessary directories for chef to run. By default, it creates the following directories:\n- ``/etc/chef``\n- ``/var/log/chef``\n- ``/var/lib/chef``\n- ``/var/cache/chef``\n- ``/var/backups/chef``\n- ``/var/run/chef``" + "description": "Create the necessary directories for chef to run. By default, it creates the following directories:\n- ``/etc/chef``\n- ``/var/log/chef``\n- ``/var/lib/chef``\n- ``/var/chef/backup``\n- ``/var/chef/cache``\n- ``/var/run/chef``" + }, + "config_path": { + "type": "string", + "default": "/etc/chef/client.rb", + "description": "Optional path for Chef configuration file. Default: ``/etc/chef/client.rb``" }, "validation_cert": { "type": "string", @@ -1257,13 +1262,13 @@ }, "file_backup_path": { "type": "string", - "default": "/var/backups/chef", - "description": "Specifies the location in which backup files are stored. By default, it uses the ``/var/backups/chef`` location." + "default": "/var/chef/backup", + "description": "Specifies the location in which backup files are stored. By default, it uses the ``/var/chef/backup`` location." }, "file_cache_path": { "type": "string", - "default": "/var/cache/chef", - "description": "Specifies the location in which chef cache files will be saved. By default, it uses the ``/var/cache/chef`` location." + "default": "/var/chef/cache", + "description": "Specifies the location in which chef cache files will be saved. By default, it uses the ``/var/chef/cache`` location." }, "json_attribs": { "type": "string", @@ -2759,7 +2764,7 @@ "additionalProperties": false, "properties": { "enabled": { - "description": "Whether vendor data is enabled or not. Default: ``true``.", + "description": "Whether vendor-data is enabled or not. Default: ``true``.", "oneOf": [ { "type": "boolean", diff --git a/.pc/no-single-process.patch/systemd/cloud-config.service b/.pc/no-single-process.patch/systemd/cloud-config.service index 54599b34..68f80d2b 100644 --- a/.pc/no-single-process.patch/systemd/cloud-config.service +++ b/.pc/no-single-process.patch/systemd/cloud-config.service @@ -16,7 +16,7 @@ Type=oneshot # process has completed this stage. The output from the return socket is piped # into a shell so that the process can send a completion message (defaults to # "done", otherwise includes an error message) and an exit code to systemd. -ExecStart=sh -c 'echo "start" | netcat -Uu -W1 /run/cloud-init/share/config.sock -s /run/cloud-init/share/config-return.sock | sh' +ExecStart=sh -c 'echo "start" | nc -Uu -W1 /run/cloud-init/share/config.sock -s /run/cloud-init/share/config-return.sock | sh' RemainAfterExit=yes TimeoutSec=0 diff --git a/.pc/no-single-process.patch/systemd/cloud-final.service b/.pc/no-single-process.patch/systemd/cloud-final.service index c48f95c4..fb74a47c 100644 --- a/.pc/no-single-process.patch/systemd/cloud-final.service +++ b/.pc/no-single-process.patch/systemd/cloud-final.service @@ -19,7 +19,7 @@ Type=oneshot # process has completed this stage. The output from the return socket is piped # into a shell so that the process can send a completion message (defaults to # "done", otherwise includes an error message) and an exit code to systemd. -ExecStart=sh -c 'echo "start" | netcat -Uu -W1 /run/cloud-init/share/final.sock -s /run/cloud-init/share/final-return.sock | sh' +ExecStart=sh -c 'echo "start" | nc -Uu -W1 /run/cloud-init/share/final.sock -s /run/cloud-init/share/final-return.sock | sh' RemainAfterExit=yes TimeoutSec=0 TasksMax=infinity diff --git a/.pc/no-single-process.patch/systemd/cloud-init-local.service.tmpl b/.pc/no-single-process.patch/systemd/cloud-init-local.service.tmpl index e6a300fd..b123193a 100644 --- a/.pc/no-single-process.patch/systemd/cloud-init-local.service.tmpl +++ b/.pc/no-single-process.patch/systemd/cloud-init-local.service.tmpl @@ -32,7 +32,7 @@ ExecStartPre=/sbin/restorecon /run/cloud-init # process has completed this stage. The output from the return socket is piped # into a shell so that the process can send a completion message (defaults to # "done", otherwise includes an error message) and an exit code to systemd. -ExecStart=sh -c 'echo "start" | netcat -Uu -W1 /run/cloud-init/share/local.sock -s /run/cloud-init/share/local-return.sock | sh' +ExecStart=sh -c 'echo "start" | nc -Uu -W1 /run/cloud-init/share/local.sock -s /run/cloud-init/share/local-return.sock | sh' RemainAfterExit=yes TimeoutSec=0 diff --git a/.pc/no-single-process.patch/systemd/cloud-init-network.service.tmpl b/.pc/no-single-process.patch/systemd/cloud-init-network.service.tmpl index af09fff3..bdc7c8f8 100644 --- a/.pc/no-single-process.patch/systemd/cloud-init-network.service.tmpl +++ b/.pc/no-single-process.patch/systemd/cloud-init-network.service.tmpl @@ -53,7 +53,7 @@ Type=oneshot # process has completed this stage. The output from the return socket is piped # into a shell so that the process can send a completion message (defaults to # "done", otherwise includes an error message) and an exit code to systemd. -ExecStart=sh -c 'echo "start" | netcat -Uu -W1 /run/cloud-init/share/network.sock -s /run/cloud-init/share/network-return.sock | sh' +ExecStart=sh -c 'echo "start" | nc -Uu -W1 /run/cloud-init/share/network.sock -s /run/cloud-init/share/network-return.sock | sh' RemainAfterExit=yes TimeoutSec=0 diff --git a/ChangeLog b/ChangeLog index 0d6c71c6..699d2345 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,116 @@ +25.1.2 + - fix: ensure MAAS datasource retries on failure (#6167) + +25.1.1 + - test: pytestify cc_chef tests, add migration test + - chef: migrate files in old config directories for backups and cache + - fix: correct the path for Chef's backups (#5994) + - fix(Azure): don't reraise FileNotFoundError during ephemeral setup (#6113) + - fix(azure): handle unexpected exceptions during obtain_lease() (#6092) + [Ksenija Stanojevic] + - Allow to set mac_address for VLAN subinterface (#6081) + [jumpojoy] (GH: 5364) + - fix: Remove erroneous EC2 reference from 503 warning (#6077) + - fix: NM reload and bring up individual network conns (#6073) [Ani Sinha] + - fix: stop warning on dual-stack request failure (#6044) + - fix: install_method: pip cannot find ansible-pull command path (#6021) + [Hasan Aliyev] (GH: 5720) + - fix: Fix DataSourceAliYun exception_cb signature (#6068) (GH: 6066) + - fix: Update OauthUrlHelper to use readurl exception_cb signature + (GH: 6065) + - test: add OauthUrlHelper tests + - test: Remove CiTestCase from test_url_helper.py + - test: pytestify test_url_helper.py + - fix: track more removed modules (#6043) + +25.1 + - ci: fix post-merge packaging CI (#6038) + - feat(azure): Fix imds-based ssh_pwauth (#6002) [Ksenija Stanojevic] + - ci: check for sorted patches (#6036) + - feat: aliyun datasource support crawl metadata at once (#5942) + [jinkangkang] + - docs: document /usr merge breaking change (#6032) + - test: Add integration test for /var mounts (#6033) + - test: Ensure pre-24.2 custom modules work (#6034) + - doc: Update references to older keys (#6022) [Pedro Ribeiro] + - fix: untyped-defs in tests/unittests/{config, net, sources} (#6023) + [Romain] + - fix: don't reference PR in post-merged CI (#6019) + - chore: explicitly skip broken ansible integration tests (#5996) [a-dubs] + - tests(oracle): fix test_install_missing_deps apt race condition (#5996) + [a-dubs] + - test(oracle): fix test_ubuntu_drivers_installed (#5996) [a-dubs] + - test(oracle): fix test_frequency_override integration test (#5996) + [a-dubs] + - chore: add type hint to IntegrationCloud's cloud_instance field (#5996) + [a-dubs] + - test(oracle): fix modules/test_lxd.py::test_storage_lvm on noble (#5996) + [a-dubs] + - commit 9e591fff266be9d4c83f74ec02a717b74993304d [a-dubs] + - net/sysconfig: do not remove all existing settings of + /etc/sysconfig/network (#5991) [Ani Sinha] (GH: 5990) + - fix: remove wrong return when checking if network necessary (#6013) + - fix: typing for rsyslog, ubuntu_pro, power_state_change (#5985) + [MostafaTarek124eru] + - fix: Retry on OpenStack HTTP status codes (#5943) [weiyang] (GH: 5687) + - fix: Ensure fqdn is treated as string in get_hostname_fqdn (#5993) + [MKhatibzadeh] (GH: 5989) + - feat(vmware): Convert imc network config to v2 (#5937) [PengpengSun] + - ci: add upstream post-merge test + - ci: check if upstream commit causes ubuntu patch conflicts + - ci: organize cla tests together + - test: eliminate obsolete cases, add non-error case + - chore: remove redundant manual schema validation + - doc: clarify subiquity docs + - chore: cleanup `len' usage (#5956) [Shreenidhi Shedi] + - Fix: GCE _get_data crashes if DHCP lease fails (#5998) [Bryan Fraschetti] + - Fixes GH-5997 + - fix: correct the path for Chef's cache (#5994) + [MostafaTarek124eru] (GH: 5090) + - fix: Run ansible with run_user instead of root for distro install_method + (#5986) [Amirhossein Shaerpour] (GH: 4092) + - fix: retry AWS hotplug for async IMDS (#5995) (GH: 5373) + - feat(integration_tests): add optional INSTANCE_TYPE setting (#5988) + [Alec Warren] + - feat(integration-tests): set boto3 and botocore to INFO to prevent + log spamming [a-dubs] + - ci: add 'tox -e integration-tests-fast' command [a-dubs] + - chore: Add feature flag for manual network waiting (#5977) + - Release 24.4.1 + - fix: Use /usr/lib/ rather than /lib in packaging code (#5970) + - Use log_with_downgradable_level for user password warnings (#5927) + [Ani Sinha] + - doc: change to hyphenated keys (#5909) (GH: 5555) + - fix: Wait for udev on openstack (#5947) [Robert Schweikert] (GH: 4125) + - test: disambiguate resource cleanup from test failure (#5926) + - fix: use program name of netcat as installed by upstream, "nc" (#5933) + (#5933) [Andreas K. Hüttel] + - ci: bump canonical/setup-lxd to version v0.1.2 (#5948) + - feat(cc_chef): Allow change of Chef configuration file (#5925) + [Sean Smith] + - docs: fix typo in generated file in LXD tutorial (#5941) [Pavel Shpak] + - feat: Identify Samsung Cloud Platform as OpenStack (#5924) [us0310306] + - fix: don't deadlock when starting network service with systemctl (#5935) + - feat: Custom keys for apt archives (#5828) [Bryan Fraschetti] (GH: 5473) + - test: improve test initialization error path (#5920) + - chore: improve logging when lxd detection fails (#5919) + - fix: Add "manual" to allowed subnet types (#5875) + [Math Marchand] (GH: 5769) + - fix: remove bad ssh_svcname setting for Gentoo/OpenRC (#5918) + [Andreas K. Hüttel] + - feat(gentoo): Add compatibility for Gentoo with systemd (#5918) + [Andreas K. Hüttel] + - fix(ovf): no warning should be log when rpctool found no value (#5915) + [PengpengSun] (GH: 5914) + - Move DS VMware to be in front of DS OVF (#5912) [PengpengSun] (GH: 4030) + - ci: Add proper 'Breaks: ' to integration testing simple deb (#5923) + - chore: Add akhuettel to CLA signers file (#5917) [Andreas K. Hüttel] + - chore: eliminate calls at import time (#5889) (GH: 5344) + - test: Add pyserial to test-requirements.txt (#5907) + - test: Allow unknown size in growpart test (#5876) + - doc: Update tutorials [Sally] + - fix: bump azure key size to 3072 (#5841) + 24.4.1 - fix: Ensure _should_wait_via_user_data() handles all user data types (#5976) - fix: Don't log error in wait_for_url (#5972) diff --git a/cloudinit/apport.py b/cloudinit/apport.py index 3bc1055c..870281e4 100644 --- a/cloudinit/apport.py +++ b/cloudinit/apport.py @@ -56,6 +56,7 @@ "OVF", "RbxCloud - (HyperOne, Rootbox, Rubikon)", "OpenTelekomCloud", + "Samsung Cloud Platform", "SAP Converged Cloud", "Scaleway", "SmartOS", diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 17367aa7..207b97cf 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -362,7 +362,6 @@ def _should_wait_via_user_data( source_uri = source_dict.get("uri", "") if source_uri and not (source_uri.startswith(("/", "file:"))): return True, "write_files with source uri found" - return False, "write_files without source uri found" if parsed_yaml.get("bootcmd"): return True, "bootcmd found" if parsed_yaml.get("random_seed", {}).get("command"): diff --git a/cloudinit/config/cc_ansible.py b/cloudinit/config/cc_ansible.py index c9d1ead2..870c8823 100644 --- a/cloudinit/config/cc_ansible.py +++ b/cloudinit/config/cc_ansible.py @@ -79,34 +79,45 @@ class AnsiblePullPip(AnsiblePull): def __init__(self, distro: Distro, user: Optional[str]): super().__init__(distro) self.run_user = user + self.add_pip_install_site_to_path() + + def add_pip_install_site_to_path(self): + if self.run_user: + user_base, _ = self.do_as( + [ + sys.executable, + "-c", + "import site; print(site.getuserbase())", + ] + ) + ansible_path = f"{user_base}/bin/" - # Add pip install site to PATH - user_base, _ = self.do_as( - [sys.executable, "-c", "'import site; print(site.getuserbase())'"] - ) - ansible_path = f"{user_base}/bin/" - old_path = self.env.get("PATH") - if old_path: - self.env["PATH"] = ":".join([old_path, ansible_path]) - else: - self.env["PATH"] = ansible_path + old_path = self.env.get("PATH") + if old_path: + self.env["PATH"] = ":".join([old_path, ansible_path]) + else: + self.env["PATH"] = ansible_path + + def bootstrap_pip_if_required(self): + try: + import pip # noqa: F401 + except ImportError: + self.distro.install_packages([self.distro.pip_package_name]) def install(self, pkg_name: str): """should cloud-init grow an interface for non-distro package managers? this seems reusable """ + self.bootstrap_pip_if_required() + if not self.is_installed(): - # bootstrap pip if required - try: - import pip # noqa: F401 - except ImportError: - self.distro.install_packages([self.distro.pip_package_name]) cmd = [ sys.executable, "-m", "pip", "install", ] + if os.path.exists( os.path.join( sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED" @@ -115,15 +126,25 @@ def install(self, pkg_name: str): cmd.append("--break-system-packages") if self.run_user: cmd.append("--user") + self.do_as([*cmd, "--upgrade", "pip"]) self.do_as([*cmd, pkg_name]) def is_installed(self) -> bool: - stdout, _ = self.do_as([sys.executable, "-m", "pip", "list"]) + cmd = [sys.executable, "-m", "pip", "list"] + + if self.run_user: + cmd.append("--user") + + stdout, _ = self.do_as(cmd) return "ansible" in stdout class AnsiblePullDistro(AnsiblePull): + def __init__(self, distro: Distro, user: Optional[str]): + super().__init__(distro) + self.run_user = user + def install(self, pkg_name: str): if not self.is_installed(): self.distro.install_packages([pkg_name]) @@ -151,7 +172,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: if install_method == "pip": ansible = AnsiblePullPip(distro, ansible_user) else: - ansible = AnsiblePullDistro(distro) + ansible = AnsiblePullDistro(distro, ansible_user) ansible.install(package_name) ansible.check_deps() ansible_config = ansible_cfg.get("ansible_config", "") diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 35445711..853e5296 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -147,8 +147,8 @@ def apply_apt(cfg, cloud, gpg): _ensure_dependencies(cfg, matcher, cloud) if util.is_false(cfg.get("preserve_sources_list", False)): - add_mirror_keys(cfg, cloud, gpg) - generate_sources_list(cfg, release, mirrors, cloud) + keys = add_mirror_keys(cfg, cloud, gpg) + generate_sources_list(cfg, release, mirrors, cloud, keys) rename_apt_lists(mirrors, arch) try: @@ -241,7 +241,7 @@ def apply_debconf_selections(cfg): LOG.debug("pkgs_cfgd: %s", pkgs_cfgd) need_reconfig = pkgs_cfgd.intersection(pkgs_installed) - if len(need_reconfig) == 0: + if not need_reconfig: LOG.debug("no need for reconfig") return @@ -421,11 +421,15 @@ def disable_suites(disabled, src, release) -> str: return retsrc -def add_mirror_keys(cfg, cloud, gpg): +def add_mirror_keys(cfg, cloud, gpg) -> Mapping[str, str]: """Adds any keys included in the primary/security mirror clauses""" + keys = {} for key in ("primary", "security"): for mirror in cfg.get(key, []): - add_apt_key(mirror, cloud, gpg, file_name=key) + resp = add_apt_key(mirror, cloud, gpg, file_name=key) + if resp: + keys[f"{key}_key"] = resp + return keys def is_deb822_sources_format(apt_src_content: str) -> bool: @@ -515,7 +519,7 @@ def get_apt_cfg() -> Dict[str, str]: } -def generate_sources_list(cfg, release, mirrors, cloud): +def generate_sources_list(cfg, release, mirrors, cloud, keys): """generate_sources_list create a source.list file based on a custom or default template by replacing mirrors and release in the template""" @@ -528,6 +532,7 @@ def generate_sources_list(cfg, release, mirrors, cloud): aptsrc_file = apt_sources_list params = {"RELEASE": release, "codename": release} + params.update(keys) for k in mirrors: params[k] = mirrors[k] params[k.lower()] = mirrors[k] diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 8ecb51f4..86cc4bda 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -28,7 +28,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: - if len(args) != 0: + if args: value = args[0] else: value = util.get_cfg_option_str(cfg, "byobu_by_default", "") diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 7e06ee06..4b350724 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -12,6 +12,7 @@ import json import logging import os +import shutil from typing import List from cloudinit import subp, temp_utils, templater, url_helper, util @@ -23,21 +24,20 @@ RUBY_VERSION_DEFAULT = "1.8" -CHEF_DIRS = tuple( - [ - "/etc/chef", - "/var/log/chef", - "/var/lib/chef", - "/var/cache/chef", - "/var/backups/chef", - "/var/run/chef", - ] -) -REQUIRED_CHEF_DIRS = tuple( - [ - "/etc/chef", - ] +CHEF_DIRS = ( + "/etc/chef", + "/var/log/chef", + "/var/lib/chef", + "/var/chef/cache", + "/var/chef/backup", + "/var/run/chef", ) +REQUIRED_CHEF_DIRS = ("/etc/chef",) + +CHEF_DIR_MIGRATION = { + "/var/cache/chef": "/var/chef/cache", + "/var/backups/chef": "/var/chef/backup", +} # Used if fetching chef from a omnibus style package OMNIBUS_URL = "https://www.chef.io/chef/install.sh" @@ -55,8 +55,8 @@ "validation_cert": None, "client_key": "/etc/chef/client.pem", "json_attribs": CHEF_FB_PATH, - "file_cache_path": "/var/cache/chef", - "file_backup_path": "/var/backups/chef", + "file_cache_path": "/var/chef/cache", + "file_backup_path": "/var/chef/backup", "pid_file": "/var/run/chef/client.pid", "show_time": True, "encrypted_data_bag_secret": None, @@ -74,22 +74,20 @@ ] ) CHEF_RB_TPL_KEYS = frozenset( - itertools.chain( - CHEF_RB_TPL_DEFAULTS.keys(), - CHEF_RB_TPL_BOOL_KEYS, - CHEF_RB_TPL_PATH_KEYS, - [ - "server_url", - "node_name", - "environment", - "validation_name", - "chef_license", - ], - ) + [ + *CHEF_RB_TPL_DEFAULTS.keys(), + *CHEF_RB_TPL_BOOL_KEYS, + *CHEF_RB_TPL_PATH_KEYS, + "server_url", + "node_name", + "environment", + "validation_name", + "chef_license", + ] ) CHEF_RB_PATH = "/etc/chef/client.rb" CHEF_EXEC_PATH = "/usr/bin/chef-client" -CHEF_EXEC_DEF_ARGS = tuple(["-d", "-i", "1800", "-s", "20"]) +CHEF_EXEC_DEF_ARGS = ("-d", "-i", "1800", "-s", "20") LOG = logging.getLogger(__name__) @@ -145,6 +143,26 @@ def get_template_params(iid, chef_cfg): return params +def migrate_chef_config_dirs(): + """Migrate legacy chef backup and cache directories to new config paths.""" + for old_dir, migrated_dir in CHEF_DIR_MIGRATION.items(): + if os.path.exists(old_dir): + for filename in os.listdir(old_dir): + if os.path.exists(os.path.join(migrated_dir, filename)): + LOG.debug( + "Ignoring migration of %s. File already exists in %s.", + os.path.join(old_dir, filename), + migrated_dir, + ) + continue + LOG.debug( + "Moving %s to %s.", + os.path.join(old_dir, filename), + migrated_dir, + ) + shutil.move(os.path.join(old_dir, filename), migrated_dir) + + def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: """Handler method activated by cloud-init.""" @@ -164,6 +182,9 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: for d in itertools.chain(chef_dirs, REQUIRED_CHEF_DIRS): util.ensure_dir(d) + # Migrate old directory cache and backups to new + migrate_chef_config_dirs() + vkey_path = chef_cfg.get("validation_key", CHEF_VALIDATION_PEM_PATH) vcert = chef_cfg.get("validation_cert") # special value 'system' means do not overwrite the file @@ -179,6 +200,9 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: ) # Create the chef config from template + cfg_filename = util.get_cfg_option_str( + chef_cfg, "config_path", default=CHEF_RB_PATH + ) template_fn = cloud.get_template_filename("chef_client.rb") if template_fn: iid = str(cloud.datasource.get_instance_id()) @@ -191,9 +215,9 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: if k in CHEF_RB_TPL_PATH_KEYS and v: param_paths.add(os.path.dirname(v)) util.ensure_dirs(param_paths) - templater.render_to_file(template_fn, CHEF_RB_PATH, params) + templater.render_to_file(template_fn, cfg_filename, params) else: - LOG.warning("No template found, not rendering to %s", CHEF_RB_PATH) + LOG.warning("No template found, not rendering to %s", cfg_filename) # Set the firstboot json fb_filename = util.get_cfg_option_str( diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 0864140c..0f3bf12d 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -377,7 +377,7 @@ def check_partition_mbr_layout(device, layout): found_layout = [] for line in out.splitlines(): _line = line.split() - if len(_line) == 0: + if not _line: continue if device in _line[0]: @@ -500,7 +500,7 @@ def get_partition_mbr_layout(size, layout): # Create a single partition, default to Linux return ",,83" - if (len(layout) == 0 and isinstance(layout, list)) or not isinstance( + if ((not layout) and isinstance(layout, list)) or not isinstance( layout, list ): raise RuntimeError("Partition layout is invalid") diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index e5e191ec..8611e540 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -38,7 +38,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: msg_in = "" - if len(args) != 0: + if args: msg_in = str(args[0]) else: msg_in = util.get_cfg_option_str(cfg, "final_message", "") diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py index 0a05859e..53d04ef5 100644 --- a/cloudinit/config/cc_locale.py +++ b/cloudinit/config/cc_locale.py @@ -27,7 +27,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: - if len(args) != 0: + if args: locale = args[0] else: locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index cb36bdb8..a5d10362 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -31,10 +31,8 @@ # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 DEVICE_NAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" -DEVICE_NAME_RE = re.compile(DEVICE_NAME_FILTER) # Name matches 'server:/path' NETWORK_NAME_FILTER = r"^.+:.*" -NETWORK_NAME_RE = re.compile(NETWORK_NAME_FILTER) FSTAB_PATH = "/etc/fstab" MNT_COMMENT = "comment=cloudconfig" MB = 2**20 @@ -57,7 +55,7 @@ def is_meta_device_name(name): def is_network_device(name): # return true if this is a network device - if NETWORK_NAME_RE.match(name): + if re.match(NETWORK_NAME_FILTER, name): return True return False @@ -114,7 +112,7 @@ def sanitize_devname(startname, transformer, aliases=None): device_path = "/dev/%s" % (device_path,) LOG.debug("Mapped metadata name %s to %s", orig, device_path) else: - if DEVICE_NAME_RE.match(startname): + if re.match(DEVICE_NAME_FILTER, startname): device_path = "/dev/%s" % (device_path,) partition_path = None @@ -550,7 +548,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: if swapfile: updated_cfg.append([swapfile, "none", "swap", "sw", "0", "0"]) - if len(updated_cfg) == 0: + if not updated_cfg: # This will only be true if there is no mount configuration at all # Even if fstab has no functional changes, we'll get past this point # as we remove any 'comment=cloudconfig' lines and then add them back diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index c9ac5e74..0501a89a 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -427,18 +427,14 @@ def write_ntp_config_template( if not peers: peers = [] - if len(servers) == 0 and len(pools) == 0 and distro_name == "cos": + if not servers and not pools and distro_name == "cos": return - if ( - len(servers) == 0 - and distro_name == "alpine" - and service_name == "ntpd" - ): + if not servers and distro_name == "alpine" and service_name == "ntpd": # Alpine's Busybox ntpd only understands "servers" configuration # and not "pool" configuration. servers = generate_server_names(distro_name) LOG.debug("Adding distro default ntp servers: %s", ",".join(servers)) - elif len(servers) == 0 and len(pools) == 0: + elif not (servers) and not (pools): pools = generate_server_names(distro_name) LOG.debug( "Adding distro default ntp pool servers: %s", ",".join(pools) diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 03b3f80b..9de45678 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -47,7 +47,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: - if len(args) != 0: + if args: ph_cfg = util.read_conf(args[0]) else: if "phone_home" not in cfg: diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index d0640d51..4bd9f8cc 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -45,7 +45,10 @@ def givecmdline(pid): (output, _err) = subp.subp(["procstat", "-c", str(pid)]) line = output.splitlines()[1] m = re.search(r"\d+ (\w|\.|-)+\s+(/\w.+)", line) - return m.group(2) + if m: + return m.group(2) + else: + return None else: return util.load_text_file("/proc/%s/cmdline" % pid) except IOError: diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index e32f337a..f7d74f19 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -249,7 +249,7 @@ def maybe_get_writable_device_path(devpath, info): def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: - if len(args) != 0: + if args: resize_root = args[0] else: resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index a52eab4d..893ec780 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -330,7 +330,7 @@ def addPool(self, pools): """ # An empty list was passed - if len(pools) == 0: + if not pools: self.log.debug("No pools to attach") return True @@ -379,7 +379,7 @@ def update_repos(self): return False # Bail if both lists are not populated - if (len(erepos) == 0) and (len(drepos) == 0): + if not (erepos) and not (drepos): self.log.debug("No repo IDs to enable or disable") return True diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index fddd970a..d7133bac 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -59,13 +59,6 @@ LOG = logging.getLogger(__name__) -COMMENT_RE = re.compile(r"[ ]*[#]+[ ]*") -HOST_PORT_RE = re.compile( - r"^(?P[@]{0,2})" - r"(([\[](?P[^\]]*)[\]])|(?P[^:]*))" - r"([:](?P[0-9]+))?$" -) - def distro_default_rsyslog_config(distro: Distro): """Construct a distro-specific rsyslog config dictionary by merging @@ -195,7 +188,7 @@ def apply_rsyslog_changes(configs, def_fname, cfg_dir): def parse_remotes_line(line, name=None): try: - data, comment = COMMENT_RE.split(line) + data, comment = re.split(r"[ ]*[#]+[ ]*", line) comment = comment.strip() except ValueError: data, comment = (line, None) @@ -209,7 +202,12 @@ def parse_remotes_line(line, name=None): else: raise ValueError("line had multiple spaces: %s" % data) - toks = HOST_PORT_RE.match(host_port) + toks = re.match( + r"^(?P[@]{0,2})" + r"(([\[](?P[^\]]*)[\]])|(?P[^:]*))" + r"([:](?P[0-9]+))?$", + host_port, + ) if not toks: raise ValueError("Invalid host specification '%s'" % host_port) @@ -248,10 +246,7 @@ def __init__( self.proto = proto self.addr = addr - if port: - self.port = int(port) - else: - self.port = None + self.port = int(port) if port is not None else None def validate(self): if self.port: diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 8cb6a1ec..22547d0f 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -45,9 +45,9 @@ def get_users_by_type(users_list: list, pw_type: str) -> list: ) -def _restart_ssh_daemon(distro, service): +def _restart_ssh_daemon(distro: Distro, service: str, *extra_args: str): try: - distro.manage_service("restart", service) + distro.manage_service("restart", service, *extra_args) LOG.debug("Restarted the SSH daemon.") except subp.ProcessExecutionError as e: LOG.warning( @@ -104,7 +104,21 @@ def handle_ssh_pwauth(pw_auth, distro: Distro): ] ).stdout.strip() if state.lower() in ["active", "activating", "reloading"]: - _restart_ssh_daemon(distro, service) + # This module runs Before=sshd.service. What that means is that + # the code can only get to this point if a user manually starts the + # network stage. While this isn't a well-supported use-case, this + # does cause a deadlock if started via systemd directly: + # "systemctl start cloud-init.service". Prevent users from causing + # this deadlock by forcing systemd to ignore dependencies when + # restarting. Note that this deadlock is not possible in newer + # versions of cloud-init, since starting the second service doesn't + # run the second stage in 24.3+. This code therefore exists solely + # for backwards compatibility so that users who think that they + # need to manually start cloud-init (why?) with systemd (again, + # why?) can do so. + _restart_ssh_daemon( + distro, service, "--job-mode=ignore-dependencies" + ) else: _restart_ssh_daemon(distro, service) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 084bc669..a7ecb1c0 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -38,9 +38,6 @@ GENERATE_KEY_NAMES = ["rsa", "ecdsa", "ed25519"] FIPS_UNSUPPORTED_KEY_NAMES = ["ed25519"] -pattern_unsupported_config_keys = re.compile( - "^(ecdsa-sk|ed25519-sk)_(private|public|certificate)$" -) KEY_FILE_TPL = "/etc/ssh/ssh_host_%s_key" PUBLISH_HOST_KEYS = True # By default publish all supported hostkey types. @@ -113,7 +110,9 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: cert_config = [] for key, val in cfg["ssh_keys"].items(): if key not in CONFIG_KEY_TO_FILE: - if pattern_unsupported_config_keys.match(key): + if re.match( + "^(ecdsa-sk|ed25519-sk)_(private|public|certificate)$", key + ): reason = "unsupported" else: reason = "unrecognized" diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index afed7e20..3e3bf056 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -47,7 +47,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: return # import for "user: XXXXX" - if len(args) != 0: + if args: user = args[0] ids = [] if len(args) > 1: diff --git a/cloudinit/config/cc_timezone.py b/cloudinit/config/cc_timezone.py index 957f7a58..c6162cf2 100644 --- a/cloudinit/config/cc_timezone.py +++ b/cloudinit/config/cc_timezone.py @@ -27,7 +27,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: - if len(args) != 0: + if args: timezone = args[0] else: timezone = util.get_cfg_option_str(cfg, "timezone", False) diff --git a/cloudinit/config/cc_ubuntu_autoinstall.py b/cloudinit/config/cc_ubuntu_autoinstall.py index a6763460..62df3670 100644 --- a/cloudinit/config/cc_ubuntu_autoinstall.py +++ b/cloudinit/config/cc_ubuntu_autoinstall.py @@ -8,11 +8,7 @@ from cloudinit import subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import ( - MetaSchema, - SchemaProblem, - SchemaValidationError, -) +from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_ONCE LOG = logging.getLogger(__name__) @@ -30,13 +26,6 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: - if "autoinstall" not in cfg: - LOG.debug( - "Skipping module named %s, no 'autoinstall' key in configuration", - name, - ) - return - util.wait_for_snap_seeded(cloud) snap_list, _ = subp.subp(["snap", "list"]) installer_present = None @@ -50,51 +39,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: ", ".join(LIVE_INSTALLER_SNAPS), ) return - validate_config_schema(cfg) LOG.debug( "Valid autoinstall schema. Config will be processed by %s", installer_present, ) - - -def validate_config_schema(cfg): - """Supplemental runtime schema validation for autoinstall yaml. - - Schema validation issues currently result in a warning log currently which - can be easily ignored because warnings do not bubble up to cloud-init - status output. - - In the case of the live-installer, we want cloud-init to raise an error - to set overall cloud-init status to 'error' so it is more discoverable - in installer environments. - - # TODO(Drop this validation When cloud-init schema is strict and errors) - - :raise: SchemaValidationError if any known schema values are present. - """ - autoinstall_cfg = cfg["autoinstall"] - if not isinstance(autoinstall_cfg, dict): - raise SchemaValidationError( - [ - SchemaProblem( - "autoinstall", - "Expected dict type but found:" - f" {type(autoinstall_cfg).__name__}", - ) - ] - ) - - if "version" not in autoinstall_cfg: - raise SchemaValidationError( - [SchemaProblem("autoinstall", "Missing required 'version' key")] - ) - elif not isinstance(autoinstall_cfg.get("version"), int): - raise SchemaValidationError( - [ - SchemaProblem( - "autoinstall.version", - f"Expected int type but found:" - f" {type(autoinstall_cfg['version']).__name__}", - ) - ] - ) diff --git a/cloudinit/config/cc_ubuntu_pro.py b/cloudinit/config/cc_ubuntu_pro.py index 81acf043..4ba48026 100644 --- a/cloudinit/config/cc_ubuntu_pro.py +++ b/cloudinit/config/cc_ubuntu_pro.py @@ -178,8 +178,8 @@ def configure_pro(token, enable=None): # Allow `ua attach` to fail in already attached machines subp.subp(attach_cmd, rcs={0, 2}, logstring=redacted_cmd) except subp.ProcessExecutionError as e: - err = str(e).replace(token, REDACTED) - msg = f"Failure attaching Ubuntu Pro:\n{err}" + er = str(e).replace(token, REDACTED) + msg = f"Failure attaching Ubuntu Pro:\n{er}" util.logexc(LOG, msg) raise RuntimeError(msg) from e diff --git a/cloudinit/config/modules.py b/cloudinit/config/modules.py index bea6a8e9..a4432b54 100644 --- a/cloudinit/config/modules.py +++ b/cloudinit/config/modules.py @@ -38,6 +38,8 @@ # from having to create upgrade scripts to avoid warnings about missing # modules. REMOVED_MODULES = [ + "cc_emit_upstart", # Removed in 22.2 + "cc_refresh_rmc_and_interface.py", # Removed in 23.2 "cc_migrator", # Removed in 24.1 "cc_rightscale_userdata", # Removed in 24.1 ] diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index ba063800..08e956b6 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -325,10 +325,6 @@ def _validator( yield error_type(msg, schema.get("deprecated_version", "devel")) -_validator_deprecated = partial(_validator, filter_key="deprecated") -_validator_changed = partial(_validator, filter_key="changed") - - def _anyOf( validator, anyOf, @@ -474,8 +470,8 @@ def get_jsonschema_validator(): # Add deprecation handling validators = dict(Draft4Validator.VALIDATORS) - validators[DEPRECATED_KEY] = _validator_deprecated - validators["changed"] = _validator_changed + validators[DEPRECATED_KEY] = partial(_validator, filter_key="deprecated") + validators["changed"] = partial(_validator, filter_key="changed") validators["oneOf"] = _oneOf validators["anyOf"] = _anyOf diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index f7a74fdd..c6458853 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -444,7 +444,7 @@ "ssh_redirect_user": { "type": "boolean", "default": false, - "description": "Boolean set to true to disable SSH logins for this user. When specified, all cloud meta-data public SSH keys will be set up in a disabled state for this username. Any SSH login as this username will timeout and prompt with a message to login instead as the **default_username** for this instance. Default: ``false``. This key can not be combined with **ssh_import_id** or **ssh_authorized_keys**." + "description": "Boolean set to true to disable SSH logins for this user. When specified, all cloud-provided public SSH keys will be set up in a disabled state for this username. Any SSH login as this username will timeout and prompt with a message to login instead as the **default_username** for this instance. Default: ``false``. This key can not be combined with **ssh_import_id** or **ssh_authorized_keys**." }, "system": { "description": "Optional. Create user as system user with no home directory. Default: ``false``.", @@ -662,7 +662,7 @@ "type": "object", "deprecated": true, "deprecated_version": "24.2", - "deprecated_description": "System and/or distro specific settings. This is not intended to be overridden by user data or vendor data." + "deprecated_description": "System and/or distro specific settings. This is not intended to be overridden by user-data or vendor-data." } } }, @@ -670,7 +670,7 @@ "type": "object", "properties": { "autoinstall": { - "description": "Opaque autoinstall schema definition for Ubuntu autoinstall. Full schema processed by live-installer. See: https://ubuntu.com/server/docs/install/autoinstall-reference.", + "description": "Cloud-init ignores this key and its values. It is used by Subiquity, the Ubuntu Autoinstaller. See: https://ubuntu.com/server/docs/install/autoinstall-reference.", "type": "object", "properties": { "version": { @@ -1219,7 +1219,12 @@ }, "minItems": 1, "uniqueItems": true, - "description": "Create the necessary directories for chef to run. By default, it creates the following directories:\n- ``/etc/chef``\n- ``/var/log/chef``\n- ``/var/lib/chef``\n- ``/var/cache/chef``\n- ``/var/backups/chef``\n- ``/var/run/chef``" + "description": "Create the necessary directories for chef to run. By default, it creates the following directories:\n- ``/etc/chef``\n- ``/var/log/chef``\n- ``/var/lib/chef``\n- ``/var/chef/backup``\n- ``/var/chef/cache``\n- ``/var/run/chef``" + }, + "config_path": { + "type": "string", + "default": "/etc/chef/client.rb", + "description": "Optional path for Chef configuration file. Default: ``/etc/chef/client.rb``" }, "validation_cert": { "type": "string", @@ -1257,13 +1262,13 @@ }, "file_backup_path": { "type": "string", - "default": "/var/backups/chef", - "description": "Specifies the location in which backup files are stored. By default, it uses the ``/var/backups/chef`` location." + "default": "/var/chef/backup", + "description": "Specifies the location in which backup files are stored. By default, it uses the ``/var/chef/backup`` location." }, "file_cache_path": { "type": "string", - "default": "/var/cache/chef", - "description": "Specifies the location in which chef cache files will be saved. By default, it uses the ``/var/cache/chef`` location." + "default": "/var/chef/cache", + "description": "Specifies the location in which chef cache files will be saved. By default, it uses the ``/var/chef/cache`` location." }, "json_attribs": { "type": "string", @@ -2759,7 +2764,7 @@ "additionalProperties": false, "properties": { "enabled": { - "description": "Whether vendor data is enabled or not. Default: ``true``.", + "description": "Whether vendor-data is enabled or not. Default: ``true``.", "oneOf": [ { "type": "boolean", diff --git a/cloudinit/config/schemas/schema-network-config-v1.json b/cloudinit/config/schemas/schema-network-config-v1.json index 8744e7e3..99e8c68b 100644 --- a/cloudinit/config/schemas/schema-network-config-v1.json +++ b/cloudinit/config/schemas/schema-network-config-v1.json @@ -384,6 +384,10 @@ "items": { "$ref": "#/$defs/config_type_subnet" } + }, + "mac_address": { + "type": "string", + "description": "When specifying MAC Address on a VLAN subinterface this value will be assigned to the vlan subinterface device and may be different than the MAC address of the physical interface. Specifying a MAC Address is optional. If ``mac_address`` is not present, then the VLAN subinterface will use the MAC Address values from one of the physical interface." } } }, @@ -484,7 +488,8 @@ "static6", "ipv6_dhcpv6-stateful", "ipv6_dhcpv6-stateless", - "ipv6_slaac" + "ipv6_slaac", + "manual" ] }, "control": { diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 183df368..b0b18ab1 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -50,6 +50,7 @@ from cloudinit.distros.package_management.utils import known_package_managers from cloudinit.distros.parsers import hosts from cloudinit.features import ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES +from cloudinit.lifecycle import log_with_downgradable_level from cloudinit.net import activators, dhcp, renderers from cloudinit.net.netops import NetOps from cloudinit.net.network_state import parse_net_config_data @@ -100,10 +101,6 @@ LOG = logging.getLogger(__name__) -# This is a best guess regex, based on current EC2 AZs on 2017-12-11. -# It could break when Amazon adds new regions and new AZs. -_EC2_AZ_RE = re.compile("^[a-z][a-z]-(?:[a-z]+-)+[0-9][a-z]$") - # Default NTP Client Configurations PREFERRED_NTP_CLIENTS = ["chrony", "systemd-timesyncd", "ntp", "ntpdate"] @@ -142,7 +139,7 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): shadow_empty_locked_passwd_patterns = ["^{username}::", "^{username}:!:"] tz_zone_dir = "/usr/share/zoneinfo" default_owner = "root:root" - init_cmd = ["service"] # systemctl, service etc + init_cmd: List[str] = ["service"] # systemctl, service etc renderer_configs: Mapping[str, MutableMapping[str, Any]] = {} _preferred_ntp_clients = None networking_cls: Type[Networking] = LinuxNetworking @@ -900,10 +897,13 @@ def create_user(self, name, **kwargs): password_key = "passwd" # Only "plain_text_passwd" and "hashed_passwd" # are valid for an existing user. - LOG.warning( - "'passwd' in user-data is ignored for existing " - "user %s", - name, + log_with_downgradable_level( + logger=LOG, + version="24.3", + requested_level=logging.WARNING, + msg="'passwd' in user-data is ignored " + "for existing user %s", + args=(name,), ) # As no password specified for the existing user in user-data @@ -941,20 +941,26 @@ def create_user(self, name, **kwargs): elif pre_existing_user: # Pre-existing user with no existing password and none # explicitly set in user-data. - LOG.warning( - "Not unlocking blank password for existing user %s." + log_with_downgradable_level( + logger=LOG, + version="24.3", + requested_level=logging.WARNING, + msg="Not unlocking blank password for existing user %s." " 'lock_passwd: false' present in user-data but no existing" " password set and no 'plain_text_passwd'/'hashed_passwd'" " provided in user-data", - name, + args=(name,), ) else: # No password (whether blank or otherwise) explicitly set - LOG.warning( - "Not unlocking password for user %s. 'lock_passwd: false'" + log_with_downgradable_level( + logger=LOG, + version="24.3", + requested_level=logging.WARNING, + msg="Not unlocking password for user %s. 'lock_passwd: false'" " present in user-data but no 'passwd'/'plain_text_passwd'/" "'hashed_passwd' provided in user-data", - name, + args=(name,), ) # Configure doas access @@ -1373,7 +1379,7 @@ def manage_service( "try-reload": [service, "restart"], "status": [service, "status"], } - cmd = list(init_cmd) + list(cmds[action]) + cmd = init_cmd + cmds[action] + list(extra_args) return subp.subp(cmd, capture=True, rcs=rcs) def set_keymap(self, layout: str, model: str, variant: str, options: str): @@ -1707,7 +1713,12 @@ def _get_package_mirror_info( # ec2 availability zones are named cc-direction-[0-9][a-d] (us-east-1b) # the region is us-east-1. so region = az[0:-1] - if _EC2_AZ_RE.match(data_source.availability_zone): + # This is a best guess regex, based on current EC2 AZs on 2017-12-11. + # It could break when Amazon adds new regions and new AZs. + if re.match( + "^[a-z][a-z]-(?:[a-z]+-)+[0-9][a-z]$", + data_source.availability_zone, + ): ec2_region = data_source.availability_zone[0:-1] if ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES: diff --git a/cloudinit/distros/aosc.py b/cloudinit/distros/aosc.py index 96fa48b8..6f9cd72c 100644 --- a/cloudinit/distros/aosc.py +++ b/cloudinit/distros/aosc.py @@ -135,7 +135,7 @@ def update_locale_conf(sys_path, locale_cfg): if v is None: continue v = str(v) - if len(v) == 0: + if not v: continue contents[k] = v updated_am += 1 diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 5ab41bbd..eb67d3fa 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -1,8 +1,10 @@ # Copyright (C) 2014 Rackspace, US Inc. # Copyright (C) 2016 Matthew Thode. +# Copyright (C) 2024 Andreas K. Huettel # # Author: Nate House # Author: Matthew Thode +# Author: Andreas K. Huettel # # This file is part of cloud-init. See LICENSE file for license information. @@ -18,7 +20,6 @@ class Distro(distros.Distro): locale_gen_fn = "/etc/locale.gen" - hostname_conf_fn = "/etc/conf.d/hostname" default_locale = "en_US.UTF-8" # C.UTF8 makes sense to generate, but is not selected @@ -27,20 +28,20 @@ class Distro(distros.Distro): def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) + + if distros.uses_systemd(): + self.hostname_conf_fn = "/etc/hostname" + else: + self.hostname_conf_fn = "/etc/conf.d/hostname" + # This will be used to restrict certain # calls from repeatedly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = "gentoo" - # Fix sshd restarts - cfg["ssh_svcname"] = "/etc/init.d/sshd" - if distros.uses_systemd(): - LOG.error("Cloud-init does not support systemd with gentoo") def apply_locale(self, _, out_fn=None): - """rc-only - not compatible with systemd - - Locales need to be added to /etc/locale.gen and generated prior + """Locales need to be added to /etc/locale.gen and generated prior to selection. Default to en_US.UTF-8 for simplicity. """ util.write_file(self.locale_gen_fn, "\n".join(self.locales), mode=644) @@ -48,7 +49,7 @@ def apply_locale(self, _, out_fn=None): # generate locales subp.subp(["locale-gen"], capture=False) - # select locale + # select locale, works for both openrc and systemd subp.subp( ["eselect", "locale", "set", self.default_locale], capture=False ) @@ -77,10 +78,17 @@ def _write_hostname(self, hostname, filename): if not conf: conf = HostnameConf("") - # Many distro's format is the hostname by itself, and that is the - # way HostnameConf works but gentoo expects it to be in - # hostname="the-actual-hostname" - conf.set_hostname('hostname="%s"' % hostname) + if distros.uses_systemd(): + # Gentoo uses the same format for /etc/hostname as everyone else- + # only the hostname by itself. Works for openrc and systemd, but + # openrc has its own config file and /etc/hostname is generated. + conf.set_hostname(hostname) + else: + # Openrc generates /etc/hostname from /etc/conf.d/hostname with the + # differing format + # hostname="the-actual-hostname" + conf.set_hostname('hostname="%s"' % hostname) + util.write_file(filename, str(conf), 0o644) def _read_system_hostname(self): diff --git a/cloudinit/distros/parsers/ifconfig.py b/cloudinit/distros/parsers/ifconfig.py index 1fc72d9e..de8fc145 100644 --- a/cloudinit/distros/parsers/ifconfig.py +++ b/cloudinit/distros/parsers/ifconfig.py @@ -102,7 +102,7 @@ def parse(self, text: str) -> Dict[str, Union[Ifstate, List[Ifstate]]]: ifs_by_mac = defaultdict(list) dev = None for line in text.splitlines(): - if len(line) == 0: + if not line: continue if line[0] not in ("\t", " "): # We hit the start of a device block in the ifconfig output diff --git a/cloudinit/distros/parsers/sys_conf.py b/cloudinit/distros/parsers/sys_conf.py index 4bbc2e4e..869c2c35 100644 --- a/cloudinit/distros/parsers/sys_conf.py +++ b/cloudinit/distros/parsers/sys_conf.py @@ -21,20 +21,19 @@ # or look at the 'param_expand()' function in the subst.c file in the bash # source tarball... SHELL_VAR_RULE = r"[a-zA-Z_]+[a-zA-Z0-9_]*" -SHELL_VAR_REGEXES = [ - # Basic variables - re.compile(r"\$" + SHELL_VAR_RULE), - # Things like $?, $0, $-, $@ - re.compile(r"\$[0-9#\?\-@\*]"), - # Things like ${blah:1} - but this one - # gets very complex so just try the - # simple path - re.compile(r"\$\{.+\}"), -] def _contains_shell_variable(text): - for r in SHELL_VAR_REGEXES: + for r in [ + # Basic variables + re.compile(r"\$" + SHELL_VAR_RULE), + # Things like $?, $0, $-, $@ + re.compile(r"\$[0-9#\?\-@\*]"), + # Things like ${blah:1} - but this one + # gets very complex so just try the + # simple path + re.compile(r"\$\{.+\}"), + ]: if r.search(text): return True return False @@ -66,7 +65,7 @@ def __str__(self): def _quote(self, value, multiline=False): if not isinstance(value, str): raise ValueError('Value "%s" is not a string' % (value)) - if len(value) == 0: + if not value: return "" quot_func = None if value[0] in ['"', "'"] and value[-1] in ['"', "'"]: diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py index 6a1b2816..4115bd2d 100644 --- a/cloudinit/distros/rhel_util.py +++ b/cloudinit/distros/rhel_util.py @@ -26,7 +26,7 @@ def update_sysconfig_file(fn, adjustments, allow_empty=False): if v is None: continue v = str(v) - if len(v) == 0 and not allow_empty: + if (not v) and (not allow_empty): continue contents[k] = v updated_am += 1 diff --git a/cloudinit/dmi.py b/cloudinit/dmi.py index e6fb68f3..e95c37b5 100644 --- a/cloudinit/dmi.py +++ b/cloudinit/dmi.py @@ -2,8 +2,7 @@ import logging import os import re -from collections import namedtuple -from typing import Optional +from typing import NamedTuple, Optional from cloudinit import performance, subp from cloudinit.util import ( @@ -18,8 +17,12 @@ # Path for DMI Data DMI_SYS_PATH = "/sys/class/dmi/id" -KernelNames = namedtuple("KernelNames", ["linux", "freebsd", "openbsd"]) -KernelNames.__new__.__defaults__ = (None, None, None) + +class KernelNames(NamedTuple): + linux: str + freebsd: Optional[str] + openbsd: Optional[str] + # FreeBSD's kenv(1) and Linux /sys/class/dmi/id/* both use different names from # dmidecode. The values are the same, and ultimately what we're interested in. diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 3000cb75..5e7eabe5 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -36,7 +36,6 @@ # a: 22 # # This gets loaded into yaml with final result {'a': 22} -DEF_MERGERS = mergers.string_extract_mergers("dict(replace)+list()+str()") CLOUD_PREFIX = "#cloud-config" JSONP_PREFIX = "#cloud-config-jsonp" @@ -103,7 +102,9 @@ def _extract_mergers(self, payload, headers): all_mergers.extend(mergers_yaml) all_mergers.extend(mergers_header) if not all_mergers: - all_mergers = DEF_MERGERS + all_mergers = mergers.string_extract_mergers( + "dict(replace)+list()+str()" + ) return (payload_yaml, all_mergers) def _merge_patch(self, payload): diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index dd853abd..8161c161 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -8,8 +8,6 @@ from cloudinit import importer, type_utils -NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$") - DEF_MERGE_TYPE = "list()+dict()+str()" MERGER_PREFIX = "m_" MERGER_ATTR = "Merger" @@ -108,7 +106,7 @@ def string_extract_mergers(merge_how): m_name = m_name.replace("-", "_") if not m_name: continue - match = NAME_MTCH.match(m_name) + match = re.match(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$", m_name) if not match: msg = "Matcher identifier '%s' is not in the right format" % ( m_name diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index acc9628a..159d08c1 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -827,7 +827,7 @@ def find_entry(mac, driver, device_id): "up": Iproute2.link_up, } - if len(ops) + len(ups) == 0: + if not (ops) and not (ups): if len(errors): LOG.warning( "Unable to rename interfaces: %s due to errors: %s", diff --git a/cloudinit/net/activators.py b/cloudinit/net/activators.py index de9a1d3c..94212894 100644 --- a/cloudinit/net/activators.py +++ b/cloudinit/net/activators.py @@ -206,9 +206,9 @@ def bring_up_interfaces(cls, device_names: Iterable[str]) -> bool: state, ) return _alter_interface( - ["systemctl", "reload-or-try-restart", "NetworkManager.service"], + ["systemctl", "try-reload-or-restart", "NetworkManager.service"], "all", - ) + ) and all(cls.bring_up_interface(device) for device in device_names) class NetplanActivator(NetworkActivator): diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index a95921c4..e67c6304 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -237,7 +237,7 @@ def parse_leases(lease_content: str) -> List[Dict[str, Any]]: """ lease_regex = re.compile(r"lease {(?P.*?)}\n", re.DOTALL) dhcp_leases: List[Dict] = [] - if len(lease_content) == 0: + if not lease_content: return [] for lease in lease_regex.findall(lease_content): lease_options = [] diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py index c429d068..9896aa02 100644 --- a/cloudinit/net/renderer.py +++ b/cloudinit/net/renderer.py @@ -21,9 +21,6 @@ def filter_by_attr(match_name): return lambda iface: (match_name in iface and iface[match_name]) -filter_by_physical = filter_by_type("physical") - - class Renderer(abc.ABC): def __init__(self, config=None): pass @@ -34,7 +31,7 @@ def _render_persistent_net(network_state: NetworkState): # TODO(harlowja): this seems shared between eni renderer and # this, so move it to a shared location. content = io.StringIO() - for iface in network_state.iter_interfaces(filter_by_physical): + for iface in network_state.iter_interfaces(filter_by_type("physical")): # for physical interfaces write out a persist net udev rule if "name" in iface and iface.get("mac_address"): driver = iface.get("driver", None) diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index d75012d2..98e4ed93 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -718,8 +718,9 @@ def _render_bonding_opts(cls, iface_cfg, iface, flavor): def _render_physical_interfaces( cls, network_state, iface_contents, flavor ): - physical_filter = renderer.filter_by_physical - for iface in network_state.iter_interfaces(physical_filter): + for iface in network_state.iter_interfaces( + renderer.filter_by_type("physical") + ): iface_name = iface.get("config_id") or iface["name"] iface_subnets = iface.get("subnets", []) iface_cfg = iface_contents[iface_name] @@ -937,7 +938,7 @@ def _render_networkmanager_conf(network_state, templates=None): ): content.set_section_keypair("main", "dns", "none") - if len(content) == 0: + if not content: return None out = "".join([_make_header(), "\n", "\n".join(content.write()), "\n"]) return out @@ -1118,6 +1119,24 @@ def render_network_state( if network_state.use_ipv6: netcfg.append("NETWORKING_IPV6=yes") netcfg.append("IPV6_AUTOCONF=no") + + # if sysconfig file exists and is not empty, append rest of the + # file content, do not remove the exsisting customizations. + if os.path.exists(sysconfig_path): + for line in util.load_text_file(sysconfig_path).splitlines(): + if ( + not any( + setting in line + for setting in [ + "NETWORKING", + "NETWORKING_IPV6", + "IPV6_AUTOCONF", + ] + ) + and line not in _make_header().splitlines() + ): + netcfg.append(line) + util.write_file( sysconfig_path, "\n".join(netcfg) + "\n", file_mode ) diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 6888693c..69cc56f8 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -189,7 +189,7 @@ def _netdev_info_ifconfig_netbsd(ifconfig_data): # fields that need to be returned in devs for each dev devs = {} for line in ifconfig_data.splitlines(): - if len(line) == 0: + if not line: continue if line[0] not in ("\t", " "): curdev = line.split()[0] @@ -237,7 +237,7 @@ def _netdev_info_ifconfig(ifconfig_data): # fields that need to be returned in devs for each dev devs = {} for line in ifconfig_data.splitlines(): - if len(line) == 0: + if not line: continue if line[0] not in ("\t", " "): curdev = line.split()[0] @@ -587,7 +587,8 @@ def netdev_pformat(): fields = ["Device", "Up", "Address", "Mask", "Scope", "Hw-Address"] tbl = SimpleTable(fields) for dev, data in sorted(netdev.items()): - for addr in data.get("ipv4"): + ipv4_addrs = data.get("ipv4") + for addr in ipv4_addrs: tbl.add_row( ( dev, @@ -598,7 +599,9 @@ def netdev_pformat(): data["hwaddr"], ) ) - for addr in data.get("ipv6"): + + ipv6_addrs = data.get("ipv6") + for addr in ipv6_addrs: tbl.add_row( ( dev, @@ -609,7 +612,7 @@ def netdev_pformat(): data["hwaddr"], ) ) - if len(data.get("ipv6")) + len(data.get("ipv4")) == 0: + if not (ipv4_addrs) and not (ipv6_addrs): tbl.add_row( (dev, data["up"], empty, empty, empty, data["hwaddr"]) ) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 011b0bbe..112494fd 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -345,7 +345,7 @@ def _break_down(self, key, meta_data, description): result_array.append(self._encode_kvp_item(subkey, value)) i += 1 des_in_json = des_in_json[room_for_desc:] - if len(des_in_json) == 0: + if not des_in_json: break return result_array diff --git a/cloudinit/settings.py b/cloudinit/settings.py index a73f2511..f2ca6585 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -28,6 +28,7 @@ "DigitalOcean", "Azure", "AltCloud", + "VMware", "OVF", "MAAS", "GCE", @@ -46,7 +47,6 @@ "Exoscale", "RbxCloud", "UpCloud", - "VMware", "NWCS", "Akamai", "WSL", diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py index d674e1fc..286ed2af 100644 --- a/cloudinit/sources/DataSourceAliYun.py +++ b/cloudinit/sources/DataSourceAliYun.py @@ -2,27 +2,43 @@ import copy import logging -from typing import List +from typing import List, Union from cloudinit import dmi, sources +from cloudinit import url_helper as uhelp +from cloudinit import util from cloudinit.event import EventScope, EventType -from cloudinit.sources import DataSourceEc2 as EC2 -from cloudinit.sources import DataSourceHostname, NicOrder +from cloudinit.net.dhcp import NoDHCPLeaseError +from cloudinit.net.ephemeral import EphemeralIPNetwork +from cloudinit.sources import DataSourceHostname +from cloudinit.sources.helpers import aliyun, ec2 LOG = logging.getLogger(__name__) ALIYUN_PRODUCT = "Alibaba Cloud ECS" -class DataSourceAliYun(EC2.DataSourceEc2): +class DataSourceAliYun(sources.DataSource): dsname = "AliYun" metadata_urls = ["http://100.100.100.200"] - # The minimum supported metadata_version from the ec2 metadata apis + # The minimum supported metadata_version from the ecs metadata apis min_metadata_version = "2016-01-01" extended_metadata_versions: List[str] = [] + # Setup read_url parameters per get_url_params. + url_max_wait = 240 + url_timeout = 50 + + # API token for accessing the metadata service + _api_token = None + # Used to cache calculated network cfg v1 + _network_config: Union[str, dict] = sources.UNSET + + # Whether we want to get network configuration from the metadata service. + perform_dhcp_setup = False + # Aliyun metadata server security enhanced mode overwrite @property def imdsv2_token_put_header(self): @@ -32,11 +48,9 @@ def __init__(self, sys_cfg, distro, paths): super(DataSourceAliYun, self).__init__(sys_cfg, distro, paths) self.default_update_events = copy.deepcopy(self.default_update_events) self.default_update_events[EventScope.NETWORK].add(EventType.BOOT) - self._fallback_nic_order = NicOrder.NIC_NAME def _unpickle(self, ci_pkl_version: int) -> None: super()._unpickle(ci_pkl_version) - self._fallback_nic_order = NicOrder.NIC_NAME def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): hostname = self.metadata.get("hostname") @@ -51,9 +65,316 @@ def get_public_ssh_keys(self): def _get_cloud_name(self): if _is_aliyun(): - return EC2.CloudNames.ALIYUN + return self.dsname.lower() + return "NO_ALIYUN_METADATA" + + @property + def platform(self): + return self.dsname.lower() + + # IMDSv2 related parameters from the ecs metadata api document + @property + def api_token_route(self): + return "latest/api/token" + + @property + def imdsv2_token_ttl_seconds(self): + return "21600" + + @property + def imdsv2_token_redact(self): + return [self.imdsv2_token_put_header, self.imdsv2_token_req_header] + + @property + def imdsv2_token_req_header(self): + return self.imdsv2_token_put_header + "-ttl-seconds" + + @property + def network_config(self): + """Return a network config dict for rendering ENI or netplan files.""" + if self._network_config != sources.UNSET: + return self._network_config + + result = {} + iface = self.distro.fallback_interface + net_md = self.metadata.get("network") + if isinstance(net_md, dict): + result = aliyun.convert_ecs_metadata_network_config( + net_md, + fallback_nic=iface, + full_network_config=util.get_cfg_option_bool( + self.ds_cfg, "apply_full_imds_network_config", True + ), + ) else: - return EC2.CloudNames.NO_EC2_METADATA + LOG.warning("Metadata 'network' key not valid: %s.", net_md) + return result + self._network_config = result + return self._network_config + + def _maybe_fetch_api_token(self, mdurls): + """Get an API token for ECS Instance Metadata Service. + + On ECS. IMDS will always answer an API token, set + HttpTokens=optional (default) when create instance will not forcefully + use the security-enhanced mode (IMDSv2). + + https://api.alibabacloud.com/api/Ecs/2014-05-26/RunInstances + """ + + urls = [] + url2base = {} + url_path = self.api_token_route + request_method = "PUT" + for url in mdurls: + cur = "{0}/{1}".format(url, url_path) + urls.append(cur) + url2base[cur] = url + + # use the self._imds_exception_cb to check for Read errors + LOG.debug("Fetching Ecs IMDSv2 API Token") + + response = None + url = None + url_params = self.get_url_params() + try: + url, response = uhelp.wait_for_url( + urls=urls, + max_wait=url_params.max_wait_seconds, + timeout=url_params.timeout_seconds, + status_cb=LOG.warning, + headers_cb=self._get_headers, + exception_cb=self._imds_exception_cb, + request_method=request_method, + headers_redact=self.imdsv2_token_redact, + connect_synchronously=False, + ) + except uhelp.UrlError: + # We use the raised exception to interupt the retry loop. + # Nothing else to do here. + pass + + if url and response: + self._api_token = response + return url2base[url] + + # If we get here, then wait_for_url timed out, waiting for IMDS + # or the IMDS HTTP endpoint is disabled + return None + + def wait_for_metadata_service(self): + mcfg = self.ds_cfg + mdurls = mcfg.get("metadata_urls", self.metadata_urls) + + # try the api token path first + metadata_address = self._maybe_fetch_api_token(mdurls) + + if metadata_address: + self.metadata_address = metadata_address + LOG.debug("Using metadata source: '%s'", self.metadata_address) + else: + LOG.warning("IMDS's HTTP endpoint is probably disabled") + return bool(metadata_address) + + def crawl_metadata(self): + """Crawl metadata service when available. + + @returns: Dictionary of crawled metadata content containing the keys: + meta-data, user-data, vendor-data and dynamic. + """ + if not self.wait_for_metadata_service(): + return {} + redact = self.imdsv2_token_redact + crawled_metadata = {} + exc_cb = self._refresh_stale_aliyun_token_cb + exc_cb_ud = self._skip_or_refresh_stale_aliyun_token_cb + skip_cb = None + exe_cb_whole_meta = self._skip_json_path_meta_path_aliyun_cb + try: + crawled_metadata["user-data"] = aliyun.get_instance_data( + self.min_metadata_version, + self.metadata_address, + headers_cb=self._get_headers, + headers_redact=redact, + exception_cb=exc_cb_ud, + item_name="user-data", + ) + crawled_metadata["vendor-data"] = aliyun.get_instance_data( + self.min_metadata_version, + self.metadata_address, + headers_cb=self._get_headers, + headers_redact=redact, + exception_cb=exc_cb_ud, + item_name="vendor-data", + ) + try: + result = aliyun.get_instance_meta_data( + self.min_metadata_version, + self.metadata_address, + headers_cb=self._get_headers, + headers_redact=redact, + exception_cb=exe_cb_whole_meta, + ) + crawled_metadata["meta-data"] = result + except Exception: + util.logexc( + LOG, + "Faild read json meta-data from %s " + "fall back directory tree style", + self.metadata_address, + ) + crawled_metadata["meta-data"] = ec2.get_instance_metadata( + self.min_metadata_version, + self.metadata_address, + headers_cb=self._get_headers, + headers_redact=redact, + exception_cb=exc_cb, + retrieval_exception_ignore_cb=skip_cb, + ) + except Exception: + util.logexc( + LOG, + "Failed reading from metadata address %s", + self.metadata_address, + ) + return {} + return crawled_metadata + + def _refresh_stale_aliyun_token_cb(self, msg, exception): + """Exception handler for Ecs to refresh token if token is stale.""" + if isinstance(exception, uhelp.UrlError) and exception.code == 401: + # With _api_token as None, _get_headers will _refresh_api_token. + LOG.debug("Clearing cached Ecs API token due to expiry") + self._api_token = None + return True # always retry + + def _skip_retry_on_codes(self, status_codes, cause): + """Returns False if cause.code is in status_codes.""" + return cause.code not in status_codes + + def _skip_or_refresh_stale_aliyun_token_cb(self, msg, exception): + """Callback will not retry on SKIP_USERDATA_VENDORDATA_CODES or + if no token is available.""" + retry = self._skip_retry_on_codes(ec2.SKIP_USERDATA_CODES, exception) + if not retry: + return False # False raises exception + return self._refresh_stale_aliyun_token_cb(msg, exception) + + def _skip_json_path_meta_path_aliyun_cb(self, msg, exception): + """Callback will not retry of whole meta_path is not found""" + if isinstance(exception, uhelp.UrlError) and exception.code == 404: + LOG.warning("whole meta_path is not found, skipping") + return False + return self._refresh_stale_aliyun_token_cb(msg, exception) + + def _get_data(self): + if self.cloud_name != self.dsname.lower(): + return False + if self.perform_dhcp_setup: # Setup networking in init-local stage. + if util.is_FreeBSD(): + LOG.debug("FreeBSD doesn't support running dhclient with -sf") + return False + try: + with EphemeralIPNetwork( + self.distro, + self.distro.fallback_interface, + ipv4=True, + ipv6=False, + ) as netw: + self._crawled_metadata = self.crawl_metadata() + LOG.debug( + "Crawled metadata service%s", + f" {netw.state_msg}" if netw.state_msg else "", + ) + + except NoDHCPLeaseError: + return False + else: + self._crawled_metadata = self.crawl_metadata() + if not self._crawled_metadata or not isinstance( + self._crawled_metadata, dict + ): + return False + self.metadata = self._crawled_metadata.get("meta-data", {}) + self.userdata_raw = self._crawled_metadata.get("user-data", {}) + self.vendordata_raw = self._crawled_metadata.get("vendor-data", {}) + return True + + def _refresh_api_token(self, seconds=None): + """Request new metadata API token. + @param seconds: The lifetime of the token in seconds + + @return: The API token or None if unavailable. + """ + + if seconds is None: + seconds = self.imdsv2_token_ttl_seconds + + LOG.debug("Refreshing Ecs metadata API token") + request_header = {self.imdsv2_token_req_header: seconds} + token_url = "{}/{}".format(self.metadata_address, self.api_token_route) + try: + response = uhelp.readurl( + token_url, + headers=request_header, + headers_redact=self.imdsv2_token_redact, + request_method="PUT", + ) + except uhelp.UrlError as e: + LOG.warning( + "Unable to get API token: %s raised exception %s", token_url, e + ) + return None + return response.contents + + def _get_headers(self, url=""): + """Return a dict of headers for accessing a url. + + If _api_token is unset on AWS, attempt to refresh the token via a PUT + and then return the updated token header. + """ + + request_token_header = { + self.imdsv2_token_req_header: self.imdsv2_token_ttl_seconds + } + if self.api_token_route in url: + return request_token_header + if not self._api_token: + # If we don't yet have an API token, get one via a PUT against + # api_token_route. This _api_token may get unset by a 403 due + # to an invalid or expired token + self._api_token = self._refresh_api_token() + if not self._api_token: + return {} + return {self.imdsv2_token_put_header: self._api_token} + + def _imds_exception_cb(self, exception=None): + """Fail quickly on proper AWS if IMDSv2 rejects API token request + + Guidance from Amazon is that if IMDSv2 had disabled token requests + by returning a 403, or cloud-init malformed requests resulting in + other 40X errors, we want the datasource detection to fail quickly + without retries as those symptoms will likely not be resolved by + retries. + + Exceptions such as requests.ConnectionError due to IMDS being + temporarily unroutable or unavailable will still retry due to the + callsite wait_for_url. + """ + if isinstance(exception, uhelp.UrlError): + # requests.ConnectionError will have exception.code == None + if exception.code and exception.code >= 400: + if exception.code == 403: + LOG.warning( + "Ecs IMDS endpoint returned a 403 error. " + "HTTP endpoint is disabled. Aborting." + ) + else: + LOG.warning( + "Fatal error while requesting Ecs IMDSv2 API tokens" + ) + raise exception + return True def _is_aliyun(): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 863bcbbd..b5c4522b 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -467,6 +467,11 @@ def _setup_ephemeral_networking( ), host_only=True, ) + except FileNotFoundError as error: + report_diagnostic_event( + "File not found during DHCP %r" % error, + logger_func=LOG.error, + ) except subp.ProcessExecutionError as error: # udevadm settle, ip link set dev eth0 up, etc. report_diagnostic_event( @@ -758,14 +763,12 @@ def crawl_metadata(self): if imds_hostname: LOG.debug("Hostname retrieved from IMDS: %s", imds_hostname) crawled_data["metadata"]["local-hostname"] = imds_hostname - if imds_disable_password: + if imds_disable_password is not None: LOG.debug( "Disable password retrieved from IMDS: %s", imds_disable_password, ) - crawled_data["metadata"][ - "disable_password" - ] = imds_disable_password + crawled_data["cfg"]["ssh_pwauth"] = not imds_disable_password if self.seed == "IMDS" and not crawled_data["files"]: try: @@ -1733,15 +1736,18 @@ def can_dev_be_reformatted(devpath, preserve_ntfs): # devpath of /dev/sd[a-z] or /dev/disk/cloud/azure_resource # where partitions are "1" or "-part1" or "p1" partitions = _partitions_on_device(devpath) - if len(partitions) == 0: + if not partitions: return False, "device %s was not partitioned" % devpath - elif len(partitions) > 2: + + partition_len = len(partitions) + if partition_len > 2: msg = "device %s had 3 or more partitions: %s" % ( devpath, " ".join([p[1] for p in partitions]), ) return False, msg - elif len(partitions) == 2: + + if partition_len == 2: cand_part, cand_path = partitions[1] else: cand_part, cand_path = partitions[0] diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index e47a0c98..88e6ec65 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -34,7 +34,6 @@ class CloudNames: - ALIYUN = "aliyun" AWS = "aws" BRIGHTBOX = "brightbox" ZSTACK = "zstack" @@ -54,7 +53,7 @@ def skip_404_tag_errors(exception): # Cloud platforms that support IMDSv2 style metadata server -IDMSV2_SUPPORTED_CLOUD_PLATFORMS = [CloudNames.AWS, CloudNames.ALIYUN] +IDMSV2_SUPPORTED_CLOUD_PLATFORMS = [CloudNames.AWS] # Only trigger hook-hotplug on NICs with Ec2 drivers. Avoid triggering # it on docker virtual NICs and the like. LP: #1946003 @@ -777,11 +776,6 @@ def warn_if_necessary(cfgval, cfg): warnings.show_warning("non_ec2_md", cfg, mode=True, sleep=sleep) -def identify_aliyun(data): - if data["product_name"] == "Alibaba Cloud ECS": - return CloudNames.ALIYUN - - def identify_aws(data): # data is a dictionary returned by _collect_platform_data. uuid_str = data["uuid"] @@ -830,7 +824,6 @@ def identify_platform(): identify_zstack, identify_e24cloud, identify_outscale, - identify_aliyun, lambda x: CloudNames.UNKNOWN, ) for checker in checks: @@ -895,7 +888,7 @@ def _build_nic_order( @return: Dictionary with macs as keys and nic orders as values. """ nic_order: Dict[str, int] = {} - if len(macs_to_nics) == 0 or len(macs_metadata) == 0: + if (not macs_to_nics) or (not macs_metadata): return nic_order valid_macs_metadata = filter( diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 8054d6f1..8b5ddb5f 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -87,6 +87,7 @@ def __init__(self, sys_cfg, distro, paths): def _get_data(self): url_params = self.get_url_params() + ret = {} if self.perform_dhcp_setup: candidate_nics = net.find_candidate_nics() if DEFAULT_PRIMARY_INTERFACE in candidate_nics: @@ -116,6 +117,9 @@ def _get_data(self): ) continue except NoDHCPLeaseError: + LOG.debug( + "Unable to obtain a DHCP lease for %s", candidate_nic + ) continue if ret["success"]: self.distro.fallback_interface = candidate_nic @@ -128,14 +132,14 @@ def _get_data(self): else: ret = read_md(address=self.metadata_address, url_params=url_params) - if not ret["success"]: - if ret["platform_reports_gce"]: - LOG.warning(ret["reason"]) + if not ret.get("success"): + if ret.get("platform_reports_gce"): + LOG.warning(ret.get("reason")) else: - LOG.debug(ret["reason"]) + LOG.debug(ret.get("reason")) return False - self.metadata = ret["meta-data"] - self.userdata_raw = ret["user-data"] + self.metadata = ret.get("meta-data") + self.userdata_raw = ret.get("user-data") return True @property diff --git a/cloudinit/sources/DataSourceLXD.py b/cloudinit/sources/DataSourceLXD.py index b23eae97..0f02c85e 100644 --- a/cloudinit/sources/DataSourceLXD.py +++ b/cloudinit/sources/DataSourceLXD.py @@ -196,7 +196,13 @@ def _unpickle(self, ci_pkl_version: int) -> None: @staticmethod def ds_detect() -> bool: """Check platform environment to report if this datasource may run.""" - return is_platform_viable() + if not os.path.exists(LXD_SOCKET_PATH): + LOG.warning("%s does not exist.", LXD_SOCKET_PATH) + return False + elif not stat.S_ISSOCK(os.lstat(LXD_SOCKET_PATH).st_mode): + LOG.warning("%s is not a socket", LXD_SOCKET_PATH) + return False + return True def _get_data(self) -> bool: """Crawl LXD socket API instance data and return True on success""" @@ -268,13 +274,6 @@ def network_config(self) -> dict: return cast(dict, self._network_config) -def is_platform_viable() -> bool: - """Return True when this platform appears to have an LXD socket.""" - if os.path.exists(LXD_SOCKET_PATH): - return stat.S_ISSOCK(os.lstat(LXD_SOCKET_PATH).st_mode) - return False - - def _get_json_response( session: requests.Session, url: str, do_raise: bool = True ): @@ -427,7 +426,7 @@ def read_metadata( when the LXD configuration setting `security.devlxd` is true. When `security.devlxd` is false, no /dev/lxd/socket file exists. This - datasource will return False from `is_platform_viable` in that case. + datasource will return False from `ds_detect` in that case. Perform a GET of /config` and walk all `user.*` configuration keys, storing all keys and values under a dict key diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 933d95c9..1ad4a98c 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -197,7 +197,7 @@ def get_id_from_ds_cfg(ds_cfg): def read_maas_seed_dir(seed_d): if seed_d.startswith("file://"): seed_d = seed_d[7:] - if not os.path.isdir(seed_d) or len(os.listdir(seed_d)) == 0: + if not os.path.isdir(seed_d) or not os.listdir(seed_d): raise MAASSeedDirNone("%s: not a directory") # seed_dir looks in seed_dir, not seed_dir/VERSION @@ -283,7 +283,7 @@ def check_seed_contents(content, seed): else: ret[dpath] = content[spath] - if len(ret) == 0: + if not ret: raise MAASSeedDirNone("%s: no data files found" % seed) if missing: diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 7b4171ab..363cf860 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -167,7 +167,7 @@ def _pp2d_callback(mp, data): # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. - if len(found) == 0: + if not found: return False # The special argument "seedfrom" indicates we should diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 20bcac58..ad2ffbf1 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -78,7 +78,7 @@ def _get_data(self): found.append(name) # There was no OVF transports found - if len(found) == 0: + if not found: return False if "seedfrom" in md and md["seedfrom"]: @@ -330,10 +330,11 @@ def query_guestinfo(rpctool, rpctool_fn): # If the first attempt at getting the data was with vmtoolsd, then # no second attempt is made. if vmtoolsd and rpctool == vmtoolsd: - # The fallback failed, log the error. - util.logexc( - LOG, "vmtoolsd failed to get guestinfo.ovfEnv: %s", error - ) + # The fallback failed and exit code is not 1, log the error. + if error.exit_code != 1: + util.logexc( + LOG, "vmtoolsd failed to get guestinfo.ovfEnv: %s", error + ) return None if not vmtoolsd: @@ -344,10 +345,11 @@ def query_guestinfo(rpctool, rpctool_fn): LOG.info("fallback to vmtoolsd") return query_guestinfo(vmtoolsd, exec_vmtoolsd) except subp.ProcessExecutionError as error: - # The fallback failed, log the error. - util.logexc( - LOG, "vmtoolsd failed to get guestinfo.ovfEnv: %s", error - ) + # The fallback failed and exit code is not 1, log the error. + if error.exit_code != 1: + util.logexc( + LOG, "vmtoolsd failed to get guestinfo.ovfEnv: %s", error + ) return None @@ -378,7 +380,7 @@ def get_properties(contents): dom.documentElement, lambda n: n.localName == "PropertySection" ) - if len(propSections) == 0: + if not propSections: raise XmlError("No 'PropertySection's") props = {} diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 019a0a12..565bd6ed 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -31,10 +31,12 @@ # -> compute.defaults.vmware.smbios_asset_tag for this value DMI_ASSET_TAG_SAPCCLOUD = "SAP CCloud VM" DMI_ASSET_TAG_HUAWEICLOUD = "HUAWEICLOUD" +DMI_ASSET_TAG_SAMSUNGCLOUDPLATFORM = "Samsung Cloud Platform" VALID_DMI_ASSET_TAGS = VALID_DMI_PRODUCT_NAMES VALID_DMI_ASSET_TAGS += [ DMI_ASSET_TAG_HUAWEICLOUD, DMI_ASSET_TAG_OPENTELEKOM, + DMI_ASSET_TAG_SAMSUNGCLOUDPLATFORM, DMI_ASSET_TAG_SAPCCLOUD, ] diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index fe9da48d..bc8d320c 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -438,7 +438,7 @@ def as_ascii(): while True: try: byte = self.fp.read(1) - if len(byte) == 0: + if not byte: raise JoyentMetadataTimeoutException(msg % as_ascii()) if byte == b"\n": return as_ascii() diff --git a/cloudinit/sources/DataSourceVMware.py b/cloudinit/sources/DataSourceVMware.py index 0e328cf3..d69f3673 100644 --- a/cloudinit/sources/DataSourceVMware.py +++ b/cloudinit/sources/DataSourceVMware.py @@ -500,7 +500,7 @@ def get_none_if_empty_val(val): # simplify the rest of this function's logic. val = util.decode_binary(val) val = val.rstrip() - if len(val) == 0 or val == GUESTINFO_EMPTY_YAML_VAL: + if (not val) or (val == GUESTINFO_EMPTY_YAML_VAL): return None return val diff --git a/cloudinit/sources/DataSourceWSL.py b/cloudinit/sources/DataSourceWSL.py index 5e146ecc..f99ecb5c 100644 --- a/cloudinit/sources/DataSourceWSL.py +++ b/cloudinit/sources/DataSourceWSL.py @@ -237,14 +237,14 @@ def merge_agent_landscape_data( cloud-init to merge separate parts. """ # Ignore agent_data if None or empty - if agent_data is None or len(agent_data.raw) == 0: - if user_data is None or len(user_data.raw) == 0: + if (agent_data is None) or (not agent_data.raw): + if (user_data is None) or (not user_data.raw): return None return user_data.raw # Ignore user_data if None or empty - if user_data is None or len(user_data.raw) == 0: - if agent_data is None or len(agent_data.raw) == 0: + if (user_data is None) or (not user_data.raw): + if (agent_data is None) or (not agent_data.raw): return None return agent_data.raw diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 7095a647..38533fe5 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -15,7 +15,6 @@ import os import pickle import re -from collections import namedtuple from enum import Enum, unique from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union @@ -177,20 +176,16 @@ def redact_sensitive_keys(metadata, redact_value=REDACT_SENSITIVE_VALUE): return md_copy -URLParams = namedtuple( - "URLParams", - [ - "max_wait_seconds", - "timeout_seconds", - "num_retries", - "sec_between_retries", - ], -) +class URLParams(NamedTuple): + max_wait_seconds: int + timeout_seconds: int + num_retries: int + sec_between_retries: int -DataSourceHostname = namedtuple( - "DataSourceHostname", - ["hostname", "is_default"], -) + +class DataSourceHostname(NamedTuple): + hostname: Optional[str] + is_default: bool class HotplugRetrySettings(NamedTuple): @@ -844,7 +839,7 @@ def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): @param metadata_only: Boolean, set True to avoid looking up hostname if meta-data doesn't have local-hostname present. - @return: a DataSourceHostname namedtuple + @return: a DataSourceHostname NamedTuple , (str, bool). is_default is a bool and it's true only if hostname is localhost and was diff --git a/cloudinit/sources/helpers/aliyun.py b/cloudinit/sources/helpers/aliyun.py new file mode 100644 index 00000000..201ceb04 --- /dev/null +++ b/cloudinit/sources/helpers/aliyun.py @@ -0,0 +1,211 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import logging +from typing import MutableMapping + +from cloudinit import net, url_helper, util +from cloudinit.sources.helpers import ec2 + +LOG = logging.getLogger(__name__) + + +def get_instance_meta_data( + api_version="latest", + metadata_address="http://100.100.100.200", + ssl_details=None, + timeout=5, + retries=5, + headers_cb=None, + headers_redact=None, + exception_cb=None, +): + ud_url = url_helper.combine_url(metadata_address, api_version) + ud_url = url_helper.combine_url(ud_url, "meta-data/all") + response = url_helper.read_file_or_url( + ud_url, + ssl_details=ssl_details, + timeout=timeout, + retries=retries, + exception_cb=exception_cb, + headers_cb=headers_cb, + headers_redact=headers_redact, + ) + meta_data_raw: object = util.load_json(response.contents) + + # meta_data_raw is a json object with the following format get + # by`meta-data/all` + # { + # "sub-private-ipv4-list": "", + # "dns-conf": { + # "nameservers": "100.100.2.136\r\n100.100.2.138" + # }, + # "zone-id": "cn-hangzhou-i", + # "instance": { + # "instance-name": "aliyun_vm_test", + # "instance-type": "ecs.g7.xlarge" + # }, + # "disks": { + # "bp1cikh4di1xxxx": { + # "name": "disk_test", + # "id": "d-bp1cikh4di1lf7pxxxx" + # } + # }, + # "instance-id": "i-bp123", + # "eipv4": "47.99.152.7", + # "private-ipv4": "192.168.0.9", + # "hibernation": { + # "configured": "false" + # }, + # "vpc-id": "vpc-bp1yeqg123", + # "mac": "00:16:3e:30:3e:ca", + # "source-address": "http://mirrors.cloud.aliyuncs.com", + # "vswitch-cidr-block": "192.168.0.0/24", + # "network": { + # "interfaces": { + # "macs": { + # "00:16:3e:30:3e:ca": { + # "vpc-cidr-block": "192.168.0.0/16", + # "netmask": "255.255.255.0" + # } + # } + # } + # }, + # "network-type": "vpc", + # "hostname": "aliyun_vm_test", + # "region-id": "cn-hangzhou", + # "ntp-conf": { + # "ntp-servers": "ntp1.aliyun.com\r\nntp2.aliyun.com" + # }, + # } + # Note: For example, in the values of dns conf: the `nameservers` + # key is a string, the format is the same as the response from the + # `meta-data/dns-conf/nameservers` endpoint. we use the same + # serialization method to ensure consistency between + # the two methods (directory tree and json path). + def _process_dict_values(d): + if isinstance(d, dict): + return {k: _process_dict_values(v) for k, v in d.items()} + elif isinstance(d, list): + return [_process_dict_values(item) for item in d] + else: + return ec2.MetadataLeafDecoder()("", d) + + return _process_dict_values(meta_data_raw) + + +def get_instance_data( + api_version="latest", + metadata_address="http://100.100.100.200", + ssl_details=None, + timeout=5, + retries=5, + headers_cb=None, + headers_redact=None, + exception_cb=None, + item_name=None, +): + ud_url = url_helper.combine_url(metadata_address, api_version) + ud_url = url_helper.combine_url(ud_url, item_name) + data = b"" + support_items_list = ["user-data", "vendor-data"] + if item_name not in support_items_list: + LOG.error( + "aliyun datasource not support the item %s", + item_name, + ) + return data + try: + response = url_helper.read_file_or_url( + ud_url, + ssl_details=ssl_details, + timeout=timeout, + retries=retries, + exception_cb=exception_cb, + headers_cb=headers_cb, + headers_redact=headers_redact, + ) + data = response.contents + except Exception: + util.logexc(LOG, "Failed fetching %s from url %s", item_name, ud_url) + return data + + +def convert_ecs_metadata_network_config( + network_md, + macs_to_nics=None, + fallback_nic=None, + full_network_config=True, +): + """Convert ecs metadata to network config version 2 data dict. + + @param: network_md: 'network' portion of ECS metadata. + generally formed as {"interfaces": {"macs": {}} where + 'macs' is a dictionary with mac address as key: + @param: macs_to_nics: Optional dict of mac addresses and nic names. If + not provided, get_interfaces_by_mac is called to get it from the OS. + @param: fallback_nic: Optionally provide the primary nic interface name. + This nic will be guaranteed to minimally have a dhcp4 configuration. + @param: full_network_config: Boolean set True to configure all networking + presented by IMDS. This includes rendering secondary IPv4 and IPv6 + addresses on all NICs and rendering network config on secondary NICs. + If False, only the primary nic will be configured and only with dhcp + (IPv4/IPv6). + + @return A dict of network config version 2 based on the metadata and macs. + """ + netcfg: MutableMapping = {"version": 2, "ethernets": {}} + if not macs_to_nics: + macs_to_nics = net.get_interfaces_by_mac() + macs_metadata = network_md["interfaces"]["macs"] + + if not full_network_config: + for mac, nic_name in macs_to_nics.items(): + if nic_name == fallback_nic: + break + dev_config: MutableMapping = { + "dhcp4": True, + "dhcp6": False, + "match": {"macaddress": mac.lower()}, + "set-name": nic_name, + } + nic_metadata = macs_metadata.get(mac) + if nic_metadata.get("ipv6s"): # Any IPv6 addresses configured + dev_config["dhcp6"] = True + netcfg["ethernets"][nic_name] = dev_config + return netcfg + nic_name_2_mac_map = dict() + for mac, nic_name in macs_to_nics.items(): + nic_metadata = macs_metadata.get(mac) + if not nic_metadata: + continue # Not a physical nic represented in metadata + nic_name_2_mac_map[nic_name] = mac + + # sorted by nic_name + orderd_nic_name_list = sorted( + nic_name_2_mac_map.keys(), key=net.natural_sort_key + ) + for nic_idx, nic_name in enumerate(orderd_nic_name_list): + nic_mac = nic_name_2_mac_map[nic_name] + nic_metadata = macs_metadata.get(nic_mac) + dhcp_override = {"route-metric": (nic_idx + 1) * 100} + dev_config = { + "dhcp4": True, + "dhcp4-overrides": dhcp_override, + "dhcp6": False, + "match": {"macaddress": nic_mac.lower()}, + "set-name": nic_name, + } + if nic_metadata.get("ipv6s"): # Any IPv6 addresses configured + dev_config["dhcp6"] = True + dev_config["dhcp6-overrides"] = dhcp_override + + netcfg["ethernets"][nic_name] = dev_config + # Remove route-metric dhcp overrides and routes / routing-policy if only + # one nic configured + if len(netcfg["ethernets"]) == 1: + for nic_name in netcfg["ethernets"].keys(): + netcfg["ethernets"][nic_name].pop("dhcp4-overrides") + netcfg["ethernets"][nic_name].pop("dhcp6-overrides", None) + netcfg["ethernets"][nic_name].pop("routes", None) + netcfg["ethernets"][nic_name].pop("routing-policy", None) + return netcfg diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 7e79f19e..3a200ee2 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -451,7 +451,7 @@ def generate_certificate(self): "-days", "32768", "-newkey", - "rsa:2048", + "rsa:3072", "-keyout", self.certificate_names["private_key"], "-out", @@ -1034,7 +1034,7 @@ def _find( matches = node.findall( "./%s:%s" % (namespace, name), OvfEnvXml.NAMESPACES ) - if len(matches) == 0: + if not matches: msg = "missing configuration for %r" % name LOG.debug(msg) if required: @@ -1058,13 +1058,14 @@ def _parse_property( default=None, ): matches = node.findall("./wa:" + name, OvfEnvXml.NAMESPACES) - if len(matches) == 0: + if not matches: msg = "missing configuration for %r" % name LOG.debug(msg) if required: raise errors.ReportableErrorOvfInvalidMetadata(msg) return default - elif len(matches) > 1: + + if len(matches) > 1: raise errors.ReportableErrorOvfInvalidMetadata( "multiple configuration matches for %r (%d)" % (name, len(matches)) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 62918599..8f682ea3 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -408,7 +408,7 @@ def read_v1(self): path = self._path_join(self.base_path, name) if os.path.exists(path): found[name] = path - if len(found) == 0: + if not found: raise NonReadable("%s: no files found" % (self.base_path)) md = {} @@ -496,7 +496,7 @@ def _path_read(self, path, decode=False): def should_retry_cb(cause): try: code = int(cause.code) - if code >= 400: + if code >= 400 and code not in [408, 429, 500, 502, 503, 504]: return False except (TypeError, ValueError): # Older versions of requests didn't have a code. @@ -732,6 +732,7 @@ def convert_net_json(network_json=None, known_macs=None): { "name": name, "vlan_id": link["vlan_id"], + "mac_address": link["vlan_mac_address"], } ) link_updates.append((cfg, "vlan_link", "%s", link["vlan_link"])) @@ -771,7 +772,11 @@ def convert_net_json(network_json=None, known_macs=None): if not mac: raise ValueError("No mac_address or name entry for %s" % d) if mac not in known_macs: - raise ValueError("Unable to find a system nic for %s" % d) + # Let's give udev a chance to catch up + util.udevadm_settle() + known_macs = net.get_interfaces_by_mac() + if mac not in known_macs: + raise ValueError("Unable to find a system nic for %s" % d) d["name"] = known_macs[mac] for cfg, key, fmt, targets in link_updates: diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index b07214a2..78526fdf 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -1,38 +1,31 @@ # Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2016 VMware INC. +# Copyright (C) 2006-2024 Broadcom. All Rights Reserved. +# Broadcom Confidential. The term "Broadcom" refers to Broadcom Inc. +# and/or its subsidiaries. # # Author: Sankar Tanguturi +# Pengpeng Sun # # This file is part of cloud-init. See LICENSE file for license information. +import ipaddress import logging import os import re from cloudinit import net, subp, util -from cloudinit.net.network_state import ipv4_mask_to_net_prefix +from cloudinit.net.network_state import ( + ipv4_mask_to_net_prefix, + ipv6_mask_to_net_prefix, +) logger = logging.getLogger(__name__) -def gen_subnet(ip, netmask): - """ - Return the subnet for a given ip address and a netmask - @return (str): the subnet - @param ip: ip address - @param netmask: netmask - """ - ip_array = ip.split(".") - mask_array = netmask.split(".") - result = [] - for index in list(range(4)): - result.append(int(ip_array[index]) & int(mask_array[index])) - - return ".".join([str(x) for x in result]) - - class NicConfigurator: - def __init__(self, nics, use_system_devices=True): + def __init__( + self, nics, name_servers, dns_suffixes, use_system_devices=True + ): """ Initialize the Nic Configurator @param nics (list) an array of nics to configure @@ -41,9 +34,9 @@ def __init__(self, nics, use_system_devices=True): the specified nics. """ self.nics = nics + self.name_servers = name_servers + self.dns_suffixes = dns_suffixes self.mac2Name = {} - self.ipv4PrimaryGateway = None - self.ipv6PrimaryGateway = None if use_system_devices: self.find_devices() @@ -87,10 +80,10 @@ def find_devices(self): name = section.split(":", 1)[0] self.mac2Name[mac] = name - def gen_one_nic(self, nic): + def gen_one_nic_v2(self, nic): """ - Return the config list needed to configure a nic - @return (list): the subnets and routes list to configure the nic + Return the config dict needed to configure a nic + @return (dict): the config dict to configure the nic @param nic (NicBase): the nic to configure """ mac = nic.mac.lower() @@ -98,137 +91,139 @@ def gen_one_nic(self, nic): if not name: raise ValueError("No known device has MACADDR: %s" % nic.mac) - nics_cfg_list = [] - - cfg = {"type": "physical", "name": name, "mac_address": mac} - - subnet_list = [] - route_list = [] - - # Customize IPv4 - (subnets, routes) = self.gen_ipv4(name, nic) - subnet_list.extend(subnets) - route_list.extend(routes) - - # Customize IPv6 - (subnets, routes) = self.gen_ipv6(name, nic) - subnet_list.extend(subnets) - route_list.extend(routes) - - cfg.update({"subnets": subnet_list}) - - nics_cfg_list.append(cfg) - if route_list: - nics_cfg_list.extend(route_list) + nic_config_dict = {} + generators = [ + self.gen_match(mac), + self.gen_set_name(name), + self.gen_wakeonlan(nic), + self.gen_dhcp4(nic), + self.gen_dhcp6(nic), + self.gen_addresses(nic), + self.gen_routes(nic), + self.gen_nameservers(), + ] + for value in generators: + if value: + nic_config_dict.update(value) - return nics_cfg_list + return {name: nic_config_dict} - def gen_ipv4(self, name, nic): - """ - Return the set of subnets and routes needed to configure the - IPv4 settings of a nic - @return (set): the set of subnet and routes to configure the gateways - @param name (str): subnet and route list for the nic - @param nic (NicBase): the nic to configure - """ + def gen_match(self, mac): + return {"match": {"macaddress": mac}} - subnet = {} - route_list = [] + def gen_set_name(self, name): + return {"set-name": name} - if nic.onboot: - subnet.update({"control": "auto"}) + def gen_wakeonlan(self, nic): + return {"wakeonlan": nic.onboot} + def gen_dhcp4(self, nic): + dhcp4 = {} bootproto = nic.bootProto.lower() if nic.ipv4_mode.lower() == "disabled": bootproto = "manual" - if bootproto != "static": - subnet.update({"type": "dhcp"}) - return ([subnet], route_list) + dhcp4.update({"dhcp4": True}) + # dhcp4-overrides + if self.name_servers or self.dns_suffixes: + dhcp4.update({"dhcp4-overrides": {"use-dns": False}}) else: - subnet.update({"type": "static"}) + dhcp4.update({"dhcp4": False}) + return dhcp4 - # Static Ipv4 - addrs = nic.staticIpv4 - if not addrs: - return ([subnet], route_list) - - v4 = addrs[0] - if v4.ip: - subnet.update({"address": v4.ip}) - if v4.netmask: - subnet.update({"netmask": v4.netmask}) - - # Add the primary gateway - if nic.primary and v4.gateways: - self.ipv4PrimaryGateway = v4.gateways[0] - subnet.update({"gateway": self.ipv4PrimaryGateway}) - return ([subnet], route_list) - - # Add routes if there is no primary nic - if not self._primaryNic and v4.gateways: - subnet.update( - {"routes": self.gen_ipv4_route(nic, v4.gateways, v4.netmask)} - ) + def gen_dhcp6(self, nic): + dhcp6 = {} + if nic.staticIpv6: + dhcp6.update({"dhcp6": False}) + # TODO: nic shall explicitly tell it's DHCP6 + # TODO: set dhcp6-overrides + return dhcp6 - return ([subnet], route_list) + def gen_addresses(self, nic): + address_list = [] + v4_cidr = 32 - def gen_ipv4_route(self, nic, gateways, netmask): - """ - Return the routes list needed to configure additional Ipv4 route - @return (list): the route list to configure the gateways - @param nic (NicBase): the nic to configure - @param gateways (str list): the list of gateways - """ - route_list = [] - - cidr = ipv4_mask_to_net_prefix(netmask) - - for gateway in gateways: - destination = "%s/%d" % (gen_subnet(gateway, netmask), cidr) - route_list.append( - { - "destination": destination, - "type": "route", - "gateway": gateway, - "metric": 10000, - } - ) - - return route_list - - def gen_ipv6(self, name, nic): - """ - Return the set of subnets and routes needed to configure the - gateways for a nic - @return (set): the set of subnets and routes to configure the gateways - @param name (str): name of the nic - @param nic (NicBase): the nic to configure - """ - - if not nic.staticIpv6: - return ([], []) - - subnet_list = [] + # Static Ipv4 + v4_addrs = nic.staticIpv4 + if v4_addrs: + v4 = v4_addrs[0] + if v4.netmask: + v4_cidr = ipv4_mask_to_net_prefix(v4.netmask) + if v4.ip: + address_list.append(f"{v4.ip}/{v4_cidr}") # Static Ipv6 - addrs = nic.staticIpv6 - - for addr in addrs: - subnet = { - "type": "static6", - "address": addr.ip, - "netmask": addr.netmask, - } - subnet_list.append(subnet) - - # TODO: Add the primary gateway + v6_addrs = nic.staticIpv6 + if v6_addrs: + for v6 in v6_addrs: + v6_cidr = ipv6_mask_to_net_prefix(v6.netmask) + address_list.append(f"{v6.ip}/{v6_cidr}") + + if address_list: + return {"addresses": address_list} + else: + return {} + def gen_routes(self, nic): route_list = [] - # TODO: Add routes if there is no primary nic - # if not self._primaryNic: - # route_list.extend(self._genIpv6Route(name, nic, addrs)) + v4_cidr = 32 + + # Ipv4 routes + v4_addrs = nic.staticIpv4 + if v4_addrs: + v4 = v4_addrs[0] + # Add the ipv4 default route + if nic.primary and v4.gateways: + route_list.append({"to": "0.0.0.0/0", "via": v4.gateways[0]}) + # Add ipv4 static routes if there is no primary nic + if not self._primaryNic and v4.gateways: + if v4.netmask: + v4_cidr = ipv4_mask_to_net_prefix(v4.netmask) + for gateway in v4.gateways: + v4_subnet = ipaddress.IPv4Network( + f"{gateway}/{v4_cidr}", strict=False + ) + route_list.append({"to": f"{v4_subnet}", "via": gateway}) + # Ipv6 routes + v6_addrs = nic.staticIpv6 + if v6_addrs: + for v6 in v6_addrs: + v6_cidr = ipv6_mask_to_net_prefix(v6.netmask) + # Add the ipv6 default route + if nic.primary and v6.gateway: + route_list.append({"to": "::/0", "via": v6.gateway}) + # Add ipv6 static routes if there is no primary nic + if not self._primaryNic and v6.gateway: + v6_subnet = ipaddress.IPv6Network( + f"{v6.gateway}/{v6_cidr}", strict=False + ) + route_list.append( + {"to": f"{v6_subnet}", "via": v6.gateway} + ) - return (subnet_list, route_list) + if route_list: + return {"routes": route_list} + else: + return {} + + def gen_nameservers(self): + nameservers_dict = {} + search_list = [] + addresses_list = [] + if self.dns_suffixes: + for dns_suffix in self.dns_suffixes: + search_list.append(dns_suffix) + if self.name_servers: + for name_server in self.name_servers: + addresses_list.append(name_server) + if search_list: + nameservers_dict.update({"search": search_list}) + if addresses_list: + nameservers_dict.update({"addresses": addresses_list}) + + if nameservers_dict: + return {"nameservers": nameservers_dict} + else: + return {} def generate(self, configure=False, osfamily=None): """Return the config elements that are needed to configure the nics""" @@ -236,12 +231,12 @@ def generate(self, configure=False, osfamily=None): logger.info("Configuring the interfaces file") self.configure(osfamily) - nics_cfg_list = [] + ethernets_dict = {} for nic in self.nics: - nics_cfg_list.extend(self.gen_one_nic(nic)) + ethernets_dict.update(self.gen_one_nic_v2(nic)) - return nics_cfg_list + return ethernets_dict def clear_dhcp(self): logger.info("Clearing DHCP leases") diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index e957621d..b77f526e 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -1,8 +1,10 @@ # Copyright (C) 2016 Canonical Ltd. -# Copyright (C) 2016-2023 VMware Inc. +# Copyright (C) 2006-2024 Broadcom. All Rights Reserved. +# Broadcom Confidential. The term "Broadcom" refers to Broadcom Inc. +# and/or its subsidiaries. # # Author: Sankar Tanguturi -# Pengpeng Sun +# Pengpeng Sun # # This file is part of cloud-init. See LICENSE file for license information. @@ -325,23 +327,19 @@ def get_non_network_data_from_vmware_cust_cfg(cust_cfg): def get_network_data_from_vmware_cust_cfg( cust_cfg, use_system_devices=True, configure=False, osfamily=None ): - nicConfigurator = NicConfigurator(cust_cfg.nics, use_system_devices) - nics_cfg_list = nicConfigurator.generate(configure, osfamily) - - return get_v1_network_config( - nics_cfg_list, cust_cfg.name_servers, cust_cfg.dns_suffixes + nicConfigurator = NicConfigurator( + cust_cfg.nics, + cust_cfg.name_servers, + cust_cfg.dns_suffixes, + use_system_devices, ) + ethernets_dict = nicConfigurator.generate(configure, osfamily) + return gen_v2_network_config(ethernets_dict) -def get_v1_network_config(nics_cfg_list=None, nameservers=None, search=None): - config_list = nics_cfg_list - - if nameservers or search: - config_list.append( - {"type": "nameserver", "address": nameservers, "search": search} - ) - return {"version": 1, "config": config_list} +def gen_v2_network_config(ethernets_dict): + return {"version": 2, "ethernets": ethernets_dict} def connect_nics(cust_cfg_dir): diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index af504d1b..9d21f23e 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -78,7 +78,7 @@ def get_interface_list(): except Exception as e: LOG.error("find_candidate_nics script exception: %s", e) - if len(ifaces) == 0: + if not ifaces: for iface in net.find_candidate_nics(): # Skip dummy if "dummy" in iface: diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 22cedd0e..d7224b4f 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -312,7 +312,7 @@ def check_permissions(username, current_path, full_path, is_file, strictmodes): # 3. no write permission (w) is given to group and world users (022) # Group and world user can still have +rx. - if strictmodes and parent_permission & 0o022 != 0: + if strictmodes and (parent_permission & 0o022): LOG.debug( "Path %s in %s must not give write" "permission to group or world users. Ignoring key.", diff --git a/cloudinit/templater.py b/cloudinit/templater.py index d43ed7b1..b33f0c95 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -40,8 +40,6 @@ JUndefined = object LOG = logging.getLogger(__name__) -TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I) -BASIC_MATCHER = re.compile(r"\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)") MISSING_JINJA_PREFIX = "CI_MISSING_JINJA_VAR/" @@ -140,7 +138,9 @@ def replacer(match): ) return str(selected_params[key]) - return BASIC_MATCHER.sub(replacer, content) + return re.sub( + r"\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)", replacer, content + ) def detect_template(text): @@ -171,7 +171,7 @@ def jinja_render(content, params): else: ident = text rest = "" - type_match = TYPE_MATCHER.match(ident) + type_match = re.match(r"##\s*template:(.*)", ident, re.I) if not type_match: return ("basic", basic_render, text) else: diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 37de5186..ee7f5956 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -432,7 +432,7 @@ def _handle_error( return None if error.code and error.code == 503: LOG.warning( - "Ec2 IMDS endpoint returned a 503 error. " + "Endpoint returned a 503 error. " "HTTP endpoint is overloaded. Retrying." ) if error.headers: @@ -697,7 +697,7 @@ def dual_stack( # No success, return the last exception but log them all for # debugging if last_exception: - LOG.warning( + LOG.debug( "Exception(s) %s during request to " "%s, raising last exception", exceptions, @@ -710,7 +710,7 @@ def dual_stack( # when max_wait expires, log but don't throw (retries happen) except TimeoutError: - LOG.warning( + LOG.debug( "Timed out waiting for addresses: %s, " "exception(s) raised while waiting: %s", " ".join(addresses), @@ -1097,7 +1097,7 @@ def readurl(self, *args, **kwargs): return self._wrapped(readurl, args, kwargs) def _exception_cb(self, extra_exception_cb, exception): - ret = None + ret = True try: if extra_exception_cb: ret = extra_exception_cb(exception) diff --git a/cloudinit/util.py b/cloudinit/util.py index 003afe93..725cd7d0 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -34,7 +34,7 @@ import sys import time from base64 import b64decode -from collections import deque, namedtuple +from collections import deque from contextlib import contextmanager, suppress from errno import ENOENT from functools import lru_cache @@ -50,6 +50,7 @@ Generator, List, Mapping, + NamedTuple, Optional, Sequence, Union, @@ -124,7 +125,7 @@ def lsb_release(): if fname in fmap: data[fmap[fname]] = val.strip() missing = [k for k in fmap.values() if k not in data] - if len(missing): + if missing: LOG.warning( "Missing fields in lsb_release --all output: %s", ",".join(missing), @@ -1196,10 +1197,10 @@ def sanitize_fqdn(fqdn): return output -HostnameFqdnInfo = namedtuple( - "HostnameFqdnInfo", - ["hostname", "fqdn", "is_default"], -) +class HostnameFqdnInfo(NamedTuple): + hostname: str + fqdn: str + is_default: bool def get_hostname_fqdn(cfg, cloud, metadata_only=False): @@ -1207,7 +1208,7 @@ def get_hostname_fqdn(cfg, cloud, metadata_only=False): @param cfg: Dictionary of merged user-data configuration (from init.cfg). @param cloud: Cloud instance from init.cloudify(). - @param metadata_only: Boolean, set True to only query cloud meta-data, + @param metadata_only: Boolean, set True to only query meta-data, returning None if not present in meta-data. @return: a namedtuple of , , (str, str, bool). @@ -1222,7 +1223,7 @@ def get_hostname_fqdn(cfg, cloud, metadata_only=False): is_default = False if "fqdn" in cfg: # user specified a fqdn. Default hostname then is based off that - fqdn = cfg["fqdn"] + fqdn = str(cfg["fqdn"]) hostname = get_cfg_option_str(cfg, "hostname", fqdn.split(".")[0]) else: if "hostname" in cfg and cfg["hostname"].find(".") > 0: @@ -1636,11 +1637,11 @@ def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None): data = in_fh.read(chunk_size) if len(data) == 0: break - else: - out_fh.write(data) - bytes_piped += len(data) - if chunk_cb: - chunk_cb(bytes_piped) + out_fh.write(data) + bytes_piped += len(data) + if chunk_cb: + chunk_cb(bytes_piped) + out_fh.flush() return bytes_piped @@ -3000,7 +3001,7 @@ def wait_for_files(flist, maxwait, naplen=0.5, log_pre=""): waited = 0 while True: need -= set([f for f in need if os.path.exists(f)]) - if len(need) == 0: + if not need: LOG.debug( "%sAll files appeared after %s seconds: %s", log_pre, diff --git a/cloudinit/version.py b/cloudinit/version.py index cccccfb0..eb772d8f 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -4,7 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -__VERSION__ = "24.4.1" +__VERSION__ = "25.1.2" _PACKAGED_VERSION = "@@PACKAGED_VERSION@@" FEATURES = [ diff --git a/debian/changelog b/debian/changelog index e1b2892e..4777f227 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,32 @@ +cloud-init (25.1.2-0ubuntu0~24.04.1) noble; urgency=medium + + * Upstream snapshot based on 25.1.2. (LP: #2104165). + List of changes from upstream can be found at + https://raw.githubusercontent.com/canonical/cloud-init/25.1.2/ChangeLog + + -- James Falcon Mon, 19 May 2025 15:00:58 -0500 + +cloud-init (25.1.1-0ubuntu1~24.04.1) noble; urgency=medium + + * Drop cpicks which are now upstream: + - cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995 + - cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting + - d/p/cpick-c60771d8-test-pytestify-test_url_helper.py + - d/p/cpick-8810a2dc-test-Remove-CiTestCase-from-test_url_helper.py + - d/p/cpick-582f16c1-test-add-OauthUrlHelper-tests + - d/p/cpick-9311e066-fix-Update-OauthUrlHelper-to-use-readurl-exception_cb + * refresh patches + - d/p/deprecation-version-boundary.patch + - d/p/grub-dpkg-support.patch + - d/p/no-nocloud-network.patch + - d/p/no-single-process.patch + * sort hunks within all patches (--sort on quilt refresh) + * Upstream snapshot based on 25.1.1. + List of changes from upstream can be found at + https://raw.githubusercontent.com/canonical/cloud-init/25.1.1/ChangeLog + + -- Chad Smith Tue, 25 Mar 2025 11:02:28 -0600 + cloud-init (24.4.1-0ubuntu0~24.04.3) noble; urgency=medium * d/control: Fix how cloud-init-base overwrites cloud-init files. @@ -5,7 +34,7 @@ cloud-init (24.4.1-0ubuntu0~24.04.3) noble; urgency=medium -- James Falcon Wed, 02 Apr 2025 10:09:15 -0500 -cloud-init (24.4.1-0ubuntu0~24.04.2) noble; urgency=medium +cloud-init (24.4.1-0ubuntu0~20.04.2) noble; urgency=medium * cherry-pick fixes for MAAS traceback (LP: #2100963) - cherry-pick c60771d8: test: pytestify test_url_helper.py @@ -15,7 +44,7 @@ cloud-init (24.4.1-0ubuntu0~24.04.2) noble; urgency=medium - cherry-pick 9311e066: fix: Update OauthUrlHelper to use readurl exception_cb - -- James Falcon Thu, 13 Mar 2025 14:37:50 -0500 + -- James Falcon Thu, 13 Mar 2025 11:28:57 -0500 cloud-init (24.4.1-0ubuntu0~24.04.1) noble; urgency=medium diff --git a/debian/cloud-init.templates b/debian/cloud-init.templates index d192fdcf..903e486e 100644 --- a/debian/cloud-init.templates +++ b/debian/cloud-init.templates @@ -1,8 +1,8 @@ Template: cloud-init/datasources Type: multiselect -Default: NoCloud, ConfigDrive, OpenNebula, DigitalOcean, Azure, AltCloud, OVF, MAAS, GCE, OpenStack, CloudSigma, SmartOS, Bigstep, Scaleway, AliYun, Ec2, CloudStack, Hetzner, IBMCloud, Oracle, Exoscale, RbxCloud, UpCloud, VMware, Vultr, LXD, NWCS, Akamai, WSL, None -Choices-C: NoCloud, ConfigDrive, OpenNebula, DigitalOcean, Azure, AltCloud, OVF, MAAS, GCE, OpenStack, CloudSigma, SmartOS, Bigstep, Scaleway, AliYun, Ec2, CloudStack, Hetzner, IBMCloud, Oracle, Exoscale, RbxCloud, UpCloud, VMware, Vultr, LXD, NWCS, Akamai, WSL, None -__Choices: NoCloud: Reads info from /var/lib/cloud/seed only, ConfigDrive: Reads data from Openstack Config Drive, OpenNebula: read from OpenNebula context disk, DigitalOcean: reads data from Droplet datasource, Azure: read from MS Azure cdrom. Requires walinux-agent, AltCloud: config disks for RHEVm and vSphere, OVF: Reads data from OVF Transports, MAAS: Reads data from Ubuntu MAAS, GCE: google compute metadata service, OpenStack: native openstack metadata service, CloudSigma: metadata over serial for cloudsigma.com, SmartOS: Read from SmartOS metadata service, Bigstep: Bigstep metadata service, Scaleway: Scaleway metadata service, AliYun: Alibaba metadata service, Ec2: reads data from EC2 Metadata service, CloudStack: Read from CloudStack metadata service, Hetzner: Hetzner Cloud, IBMCloud: IBM Cloud. Previously softlayer or bluemix., Oracle: Oracle Compute Infrastructure, Exoscale: Exoscale, RbxCloud: HyperOne and Rootbox platforms, UpCloud: UpCloud, VMware: reads data from guestinfo table or env vars, Vultr: Vultr Cloud, LXD: Reads /dev/lxd/sock representation of instance data, NWCS: NWCS, Akamai: Akamai and Linode platforms, WSL: Windows Subsystem for Linux, None: Failsafe datasource +Default: NoCloud, ConfigDrive, OpenNebula, DigitalOcean, Azure, AltCloud, VMware, OVF, MAAS, GCE, OpenStack, CloudSigma, SmartOS, Bigstep, Scaleway, AliYun, Ec2, CloudStack, Hetzner, IBMCloud, Oracle, Exoscale, RbxCloud, UpCloud, Vultr, LXD, NWCS, Akamai, WSL, CloudCIX, None +Choices-C: NoCloud, ConfigDrive, OpenNebula, DigitalOcean, Azure, AltCloud, VMware, OVF, MAAS, GCE, OpenStack, CloudSigma, SmartOS, Bigstep, Scaleway, AliYun, Ec2, CloudStack, Hetzner, IBMCloud, Oracle, Exoscale, RbxCloud, UpCloud, Vultr, LXD, NWCS, Akamai, WSL, CloudCIX, None +__Choices: NoCloud: Reads info from /var/lib/cloud/seed only, ConfigDrive: Reads data from Openstack Config Drive, OpenNebula: read from OpenNebula context disk, DigitalOcean: reads data from Droplet datasource, Azure: read from MS Azure cdrom. Requires walinux-agent, AltCloud: config disks for RHEVm and vSphere, VMware: reads data from guestinfo table or env vars, OVF: Reads data from OVF Transports, MAAS: Reads data from Ubuntu MAAS, GCE: google compute metadata service, OpenStack: native openstack metadata service, CloudSigma: metadata over serial for cloudsigma.com, SmartOS: Read from SmartOS metadata service, Bigstep: Bigstep metadata service, Scaleway: Scaleway metadata service, AliYun: Alibaba metadata service, Ec2: reads data from EC2 Metadata service, CloudStack: Read from CloudStack metadata service, Hetzner: Hetzner Cloud, IBMCloud: IBM Cloud. Previously softlayer or bluemix., Oracle: Oracle Compute Infrastructure, Exoscale: Exoscale, RbxCloud: HyperOne and Rootbox platforms, UpCloud: UpCloud, Vultr: Vultr Cloud, LXD: Reads /dev/lxd/sock representation of instance data, NWCS: NWCS, Akamai: Akamai and Linode platforms, WSL: Windows Subsystem for Linux, CloudCIX: Reads from CloudCIX metadata service, None: Failsafe datasource _Description: Which data sources should be searched? Cloud-init supports searching different "Data Sources" for information that it uses to configure a cloud instance. diff --git a/debian/patches/cpick-582f16c1-test-add-OauthUrlHelper-tests b/debian/patches/cpick-582f16c1-test-add-OauthUrlHelper-tests deleted file mode 100644 index 50d70e35..00000000 --- a/debian/patches/cpick-582f16c1-test-add-OauthUrlHelper-tests +++ /dev/null @@ -1,156 +0,0 @@ -From 582f16c143b1d071c78f259bb978296b1439e186 Mon Sep 17 00:00:00 2001 -From: James Falcon -Date: Mon, 3 Mar 2025 13:25:24 -0600 -Subject: [PATCH] test: add OauthUrlHelper tests -Bug: https://github.com/canonical/cloud-init/issues/6065 -Bug-Ubuntu: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/2100963 - ---- - tests/unittests/test_url_helper.py | 102 ++++++++++++++++++++++++++++- - 1 file changed, 99 insertions(+), 3 deletions(-) - ---- a/tests/unittests/test_url_helper.py -+++ b/tests/unittests/test_url_helper.py -@@ -6,13 +6,14 @@ import pathlib - from functools import partial - from threading import Event - from time import process_time -+from unittest import mock - from unittest.mock import ANY, call - - import pytest - import requests - import responses - --from cloudinit import util, version -+from cloudinit import url_helper, util, version - from cloudinit.url_helper import ( - REDACTED, - UrlError, -@@ -24,7 +25,6 @@ from cloudinit.url_helper import ( - readurl, - wait_for_url, - ) --from tests.unittests.helpers import mock, skipIf - - try: - import oauthlib -@@ -38,6 +38,14 @@ except ImportError: - M_PATH = "cloudinit.url_helper." - - -+def exception_cb(exception): -+ return True -+ -+ -+def headers_cb(url): -+ return {"cb_key": "cb_value"} -+ -+ - class TestOAuthHeaders: - def test_oauth_headers_raises_not_implemented_when_oathlib_missing(self): - """oauth_headers raises a NotImplemented error when oauth absent.""" -@@ -46,7 +54,9 @@ class TestOAuthHeaders: - oauth_headers(1, 2, 3, 4, 5) - assert "oauth support is not available" == str(context_manager.value) - -- @skipIf(_missing_oauthlib_dep, "No python-oauthlib dependency") -+ @pytest.mark.skipif( -+ _missing_oauthlib_dep, reason="No python-oauthlib dependency" -+ ) - @mock.patch("oauthlib.oauth1.Client") - def test_oauth_headers_calls_oathlibclient_when_available(self, m_client): - """oauth_headers calls oaut1.hClient.sign with the provided url.""" -@@ -68,6 +78,92 @@ class TestOAuthHeaders: - assert "url" == return_value - - -+class TestOauthUrlHelper: -+ @responses.activate -+ def test_wrapped_readurl(self): -+ """Test the wrapped readurl happy path.""" -+ oauth_helper = url_helper.OauthUrlHelper() -+ url = "http://myhost/path" -+ data = b"This is my url content" -+ responses.add(responses.GET, url, data) -+ assert oauth_helper.readurl(url).contents == data -+ -+ @responses.activate -+ def test_default_exception_cb(self, tmp_path, caplog): -+ """Test that the default exception_cb is used.""" -+ skew_file = tmp_path / "skew.json" -+ oauth_helper = url_helper.OauthUrlHelper(skew_data_file=skew_file) -+ url = "http://myhost/path" -+ data = b"This is my url content" -+ response = requests.Response() -+ response.status_code = 401 -+ response._content = data -+ response.headers["date"] = "Wed, 21 Oct 2015 07:28:00 GMT" -+ responses.add_callback( -+ responses.GET, -+ url, -+ callback=lambda _: requests.HTTPError(response=response), -+ ) -+ with pytest.raises(UrlError) as e: -+ oauth_helper.readurl(url) -+ assert e.value.code == 401 -+ assert "myhost" in skew_file.read_text() -+ -+ @responses.activate -+ def test_custom_exception_cb(self): -+ """Test that a custom exception_cb is used.""" -+ oauth_helper = url_helper.OauthUrlHelper() -+ url = "http://myhost/path" -+ data = b"This is my url content" -+ responses.add(responses.GET, url, data, status=401) -+ exception_cb = mock.Mock(return_value=True) -+ -+ with pytest.raises(UrlError): -+ oauth_helper.readurl(url, exception_cb=exception_cb) -+ exception_cb.assert_called_once() -+ assert isinstance(exception_cb.call_args[0][0], UrlError) -+ assert exception_cb.call_args[0][0].code == 401 -+ -+ @responses.activate -+ def test_default_headers_cb(self, mocker): -+ """Test that the default headers_cb is used.""" -+ m_headers = mocker.patch( -+ "cloudinit.url_helper.oauth_headers", -+ return_value={"key1": "value1"}, -+ ) -+ mocker.patch("time.time", return_value=5) -+ oauth_helper = url_helper.OauthUrlHelper() -+ oauth_helper.skew_data = {"myhost": 125} -+ oauth_helper._do_oauth = True -+ url = "http://myhost/path" -+ data = b"This is my url content" -+ responses.add(responses.GET, url, data) -+ response = oauth_helper.readurl(url) -+ request_headers = response._response.request.headers -+ assert "key1" in request_headers -+ assert request_headers["key1"] == "value1" -+ assert m_headers.call_args[1]["timestamp"] == 130 -+ -+ @responses.activate -+ def test_custom_headers_cb(self, mocker): -+ """Test that a custom headers_cb is used.""" -+ mocker.patch( -+ "cloudinit.url_helper.oauth_headers", -+ return_value={"key1": "value1"}, -+ ) -+ oauth_helper = url_helper.OauthUrlHelper() -+ oauth_helper._do_oauth = True -+ url = "http://myhost/path" -+ data = b"This is my url content" -+ responses.add(responses.GET, url, data) -+ response = oauth_helper.readurl(url, headers_cb=headers_cb) -+ request_headers = response._response.request.headers -+ assert "key1" in request_headers -+ assert "cb_key" in request_headers -+ assert request_headers["key1"] == "value1" -+ assert request_headers["cb_key"] == "cb_value" -+ -+ - class TestReadFileOrUrl: - def test_read_file_or_url_str_from_file(self, tmp_path: pathlib.Path): - """Test that str(result.contents) on file is text version of contents. diff --git a/debian/patches/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting b/debian/patches/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting deleted file mode 100644 index 17d3fe3d..00000000 --- a/debian/patches/cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting +++ /dev/null @@ -1,179 +0,0 @@ -From 84806336bdd6655b048a0c0d8190d08a5aa15a15 Mon Sep 17 00:00:00 2001 -From: James Falcon -Date: Wed, 15 Jan 2025 15:36:17 -0600 -Subject: [PATCH] chore: Add feature flag for manual network waiting - -Controls the behavior added in e30549e for easier downstream patching ---- - cloudinit/cmd/main.py | 35 ++++++++++--------- - cloudinit/features.py | 11 ++++++ - .../datasources/test_nocloud.py | 7 ++-- - .../modules/test_boothook.py | 3 +- - tests/unittests/cmd/test_main.py | 4 +-- - tests/unittests/test_data.py | 1 + - tests/unittests/test_features.py | 2 ++ - 7 files changed, 42 insertions(+), 21 deletions(-) - ---- a/cloudinit/cmd/main.py -+++ b/cloudinit/cmd/main.py -@@ -21,7 +21,7 @@ import logging - import yaml - from typing import Optional, Tuple, Callable, Union - --from cloudinit import netinfo -+from cloudinit import features, netinfo - from cloudinit import signal_handler - from cloudinit import sources - from cloudinit import socket -@@ -486,7 +486,9 @@ def main_init(name, args): - mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK - - if mode == sources.DSMODE_NETWORK: -- if not os.path.exists(init.paths.get_runpath(".skip-network")): -+ if features.MANUAL_NETWORK_WAIT and not os.path.exists( -+ init.paths.get_runpath(".skip-network") -+ ): - LOG.debug("Will wait for network connectivity before continuing") - init.distro.wait_for_network() - existing = "trust" -@@ -560,20 +562,21 @@ def main_init(name, args): - init.apply_network_config(bring_up=bring_up_interfaces) - - if mode == sources.DSMODE_LOCAL: -- should_wait, reason = _should_wait_on_network(init.datasource) -- if should_wait: -- LOG.debug( -- "Network connectivity determined necessary for " -- "cloud-init's network stage. Reason: %s", -- reason, -- ) -- else: -- LOG.debug( -- "Network connectivity determined unnecessary for " -- "cloud-init's network stage. Reason: %s", -- reason, -- ) -- util.write_file(init.paths.get_runpath(".skip-network"), "") -+ if features.MANUAL_NETWORK_WAIT: -+ should_wait, reason = _should_wait_on_network(init.datasource) -+ if should_wait: -+ LOG.debug( -+ "Network connectivity determined necessary for " -+ "cloud-init's network stage. Reason: %s", -+ reason, -+ ) -+ else: -+ LOG.debug( -+ "Network connectivity determined unnecessary for " -+ "cloud-init's network stage. Reason: %s", -+ reason, -+ ) -+ util.write_file(init.paths.get_runpath(".skip-network"), "") - - if init.datasource.dsmode != mode: - LOG.debug( ---- a/cloudinit/features.py -+++ b/cloudinit/features.py -@@ -87,6 +87,17 @@ On Debian and Ubuntu systems, cc_apt_con - to write /etc/apt/sources.list directly. - """ - -+MANUAL_NETWORK_WAIT = True -+""" -+On Ubuntu systems, cloud-init-network.service will start immediately after -+cloud-init-local.service and manually wait for network online when necessary. -+If False, rely on systemd ordering to ensure network is available before -+starting cloud-init-network.service. -+ -+Note that in addition to this flag, downstream patches are also likely needed -+to modify the systemd unit files. -+""" -+ - DEPRECATION_INFO_BOUNDARY = "24.1" - """ - DEPRECATION_INFO_BOUNDARY is used by distros to configure at which upstream ---- a/tests/integration_tests/datasources/test_nocloud.py -+++ b/tests/integration_tests/datasources/test_nocloud.py -@@ -5,7 +5,7 @@ from textwrap import dedent - import pytest - from pycloudlib.lxd.instance import LXDInstance - --from cloudinit import lifecycle -+from cloudinit import features, lifecycle - from cloudinit.subp import subp - from tests.integration_tests.instances import IntegrationInstance - from tests.integration_tests.integration_settings import PLATFORM -@@ -100,7 +100,10 @@ def test_nocloud_seedfrom_vendordata(cli - client.restart() - assert client.execute("cloud-init status").ok - assert "seeded_vendordata_test_file" in client.execute("ls /var/tmp") -- assert network_wait_logged(client.execute("cat /var/log/cloud-init.log")) -+ assert ( -+ network_wait_logged(client.execute("cat /var/log/cloud-init.log")) -+ == features.MANUAL_NETWORK_WAIT -+ ) - - - SMBIOS_USERDATA = """\ ---- a/tests/integration_tests/modules/test_boothook.py -+++ b/tests/integration_tests/modules/test_boothook.py -@@ -3,6 +3,7 @@ import re - - import pytest - -+from cloudinit import features - from tests.integration_tests.instances import IntegrationInstance - from tests.integration_tests.util import ( - network_wait_logged, -@@ -57,4 +58,4 @@ class TestBoothook: - client = class_client - assert network_wait_logged( - client.read_from_file("/var/log/cloud-init.log") -- ) -+ ) == features.MANUAL_NETWORK_WAIT ---- a/tests/unittests/cmd/test_main.py -+++ b/tests/unittests/cmd/test_main.py -@@ -9,7 +9,7 @@ from unittest import mock - - import pytest - --from cloudinit import safeyaml, util -+from cloudinit import features, safeyaml, util - from cloudinit.cmd import main - from cloudinit.util import ensure_dir, load_text_file, write_file - -@@ -363,7 +363,7 @@ class TestMain: - skip_log_setup=False, - ) - main.main_init("init", cmdargs) -- if expected_add_wait: -+ if features.MANUAL_NETWORK_WAIT and expected_add_wait: - m_nm.assert_called_once() - m_subp.assert_called_with( - ["systemctl", "start", "systemd-networkd-wait-online.service"] ---- a/tests/unittests/test_data.py -+++ b/tests/unittests/test_data.py -@@ -516,6 +516,7 @@ c: 4 - "DEPRECATION_INFO_BOUNDARY": "devel", - "NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH": False, - "APT_DEB822_SOURCE_LIST_FILE": True, -+ "MANUAL_NETWORK_WAIT": True, - }, - "system_info": { - "default_user": {"name": "ubuntu"}, ---- a/tests/unittests/test_features.py -+++ b/tests/unittests/test_features.py -@@ -22,6 +22,7 @@ class TestGetFeatures: - DEPRECATION_INFO_BOUNDARY="devel", - NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH=False, - APT_DEB822_SOURCE_LIST_FILE=True, -+ MANUAL_NETWORK_WAIT=False, - ): - assert { - "ERROR_ON_USER_DATA_FAILURE": True, -@@ -31,4 +32,5 @@ class TestGetFeatures: - "NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH": False, - "APT_DEB822_SOURCE_LIST_FILE": True, - "DEPRECATION_INFO_BOUNDARY": "devel", -+ "MANUAL_NETWORK_WAIT": False, - } == features.get_features() diff --git a/debian/patches/cpick-8810a2dc-test-Remove-CiTestCase-from-test_url_helper.py b/debian/patches/cpick-8810a2dc-test-Remove-CiTestCase-from-test_url_helper.py deleted file mode 100644 index 76bf6560..00000000 --- a/debian/patches/cpick-8810a2dc-test-Remove-CiTestCase-from-test_url_helper.py +++ /dev/null @@ -1,100 +0,0 @@ -From 8810a2dccf8502549f2498a96ad7ff379fa93b87 Mon Sep 17 00:00:00 2001 -From: James Falcon -Date: Mon, 3 Mar 2025 08:40:54 -0600 -Subject: [PATCH] test: Remove CiTestCase from test_url_helper.py -Bug: https://github.com/canonical/cloud-init/issues/6065 -Bug-Ubuntu: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/2100963 - ---- - tests/unittests/test_url_helper.py | 32 ++++++++++++++---------------- - 1 file changed, 15 insertions(+), 17 deletions(-) - ---- a/tests/unittests/test_url_helper.py -+++ b/tests/unittests/test_url_helper.py -@@ -2,6 +2,7 @@ - # pylint: disable=attribute-defined-outside-init - - import logging -+import pathlib - from functools import partial - from threading import Event - from time import process_time -@@ -23,7 +24,7 @@ from cloudinit.url_helper import ( - readurl, - wait_for_url, - ) --from tests.unittests.helpers import CiTestCase, mock, skipIf -+from tests.unittests.helpers import mock, skipIf - - try: - import oauthlib -@@ -37,7 +38,7 @@ except ImportError: - M_PATH = "cloudinit.url_helper." - - --class TestOAuthHeaders(CiTestCase): -+class TestOAuthHeaders: - def test_oauth_headers_raises_not_implemented_when_oathlib_missing(self): - """oauth_headers raises a NotImplemented error when oauth absent.""" - with mock.patch.dict("sys.modules", {"oauthlib": None}): -@@ -67,17 +68,14 @@ class TestOAuthHeaders(CiTestCase): - assert "url" == return_value - - --class TestReadFileOrUrl(CiTestCase): -- -- with_logs = True -- -- def test_read_file_or_url_str_from_file(self): -+class TestReadFileOrUrl: -+ def test_read_file_or_url_str_from_file(self, tmp_path: pathlib.Path): - """Test that str(result.contents) on file is text version of contents. - It should not be "b'data'", but just "'data'" """ -- tmpf = self.tmp_path("myfile1") -+ tmpf = tmp_path / "myfile1" - data = b"This is my file content\n" - util.write_file(tmpf, data, omode="wb") -- result = read_file_or_url("file://%s" % tmpf) -+ result = read_file_or_url(f"file://{tmpf}") - assert result.contents == data - assert str(result) == data.decode("utf-8") - -@@ -105,7 +103,9 @@ class TestReadFileOrUrl(CiTestCase): - assert str(result) == data.decode("utf-8") - - @responses.activate -- def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self): -+ def test_read_file_or_url_str_from_url_redacting_headers_from_logs( -+ self, caplog -+ ): - """Headers are redacted from logs but unredacted in requests.""" - url = "http://hostname/path" - headers = {"sensitive": "sekret", "server": "blah"} -@@ -118,12 +118,11 @@ class TestReadFileOrUrl(CiTestCase): - responses.add_callback(responses.GET, url, callback=_request_callback) - - read_file_or_url(url, headers=headers, headers_redact=["sensitive"]) -- logs = self.logs.getvalue() -- assert REDACTED in logs -- assert "sekret" not in logs -+ assert REDACTED in caplog.text -+ assert "sekret" not in caplog.text - - @responses.activate -- def test_read_file_or_url_str_from_url_redacts_noheaders(self): -+ def test_read_file_or_url_str_from_url_redacts_noheaders(self, caplog): - """When no headers_redact, header values are in logs and requests.""" - url = "http://hostname/path" - headers = {"sensitive": "sekret", "server": "blah"} -@@ -136,9 +135,8 @@ class TestReadFileOrUrl(CiTestCase): - responses.add_callback(responses.GET, url, callback=_request_callback) - - read_file_or_url(url, headers=headers) -- logs = self.logs.getvalue() -- assert REDACTED not in logs -- assert "sekret" in logs -+ assert REDACTED not in caplog.text -+ assert "sekret" in caplog.text - - def test_wb_read_url_defaults_honored_by_read_file_or_url_callers(self): - """Readurl param defaults used when unspecified by read_file_or_url diff --git a/debian/patches/cpick-9311e066-fix-Update-OauthUrlHelper-to-use-readurl-exception_cb b/debian/patches/cpick-9311e066-fix-Update-OauthUrlHelper-to-use-readurl-exception_cb deleted file mode 100644 index e4c83e84..00000000 --- a/debian/patches/cpick-9311e066-fix-Update-OauthUrlHelper-to-use-readurl-exception_cb +++ /dev/null @@ -1,41 +0,0 @@ -From 9311e066f1eafea68fc714183173d8a5cc197d73 Mon Sep 17 00:00:00 2001 -From: James Falcon -Date: Mon, 3 Mar 2025 13:27:50 -0600 -Subject: [PATCH] fix: Update OauthUrlHelper to use readurl exception_cb - signature - -Fixes GH-6065 -Bug: https://github.com/canonical/cloud-init/issues/6065 -Bug-Ubuntu: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/2100963 ---- - cloudinit/url_helper.py | 8 ++++---- - 1 file changed, 4 insertions(+), 4 deletions(-) - ---- a/cloudinit/url_helper.py -+++ b/cloudinit/url_helper.py -@@ -1035,7 +1035,7 @@ class OauthUrlHelper: - ) as fp: - fp.write(json.dumps(cur)) - -- def exception_cb(self, msg, exception): -+ def exception_cb(self, exception): - if not ( - isinstance(exception, UrlError) - and (exception.code == 403 or exception.code == 401) -@@ -1096,13 +1096,13 @@ class OauthUrlHelper: - def readurl(self, *args, **kwargs): - return self._wrapped(readurl, args, kwargs) - -- def _exception_cb(self, extra_exception_cb, msg, exception): -+ def _exception_cb(self, extra_exception_cb, exception): - ret = None - try: - if extra_exception_cb: -- ret = extra_exception_cb(msg, exception) -+ ret = extra_exception_cb(exception) - finally: -- self.exception_cb(msg, exception) -+ self.exception_cb(exception) - return ret - - def _headers_cb(self, extra_headers_cb, url): diff --git a/debian/patches/cpick-c60771d8-test-pytestify-test_url_helper.py b/debian/patches/cpick-c60771d8-test-pytestify-test_url_helper.py deleted file mode 100644 index 03fca450..00000000 --- a/debian/patches/cpick-c60771d8-test-pytestify-test_url_helper.py +++ /dev/null @@ -1,153 +0,0 @@ -From c60771d8ef005154bacd5beb740949a7a830aeb1 Mon Sep 17 00:00:00 2001 -From: James Falcon -Date: Mon, 3 Mar 2025 08:33:41 -0600 -Subject: [PATCH] test: pytestify test_url_helper.py -Bug: https://github.com/canonical/cloud-init/issues/6065 -Bug-Ubuntu: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/2100963 - ---- - tests/unittests/test_url_helper.py | 59 ++++++++++++++---------------- - 1 file changed, 27 insertions(+), 32 deletions(-) - ---- a/tests/unittests/test_url_helper.py -+++ b/tests/unittests/test_url_helper.py -@@ -41,11 +41,9 @@ class TestOAuthHeaders(CiTestCase): - def test_oauth_headers_raises_not_implemented_when_oathlib_missing(self): - """oauth_headers raises a NotImplemented error when oauth absent.""" - with mock.patch.dict("sys.modules", {"oauthlib": None}): -- with self.assertRaises(NotImplementedError) as context_manager: -+ with pytest.raises(NotImplementedError) as context_manager: - oauth_headers(1, 2, 3, 4, 5) -- self.assertEqual( -- "oauth support is not available", str(context_manager.exception) -- ) -+ assert "oauth support is not available" == str(context_manager.value) - - @skipIf(_missing_oauthlib_dep, "No python-oauthlib dependency") - @mock.patch("oauthlib.oauth1.Client") -@@ -66,7 +64,7 @@ class TestOAuthHeaders(CiTestCase): - "token_secret", - "consumer_secret", - ) -- self.assertEqual("url", return_value) -+ assert "url" == return_value - - - class TestReadFileOrUrl(CiTestCase): -@@ -80,8 +78,8 @@ class TestReadFileOrUrl(CiTestCase): - data = b"This is my file content\n" - util.write_file(tmpf, data, omode="wb") - result = read_file_or_url("file://%s" % tmpf) -- self.assertEqual(result.contents, data) -- self.assertEqual(str(result), data.decode("utf-8")) -+ assert result.contents == data -+ assert str(result) == data.decode("utf-8") - - @responses.activate - def test_read_file_or_url_str_from_url(self): -@@ -91,8 +89,8 @@ class TestReadFileOrUrl(CiTestCase): - data = b"This is my url content\n" - responses.add(responses.GET, url, data) - result = read_file_or_url(url) -- self.assertEqual(result.contents, data) -- self.assertEqual(str(result), data.decode("utf-8")) -+ assert result.contents == data -+ assert str(result) == data.decode("utf-8") - - @responses.activate - def test_read_file_or_url_str_from_url_streamed(self): -@@ -103,8 +101,8 @@ class TestReadFileOrUrl(CiTestCase): - responses.add(responses.GET, url, data) - result = read_file_or_url(url, stream=True) - assert isinstance(result, UrlResponse) -- self.assertEqual(result.contents, data) -- self.assertEqual(str(result), data.decode("utf-8")) -+ assert result.contents == data -+ assert str(result) == data.decode("utf-8") - - @responses.activate - def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self): -@@ -114,15 +112,15 @@ class TestReadFileOrUrl(CiTestCase): - - def _request_callback(request): - for k in headers.keys(): -- self.assertEqual(headers[k], request.headers[k]) -+ assert headers[k] == request.headers[k] - return (200, request.headers, "does_not_matter") - - responses.add_callback(responses.GET, url, callback=_request_callback) - - read_file_or_url(url, headers=headers, headers_redact=["sensitive"]) - logs = self.logs.getvalue() -- self.assertIn(REDACTED, logs) -- self.assertNotIn("sekret", logs) -+ assert REDACTED in logs -+ assert "sekret" not in logs - - @responses.activate - def test_read_file_or_url_str_from_url_redacts_noheaders(self): -@@ -132,15 +130,15 @@ class TestReadFileOrUrl(CiTestCase): - - def _request_callback(request): - for k in headers.keys(): -- self.assertEqual(headers[k], request.headers[k]) -+ assert headers[k] == request.headers[k] - return (200, request.headers, "does_not_matter") - - responses.add_callback(responses.GET, url, callback=_request_callback) - - read_file_or_url(url, headers=headers) - logs = self.logs.getvalue() -- self.assertNotIn(REDACTED, logs) -- self.assertIn("sekret", logs) -+ assert REDACTED not in logs -+ assert "sekret" in logs - - def test_wb_read_url_defaults_honored_by_read_file_or_url_callers(self): - """Readurl param defaults used when unspecified by read_file_or_url -@@ -161,19 +159,16 @@ class TestReadFileOrUrl(CiTestCase): - class FakeSession(requests.Session): - @classmethod - def request(cls, **kwargs): -- self.assertEqual( -- { -- "url": url, -- "allow_redirects": True, -- "method": "GET", -- "headers": { -- "User-Agent": "Cloud-Init/%s" -- % (version.version_string()) -- }, -- "stream": False, -+ assert { -+ "url": url, -+ "allow_redirects": True, -+ "method": "GET", -+ "headers": { -+ "User-Agent": "Cloud-Init/%s" -+ % (version.version_string()) - }, -- kwargs, -- ) -+ "stream": False, -+ } == kwargs - return m_response - - with mock.patch(M_PATH + "requests.Session") as m_session: -@@ -182,13 +177,13 @@ class TestReadFileOrUrl(CiTestCase): - FakeSession(), - ] - # assert no retries and check_status == True -- with self.assertRaises(UrlError) as context_manager: -+ with pytest.raises(UrlError) as context_manager: - response = read_file_or_url(url) -- self.assertEqual("broke", str(context_manager.exception)) -+ assert "broke" == str(context_manager.value) - # assert default headers, method, url and allow_redirects True - # Success on 2nd call with FakeSession - response = read_file_or_url(url) -- self.assertEqual(m_response, response._response) -+ assert m_response == response._response - - - class TestReadFileOrUrlParameters: diff --git a/debian/patches/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995 b/debian/patches/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995 deleted file mode 100644 index 8b3168e3..00000000 --- a/debian/patches/cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995 +++ /dev/null @@ -1,257 +0,0 @@ -From: Brett Holman -Last-Update: Mon, 3 Feb 2025 10:20:46 -0700 -Bug: https://github.com/canonical/cloud-init/issues/5971 -Bug-Ubuntu: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/2097319 -Subject: [PATCH] fix: retry AWS hotplug for async IMDS (#5995) - -Make tests more robust to temporary network failure. -Document hotplug limitations. - -Fixes GH-5373 ---- - cloudinit/cmd/devel/hotplug_hook.py | 15 +++++- - cloudinit/sources/DataSourceEc2.py | 4 +- - cloudinit/sources/__init__.py | 17 +++++++ - doc/module-docs/cc_install_hotplug/data.yaml | 5 ++ - .../integration_tests/modules/test_hotplug.py | 47 ++++++++++++++++--- - .../unittests/cmd/devel/test_hotplug_hook.py | 1 + - tools/hook-hotplug | 5 ++ - 7 files changed, 85 insertions(+), 9 deletions(-) - ---- a/cloudinit/cmd/devel/hotplug_hook.py -+++ b/cloudinit/cmd/devel/hotplug_hook.py -@@ -204,7 +204,7 @@ - return datasource - - --def handle_hotplug(hotplug_init: Init, devpath, subsystem, udevaction): -+def handle_hotplug(hotplug_init: Init, devpath, subsystem, udevaction) -> None: - datasource = initialize_datasource(hotplug_init, subsystem) - if not datasource: - return -@@ -216,6 +216,19 @@ - action=udevaction, - success_fn=hotplug_init._write_to_cache, - ) -+ start = time.time() -+ if not datasource.hotplug_retry_settings.force_retry: -+ try_hotplug(subsystem, event_handler, datasource) -+ return -+ while time.time() - start < datasource.hotplug_retry_settings.sleep_total: -+ try_hotplug(subsystem, event_handler, datasource) -+ LOG.debug( -+ "Gathering network configuration again due to IMDS limitations." -+ ) -+ time.sleep(datasource.hotplug_retry_settings.sleep_period) -+ -+ -+def try_hotplug(subsystem, event_handler, datasource) -> None: - wait_times = [1, 3, 5, 10, 30] - last_exception = Exception("Bug while processing hotplug event.") - for attempt, wait in enumerate(wait_times): ---- a/cloudinit/sources/DataSourceEc2.py -+++ b/cloudinit/sources/DataSourceEc2.py -@@ -24,7 +24,7 @@ - from cloudinit.net import netplan - from cloudinit.net.dhcp import NoDHCPLeaseError - from cloudinit.net.ephemeral import EphemeralIPNetwork --from cloudinit.sources import NicOrder -+from cloudinit.sources import HotplugRetrySettings, NicOrder - from cloudinit.sources.helpers import ec2 - - LOG = logging.getLogger(__name__) -@@ -114,6 +114,7 @@ - } - - extra_hotplug_udev_rules = _EXTRA_HOTPLUG_UDEV_RULES -+ hotplug_retry_settings = HotplugRetrySettings(True, 5, 30) - - def __init__(self, sys_cfg, distro, paths): - super(DataSourceEc2, self).__init__(sys_cfg, distro, paths) -@@ -125,6 +126,7 @@ - super()._unpickle(ci_pkl_version) - self.extra_hotplug_udev_rules = _EXTRA_HOTPLUG_UDEV_RULES - self._fallback_nic_order = NicOrder.MAC -+ self.hotplug_retry_settings = HotplugRetrySettings(True, 5, 30) - - def _get_cloud_name(self): - """Return the cloud name as identified during _get_data.""" ---- a/cloudinit/sources/__init__.py -+++ b/cloudinit/sources/__init__.py -@@ -17,7 +17,7 @@ - import re - from collections import namedtuple - from enum import Enum, unique --from typing import Any, Dict, List, Optional, Tuple, Union -+from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union - - from cloudinit import ( - atomic_helper, -@@ -193,6 +193,14 @@ - ) - - -+class HotplugRetrySettings(NamedTuple): -+ """in seconds""" -+ -+ force_retry: bool -+ sleep_period: int -+ sleep_total: int -+ -+ - class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): - - dsmode = DSMODE_NETWORK -@@ -316,6 +324,14 @@ - # in the updated metadata - skip_hotplug_detect = False - -+ # AWS interface data propagates to the IMDS without a syncronization method -+ # Since no better alternative exists, use a datasource-specific mechanism -+ # which retries periodically for a set amount of time - apply configuration -+ # as needed. Do not force retry on other datasources. -+ # -+ # https://github.com/amazonlinux/amazon-ec2-net-utils/blob/601bc3513fa7b8a6ab46d9496b233b079e55f2e9/lib/lib.sh#L483 -+ hotplug_retry_settings = HotplugRetrySettings(False, 0, 0) -+ - # Extra udev rules for cc_install_hotplug - extra_hotplug_udev_rules: Optional[str] = None - -@@ -360,6 +376,7 @@ - "skip_hotplug_detect": False, - "vendordata2": None, - "vendordata2_raw": None, -+ "hotplug_retry_settings": HotplugRetrySettings(False, 0, 0), - } - for key, value in expected_attrs.items(): - if not hasattr(self, key): ---- a/doc/module-docs/cc_install_hotplug/data.yaml -+++ b/doc/module-docs/cc_install_hotplug/data.yaml -@@ -9,6 +9,11 @@ - refresh the instance metadata from the datasource, detect the device in - the updated metadata, then apply the updated network configuration. - -+ Udev rules are installed while cloud-init is running, which means that -+ devices which are added during boot might not be configured. To work -+ around this limitation, one can wait until cloud-init has completed -+ before hotplugging devices. -+ - Currently supported datasources: Openstack, EC2 - examples: - - comment: | ---- a/tests/integration_tests/modules/test_hotplug.py -+++ b/tests/integration_tests/modules/test_hotplug.py -@@ -1,3 +1,4 @@ -+import logging - import time - from collections import namedtuple - -@@ -36,6 +37,7 @@ - when: ['boot-new-instance'] - """ - -+LOG = logging.getLogger() - ip_addr = namedtuple("ip_addr", "interface state ip4 ip6") - - -@@ -319,6 +321,12 @@ - ips_before = _get_ip_addr(client) - primary_priv_ip4 = ips_before[1].ip4 - primary_priv_ip6 = ips_before[1].ip6 -+ # cloud-init is incapable of hotplugged devices until after -+ # completion (cloud-init.target / cloud-init status --wait) -+ # -+ # To reliably test cloud-init hotplug, wait for completion before -+ # testing behaviors. -+ wait_for_cloud_init(client) - client.instance.add_network_interface(ipv6_address_count=1) - - _wait_till_hotplug_complete(client) -@@ -348,13 +356,14 @@ - assert len(ips_after_add) == len(ips_before) + 1 - - # pings to primary and secondary NICs work -- r = bastion.execute(f"ping -c1 {primary_priv_ip4}") -+ # use -w so that test is less flaky with temporary network failure -+ r = bastion.execute(f"ping -c1 -w5 {primary_priv_ip4}") - assert r.ok, r.stdout -- r = bastion.execute(f"ping -c1 {secondary_priv_ip4}") -+ r = bastion.execute(f"ping -c1 -w5 {secondary_priv_ip4}") - assert r.ok, r.stdout -- r = bastion.execute(f"ping -c1 {primary_priv_ip6}") -+ r = bastion.execute(f"ping -c1 -w5 {primary_priv_ip6}") - assert r.ok, r.stdout -- r = bastion.execute(f"ping -c1 {secondary_priv_ip6}") -+ r = bastion.execute(f"ping -c1 -w5 {secondary_priv_ip6}") - assert r.ok, r.stdout - - # Check every route has metrics associated. See LP: #2055397 -@@ -368,12 +377,31 @@ - _wait_till_hotplug_complete(client, expected_runs=2) - - # ping to primary NIC works -- assert bastion.execute(f"ping -c1 {primary_priv_ip4}").ok -- assert bastion.execute(f"ping -c1 {primary_priv_ip6}").ok -+ retries = 32 -+ error = "" -+ for i in range(retries): -+ if bastion.execute(f"ping -c1 -w5 {primary_priv_ip4}").ok: -+ break -+ LOG.info("Failed to ping %s on try #%s", primary_priv_ip4, i + 1) -+ else: -+ error = ( -+ f"Failed to ping {primary_priv_ip4} after {retries} retries" -+ ) -+ -+ for i in range(retries): -+ if bastion.execute(f"ping -c1 -w5 {primary_priv_ip6}").ok: -+ break -+ LOG.info("Failed to ping %s on try #%s", primary_priv_ip6, i + 1) -+ else: -+ error = ( -+ f"Failed to ping {primary_priv_ip6} after {retries} retries" -+ ) - - log_content = client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log_content) - verify_clean_boot(client) -+ if error: -+ raise Exception(error) - - - @pytest.mark.skipif(PLATFORM != "ec2", reason="test is ec2 specific") ---- a/tests/unittests/cmd/devel/test_hotplug_hook.py -+++ b/tests/unittests/cmd/devel/test_hotplug_hook.py -@@ -36,6 +36,7 @@ - m_datasource = mock.MagicMock(spec=DataSource) - m_datasource.distro = m_distro - m_datasource.skip_hotplug_detect = False -+ m_datasource.hotplug_retry_settings = DataSource.hotplug_retry_settings - m_init.datasource = m_datasource - m_init.fetch.return_value = m_datasource - ---- a/tools/hook-hotplug -+++ b/tools/hook-hotplug -@@ -5,6 +5,7 @@ - # cloud-init is ready; if so invoke cloud-init hotplug-hook - - fifo=/run/cloud-init/hook-hotplug-cmd -+log_file=/run/cloud-init/hook-hotplug.log - - should_run() { - if [ -d /run/systemd ]; then -@@ -17,6 +18,9 @@ - } - - if ! should_run; then -+ # This happens when a device is hotplugged before cloud-init-hotplugd.socket is -+ # listening on the socket. -+ echo "Not running hotplug, not ready yet" >> ${log_file} - exit 0 - fi - -@@ -25,3 +29,4 @@ - env_params=" --subsystem=${SUBSYSTEM} handle --devpath=${DEVPATH} --udevaction=${ACTION}" - # write params to cloud-init's hotplug-hook fifo - echo "${env_params}" >&3 -+echo "Running hotplug hook: $env_params" >> ${log_file} diff --git a/debian/patches/deprecation-version-boundary.patch b/debian/patches/deprecation-version-boundary.patch index 0709b117..6ffb28ad 100644 --- a/debian/patches/deprecation-version-boundary.patch +++ b/debian/patches/deprecation-version-boundary.patch @@ -9,8 +9,8 @@ Last-Update: 2024-06-28 This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ --- a/cloudinit/features.py +++ b/cloudinit/features.py -@@ -87,7 +87,7 @@ On Debian and Ubuntu systems, cc_apt_con - to write /etc/apt/sources.list directly. +@@ -98,7 +98,7 @@ Note that in addition to this flag, down + to modify the systemd unit files. """ -DEPRECATION_INFO_BOUNDARY = "devel" diff --git a/debian/patches/grub-dpkg-support.patch b/debian/patches/grub-dpkg-support.patch index ceda7113..e62b9f66 100644 --- a/debian/patches/grub-dpkg-support.patch +++ b/debian/patches/grub-dpkg-support.patch @@ -28,7 +28,7 @@ This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ return --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json -@@ -1611,8 +1611,8 @@ +@@ -1616,8 +1616,8 @@ "properties": { "enabled": { "type": "boolean", diff --git a/debian/patches/no-nocloud-network.patch b/debian/patches/no-nocloud-network.patch index f81d82b3..1577e7e6 100644 --- a/debian/patches/no-nocloud-network.patch +++ b/debian/patches/no-nocloud-network.patch @@ -26,7 +26,7 @@ Last-Update: 2024-08-02 # Now that we have exhausted any other places merge in the defaults --- a/cloudinit/util.py +++ b/cloudinit/util.py -@@ -1011,7 +1011,6 @@ def read_seeded(base="", ext="", timeout +@@ -1012,7 +1012,6 @@ def read_seeded(base="", ext="", timeout ud_url = base.replace("%s", "user-data" + ext) vd_url = base.replace("%s", "vendor-data" + ext) md_url = base.replace("%s", "meta-data" + ext) @@ -34,7 +34,7 @@ Last-Update: 2024-08-02 else: if features.NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH: if base[-1] != "/" and parse.urlparse(base).query == "": -@@ -1020,17 +1019,7 @@ def read_seeded(base="", ext="", timeout +@@ -1021,17 +1020,7 @@ def read_seeded(base="", ext="", timeout ud_url = "%s%s%s" % (base, "user-data", ext) vd_url = "%s%s%s" % (base, "vendor-data", ext) md_url = "%s%s%s" % (base, "meta-data", ext) @@ -54,7 +54,7 @@ Last-Update: 2024-08-02 ) --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py -@@ -2482,7 +2482,7 @@ class TestReadOptionalSeed: +@@ -2498,7 +2498,7 @@ class TestReadOptionalSeed: { "meta-data": {"md": "val"}, "user-data": b"ud", @@ -63,7 +63,7 @@ Last-Update: 2024-08-02 "vendor-data": None, }, True, -@@ -2537,7 +2537,7 @@ class TestReadSeeded: +@@ -2553,7 +2553,7 @@ class TestReadSeeded: assert found_md == {"key1": "val1"} assert found_ud == ud assert found_vd == vd @@ -72,7 +72,7 @@ Last-Update: 2024-08-02 @pytest.mark.parametrize( "base, feature_flag, req_urls", -@@ -2546,7 +2546,6 @@ class TestReadSeeded: +@@ -2562,7 +2562,6 @@ class TestReadSeeded: "http://10.0.0.1/%s?qs=1", True, [ @@ -80,7 +80,7 @@ Last-Update: 2024-08-02 "http://10.0.0.1/meta-data?qs=1", "http://10.0.0.1/user-data?qs=1", "http://10.0.0.1/vendor-data?qs=1", -@@ -2557,7 +2556,6 @@ class TestReadSeeded: +@@ -2573,7 +2572,6 @@ class TestReadSeeded: "https://10.0.0.1:8008/", True, [ @@ -88,7 +88,7 @@ Last-Update: 2024-08-02 "https://10.0.0.1:8008/meta-data", "https://10.0.0.1:8008/user-data", "https://10.0.0.1:8008/vendor-data", -@@ -2568,7 +2566,6 @@ class TestReadSeeded: +@@ -2584,7 +2582,6 @@ class TestReadSeeded: "https://10.0.0.1:8008", True, [ @@ -96,7 +96,7 @@ Last-Update: 2024-08-02 "https://10.0.0.1:8008/meta-data", "https://10.0.0.1:8008/user-data", "https://10.0.0.1:8008/vendor-data", -@@ -2579,7 +2576,6 @@ class TestReadSeeded: +@@ -2595,7 +2592,6 @@ class TestReadSeeded: "https://10.0.0.1:8008", False, [ @@ -104,7 +104,7 @@ Last-Update: 2024-08-02 "https://10.0.0.1:8008meta-data", "https://10.0.0.1:8008user-data", "https://10.0.0.1:8008vendor-data", -@@ -2590,7 +2586,6 @@ class TestReadSeeded: +@@ -2606,7 +2602,6 @@ class TestReadSeeded: "https://10.0.0.1:8008?qs=", True, [ @@ -112,7 +112,7 @@ Last-Update: 2024-08-02 "https://10.0.0.1:8008?qs=meta-data", "https://10.0.0.1:8008?qs=user-data", "https://10.0.0.1:8008?qs=vendor-data", -@@ -2629,7 +2624,7 @@ class TestReadSeeded: +@@ -2645,7 +2640,7 @@ class TestReadSeeded: # user-data, vendor-data read raw. It could be scripts or other format assert found_ud == "/user-data: 1" assert found_vd == "/vendor-data: 1" @@ -121,7 +121,7 @@ Last-Update: 2024-08-02 assert [ mock.call(req_url, timeout=5, retries=10) for req_url in req_urls ] == m_read.call_args_list -@@ -2659,7 +2654,7 @@ class TestReadSeededWithoutVendorData(he +@@ -2675,7 +2670,7 @@ class TestReadSeededWithoutVendorData(he self.assertEqual(found_md, {"key1": "val1"}) self.assertEqual(found_ud, ud) self.assertEqual(found_vd, vd) diff --git a/debian/patches/no-single-process.patch b/debian/patches/no-single-process.patch index 44e755c6..46c2f2c2 100644 --- a/debian/patches/no-single-process.patch +++ b/debian/patches/no-single-process.patch @@ -4,6 +4,47 @@ This optimization is a big change in behavior, patch it out. Author: Brett Holman Last-Update: 2024-08-02 +--- a/cloudinit/cmd/status.py ++++ b/cloudinit/cmd/status.py +@@ -318,9 +318,8 @@ def systemd_failed(wait: bool) -> bool: + for service in [ + "cloud-final.service", + "cloud-config.service", +- "cloud-init-network.service", ++ "cloud-init.service", + "cloud-init-local.service", +- "cloud-init-main.service", + ]: + try: + stdout = query_systemctl( +--- a/cloudinit/config/cc_mounts.py ++++ b/cloudinit/config/cc_mounts.py +@@ -519,7 +519,7 @@ def handle(name: str, cfg: Config, cloud + # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno + uses_systemd = cloud.distro.uses_systemd() + default_mount_options = ( +- "defaults,nofail,x-systemd.after=cloud-init-network.service,_netdev" ++ "defaults,nofail,x-systemd.after=cloud-init.service,_netdev" + if uses_systemd + else "defaults,nobootwait" + ) +--- a/cloudinit/config/schemas/schema-cloud-config-v1.json ++++ b/cloudinit/config/schemas/schema-cloud-config-v1.json +@@ -2034,12 +2034,12 @@ + }, + "mount_default_fields": { + "type": "array", +- "description": "Default mount configuration for any mount entry with less than 6 options provided. When specified, 6 items are required and represent ``/etc/fstab`` entries. Default: ``defaults,nofail,x-systemd.after=cloud-init-network.service,_netdev``.", ++ "description": "Default mount configuration for any mount entry with less than 6 options provided. When specified, 6 items are required and represent ``/etc/fstab`` entries. Default: ``defaults,nofail,x-systemd.after=cloud-init.service,_netdev``.", + "default": [ + null, + null, + "auto", +- "defaults,nofail,x-systemd.after=cloud-init-network.service", ++ "defaults,nofail,x-systemd.after=cloud-init.service", + "0", + "2" + ], --- a/systemd/cloud-config.service +++ b/systemd/cloud-config.service @@ -9,14 +9,7 @@ ConditionEnvironment=!KERNEL_CMDLINE=clo @@ -17,11 +58,21 @@ Last-Update: 2024-08-02 -# process has completed this stage. The output from the return socket is piped -# into a shell so that the process can send a completion message (defaults to -# "done", otherwise includes an error message) and an exit code to systemd. --ExecStart=sh -c 'echo "start" | netcat -Uu -W1 /run/cloud-init/share/config.sock -s /run/cloud-init/share/config-return.sock | sh' +-ExecStart=sh -c 'echo "start" | nc -Uu -W1 /run/cloud-init/share/config.sock -s /run/cloud-init/share/config-return.sock | sh' +ExecStart=/usr/bin/cloud-init modules --mode=config RemainAfterExit=yes TimeoutSec=0 +--- a/systemd/cloud-config.target ++++ b/systemd/cloud-config.target +@@ -14,5 +14,5 @@ + + [Unit] + Description=Cloud-config availability +-Wants=cloud-init-local.service cloud-init-network.service +-After=cloud-init-local.service cloud-init-network.service ++Wants=cloud-init-local.service cloud-init.service ++After=cloud-init-local.service cloud-init.service --- a/systemd/cloud-final.service +++ b/systemd/cloud-final.service @@ -12,16 +12,10 @@ ConditionEnvironment=!KERNEL_CMDLINE=clo @@ -35,7 +86,7 @@ Last-Update: 2024-08-02 -# process has completed this stage. The output from the return socket is piped -# into a shell so that the process can send a completion message (defaults to -# "done", otherwise includes an error message) and an exit code to systemd. --ExecStart=sh -c 'echo "start" | netcat -Uu -W1 /run/cloud-init/share/final.sock -s /run/cloud-init/share/final-return.sock | sh' +-ExecStart=sh -c 'echo "start" | nc -Uu -W1 /run/cloud-init/share/final.sock -s /run/cloud-init/share/final-return.sock | sh' +ExecStart=/usr/bin/cloud-init modules --mode=final RemainAfterExit=yes TimeoutSec=0 @@ -53,7 +104,7 @@ Last-Update: 2024-08-02 Before=network-pre.target Before=shutdown.target {% if variant in ["almalinux", "cloudlinux", "rhel"] %} -@@ -16,6 +17,7 @@ +@@ -16,6 +17,7 @@ Before=firewalld.target Before=sysinit.target {% endif %} Conflicts=shutdown.target @@ -61,7 +112,7 @@ Last-Update: 2024-08-02 ConditionPathExists=!/etc/cloud/cloud-init.disabled ConditionKernelCommandLine=!cloud-init=disabled ConditionEnvironment=!KERNEL_CMDLINE=cloud-init=disabled -@@ -25,14 +27,7 @@ +@@ -25,14 +27,7 @@ Type=oneshot {% if variant in ["almalinux", "cloudlinux", "rhel"] %} ExecStartPre=/sbin/restorecon /run/cloud-init {% endif %} @@ -72,135 +123,11 @@ Last-Update: 2024-08-02 -# process has completed this stage. The output from the return socket is piped -# into a shell so that the process can send a completion message (defaults to -# "done", otherwise includes an error message) and an exit code to systemd. --ExecStart=sh -c 'echo "start" | netcat -Uu -W1 /run/cloud-init/share/local.sock -s /run/cloud-init/share/local-return.sock | sh' +-ExecStart=sh -c 'echo "start" | nc -Uu -W1 /run/cloud-init/share/local.sock -s /run/cloud-init/share/local-return.sock | sh' +ExecStart=/usr/bin/cloud-init init --local RemainAfterExit=yes TimeoutSec=0 ---- /dev/null -+++ b/systemd/cloud-init.service.tmpl -@@ -0,0 +1,56 @@ -+## template:jinja -+[Unit] -+# https://docs.cloud-init.io/en/latest/explanation/boot.html -+Description=Cloud-init: Network Stage -+{% if variant not in ["almalinux", "cloudlinux", "photon", "rhel"] %} -+DefaultDependencies=no -+{% endif %} -+Wants=cloud-init-local.service -+Wants=sshd-keygen.service -+Wants=sshd.service -+After=cloud-init-local.service -+After=systemd-networkd-wait-online.service -+{% if variant in ["ubuntu", "unknown", "debian"] %} -+After=networking.service -+{% endif %} -+{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", -+ "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", -+ "suse", "TencentOS", "virtuozzo"] %} -+ -+After=NetworkManager.service -+After=NetworkManager-wait-online.service -+{% endif %} -+{% if variant in ["suse"] %} -+After=wicked.service -+# setting hostname via hostnamectl depends on dbus, which otherwise -+# would not be guaranteed at this point. -+After=dbus.service -+{% endif %} -+Before=network-online.target -+Before=sshd-keygen.service -+Before=sshd.service -+Before=systemd-user-sessions.service -+{% if variant in ["ubuntu", "unknown", "debian"] %} -+Before=sysinit.target -+Before=shutdown.target -+Conflicts=shutdown.target -+{% endif %} -+{% if variant in ["suse"] %} -+Before=shutdown.target -+Conflicts=shutdown.target -+{% endif %} -+ConditionPathExists=!/etc/cloud/cloud-init.disabled -+ConditionKernelCommandLine=!cloud-init=disabled -+ConditionEnvironment=!KERNEL_CMDLINE=cloud-init=disabled -+ -+[Service] -+Type=oneshot -+ExecStart=/usr/bin/cloud-init init -+RemainAfterExit=yes -+TimeoutSec=0 -+ -+# Output needs to appear in instance console output -+StandardOutput=journal+console -+ -+[Install] -+WantedBy=cloud-init.target ---- a/cloudinit/cmd/status.py -+++ b/cloudinit/cmd/status.py -@@ -318,9 +318,8 @@ def systemd_failed(wait: bool) -> bool: - for service in [ - "cloud-final.service", - "cloud-config.service", -- "cloud-init-network.service", -+ "cloud-init.service", - "cloud-init-local.service", -- "cloud-init-main.service", - ]: - try: - stdout = query_systemctl( ---- a/cloudinit/config/cc_mounts.py -+++ b/cloudinit/config/cc_mounts.py -@@ -521,7 +521,7 @@ def handle(name: str, cfg: Config, cloud - # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno - uses_systemd = cloud.distro.uses_systemd() - default_mount_options = ( -- "defaults,nofail,x-systemd.after=cloud-init-network.service,_netdev" -+ "defaults,nofail,x-systemd.after=cloud-init.service,_netdev" - if uses_systemd - else "defaults,nobootwait" - ) ---- a/cloudinit/config/schemas/schema-cloud-config-v1.json -+++ b/cloudinit/config/schemas/schema-cloud-config-v1.json -@@ -2029,12 +2029,12 @@ - }, - "mount_default_fields": { - "type": "array", -- "description": "Default mount configuration for any mount entry with less than 6 options provided. When specified, 6 items are required and represent ``/etc/fstab`` entries. Default: ``defaults,nofail,x-systemd.after=cloud-init-network.service,_netdev``.", -+ "description": "Default mount configuration for any mount entry with less than 6 options provided. When specified, 6 items are required and represent ``/etc/fstab`` entries. Default: ``defaults,nofail,x-systemd.after=cloud-init.service,_netdev``.", - "default": [ - null, - null, - "auto", -- "defaults,nofail,x-systemd.after=cloud-init-network.service", -+ "defaults,nofail,x-systemd.after=cloud-init.service", - "0", - "2" - ], ---- a/systemd/cloud-config.target -+++ b/systemd/cloud-config.target -@@ -14,5 +14,5 @@ - - [Unit] - Description=Cloud-config availability --Wants=cloud-init-local.service cloud-init-network.service --After=cloud-init-local.service cloud-init-network.service -+Wants=cloud-init-local.service cloud-init.service -+After=cloud-init-local.service cloud-init.service ---- a/tests/unittests/config/test_cc_mounts.py -+++ b/tests/unittests/config/test_cc_mounts.py -@@ -566,9 +566,9 @@ class TestFstabHandling: - LABEL=keepme none ext4 defaults 0 0 - LABEL=UEFI - /dev/sda4 /mnt2 auto nofail,comment=cloudconfig 1 2 -- /dev/sda5 /mnt3 auto defaults,nofail,x-systemd.after=cloud-init-network.service,_netdev,comment=cloudconfig 0 2 -+ /dev/sda5 /mnt3 auto defaults,nofail,x-systemd.after=cloud-init.service,_netdev,comment=cloudconfig 0 2 - /dev/sda1 /mnt xfs auto,comment=cloudconfig 0 2 -- /dev/sda3 /mnt4 btrfs defaults,nofail,x-systemd.after=cloud-init-network.service,_netdev,comment=cloudconfig 0 2 -+ /dev/sda3 /mnt4 btrfs defaults,nofail,x-systemd.after=cloud-init.service,_netdev,comment=cloudconfig 0 2 - /dev/sdb1 none swap sw,comment=cloudconfig 0 0 - """ # noqa: E501 - ).strip() --- a/systemd/cloud-init-main.service.tmpl +++ /dev/null @@ -1,42 +0,0 @@ @@ -304,7 +231,7 @@ Last-Update: 2024-08-02 -# process has completed this stage. The output from the return socket is piped -# into a shell so that the process can send a completion message (defaults to -# "done", otherwise includes an error message) and an exit code to systemd. --ExecStart=sh -c 'echo "start" | netcat -Uu -W1 /run/cloud-init/share/network.sock -s /run/cloud-init/share/network-return.sock | sh' +-ExecStart=sh -c 'echo "start" | nc -Uu -W1 /run/cloud-init/share/network.sock -s /run/cloud-init/share/network-return.sock | sh' -RemainAfterExit=yes -TimeoutSec=0 - @@ -313,3 +240,76 @@ Last-Update: 2024-08-02 - -[Install] -WantedBy=cloud-init.target +--- /dev/null ++++ b/systemd/cloud-init.service.tmpl +@@ -0,0 +1,56 @@ ++## template:jinja ++[Unit] ++# https://docs.cloud-init.io/en/latest/explanation/boot.html ++Description=Cloud-init: Network Stage ++{% if variant not in ["almalinux", "cloudlinux", "photon", "rhel"] %} ++DefaultDependencies=no ++{% endif %} ++Wants=cloud-init-local.service ++Wants=sshd-keygen.service ++Wants=sshd.service ++After=cloud-init-local.service ++After=systemd-networkd-wait-online.service ++{% if variant in ["ubuntu", "unknown", "debian"] %} ++After=networking.service ++{% endif %} ++{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", ++ "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", ++ "suse", "TencentOS", "virtuozzo"] %} ++ ++After=NetworkManager.service ++After=NetworkManager-wait-online.service ++{% endif %} ++{% if variant in ["suse"] %} ++After=wicked.service ++# setting hostname via hostnamectl depends on dbus, which otherwise ++# would not be guaranteed at this point. ++After=dbus.service ++{% endif %} ++Before=network-online.target ++Before=sshd-keygen.service ++Before=sshd.service ++Before=systemd-user-sessions.service ++{% if variant in ["ubuntu", "unknown", "debian"] %} ++Before=sysinit.target ++Before=shutdown.target ++Conflicts=shutdown.target ++{% endif %} ++{% if variant in ["suse"] %} ++Before=shutdown.target ++Conflicts=shutdown.target ++{% endif %} ++ConditionPathExists=!/etc/cloud/cloud-init.disabled ++ConditionKernelCommandLine=!cloud-init=disabled ++ConditionEnvironment=!KERNEL_CMDLINE=cloud-init=disabled ++ ++[Service] ++Type=oneshot ++ExecStart=/usr/bin/cloud-init init ++RemainAfterExit=yes ++TimeoutSec=0 ++ ++# Output needs to appear in instance console output ++StandardOutput=journal+console ++ ++[Install] ++WantedBy=cloud-init.target +--- a/tests/unittests/config/test_cc_mounts.py ++++ b/tests/unittests/config/test_cc_mounts.py +@@ -566,9 +566,9 @@ class TestFstabHandling: + LABEL=keepme none ext4 defaults 0 0 + LABEL=UEFI + /dev/sda4 /mnt2 auto nofail,comment=cloudconfig 1 2 +- /dev/sda5 /mnt3 auto defaults,nofail,x-systemd.after=cloud-init-network.service,_netdev,comment=cloudconfig 0 2 ++ /dev/sda5 /mnt3 auto defaults,nofail,x-systemd.after=cloud-init.service,_netdev,comment=cloudconfig 0 2 + /dev/sda1 /mnt xfs auto,comment=cloudconfig 0 2 +- /dev/sda3 /mnt4 btrfs defaults,nofail,x-systemd.after=cloud-init-network.service,_netdev,comment=cloudconfig 0 2 ++ /dev/sda3 /mnt4 btrfs defaults,nofail,x-systemd.after=cloud-init.service,_netdev,comment=cloudconfig 0 2 + /dev/sdb1 none swap sw,comment=cloudconfig 0 0 + """ # noqa: E501 + ).strip() diff --git a/debian/patches/series b/debian/patches/series index 4fb6fca6..8c0008e5 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -2,10 +2,4 @@ deprecation-version-boundary.patch no-single-process.patch no-nocloud-network.patch grub-dpkg-support.patch -cpick-84806336-chore-Add-feature-flag-for-manual-network-waiting no-remove-networkd-online.patch -cpick-d75840be-fix-retry-AWS-hotplug-for-async-IMDS-5995 -cpick-c60771d8-test-pytestify-test_url_helper.py -cpick-8810a2dc-test-Remove-CiTestCase-from-test_url_helper.py -cpick-582f16c1-test-add-OauthUrlHelper-tests -cpick-9311e066-fix-Update-OauthUrlHelper-to-use-readurl-exception_cb diff --git a/doc/examples/cloud-config-add-apt-repos.txt b/doc/examples/cloud-config-add-apt-repos.txt index fe027171..ba45caa0 100644 --- a/doc/examples/cloud-config-add-apt-repos.txt +++ b/doc/examples/cloud-config-add-apt-repos.txt @@ -6,7 +6,7 @@ # Additional apt configuration and repositories section. # # -# Default: auto select based on cloud metadata +# Default: auto select based on instance-data # in ec2, the default is .archive.ubuntu.com # apt: # primary: @@ -24,7 +24,7 @@ # # if no mirror is provided by the DataSource, but 'search_dns' is # true, then search for dns names '-mirror' in each of -# - fqdn of this host per cloud metadata +# - fqdn of this host per meta-data # - localdomain # - no domain (which would search domains listed in /etc/resolv.conf) # If there is a dns entry for -mirror, then it is assumed that there diff --git a/doc/examples/cloud-config-apt.txt b/doc/examples/cloud-config-apt.txt index 04968035..57153d76 100644 --- a/doc/examples/cloud-config-apt.txt +++ b/doc/examples/cloud-config-apt.txt @@ -69,7 +69,7 @@ apt: # 1.3 primary/security archives # - # Default: none - instead it is auto select based on cloud metadata + # Default: none - instead it is auto select based on instance-data # so if neither "uri" nor "search", nor "search_dns" is set (the default) # then use the mirror provided by the DataSource found. # In EC2, that means using .ec2.archive.ubuntu.com @@ -96,7 +96,7 @@ apt: - http://us.archive.ubuntu.com/ubuntu # if no mirror is provided by uri or search but 'search_dns' is # true, then search for dns names '-mirror' in each of - # - fqdn of this host per cloud metadata + # - fqdn of this host per meta-data # - localdomain # - no domain (which would search domains listed in /etc/resolv.conf) # If there is a dns entry for -mirror, then it is assumed that diff --git a/doc/man/cloud-init.1 b/doc/man/cloud-init.1 index d69c5aba..b402b21e 100644 --- a/doc/man/cloud-init.1 +++ b/doc/man/cloud-init.1 @@ -9,8 +9,8 @@ cloud-init \- Cloud instance initialization .SH DESCRIPTION Cloud-init provides a mechanism for cloud instance initialization. This is done by identifying the cloud platform that is in use, reading -provided cloud metadata and optional vendor and user -data, and then initializing the instance as requested. +cloud meta-data, vendor-data and user-data, and then initializing the +instance as requested. .SH OPTIONS .TP diff --git a/doc/module-docs/cc_disable_ec2_metadata/data.yaml b/doc/module-docs/cc_disable_ec2_metadata/data.yaml index ce47a30b..b0bb0c80 100644 --- a/doc/module-docs/cc_disable_ec2_metadata/data.yaml +++ b/doc/module-docs/cc_disable_ec2_metadata/data.yaml @@ -7,5 +7,5 @@ cc_disable_ec2_metadata: - comment: | Example 1: file: cc_disable_ec2_metadata/example1.yaml - name: Disable EC2 Metadata - title: Disable AWS EC2 Metadata + name: Disable EC2 Instance Metadata Service + title: Disable AWS EC2 Instance Metadata Service diff --git a/doc/module-docs/cc_install_hotplug/data.yaml b/doc/module-docs/cc_install_hotplug/data.yaml index 310482eb..2277c9f2 100644 --- a/doc/module-docs/cc_install_hotplug/data.yaml +++ b/doc/module-docs/cc_install_hotplug/data.yaml @@ -1,7 +1,7 @@ cc_install_hotplug: description: | This module will install the udev rules to enable hotplug if supported by - the datasource and enabled in the userdata. The udev rules will be + the datasource and enabled in the user-data. The udev rules will be installed as ``/etc/udev/rules.d/90-cloud-init-hook-hotplug.rules``. When hotplug is enabled, newly added network devices will be added to the diff --git a/doc/module-docs/cc_ubuntu_autoinstall/data.yaml b/doc/module-docs/cc_ubuntu_autoinstall/data.yaml index d0ea9a0d..3c9a2939 100644 --- a/doc/module-docs/cc_ubuntu_autoinstall/data.yaml +++ b/doc/module-docs/cc_ubuntu_autoinstall/data.yaml @@ -1,18 +1,21 @@ cc_ubuntu_autoinstall: description: | - The ``autoinstall`` key indicates a configuration for the Ubuntu installer. - It is not acted on by cloud-init other than to ensure that the - configuration is schema compliant and that the installer package is - present on the system. + Cloud-init is used by the Ubuntu installer in two stages. + The ``autoinstall`` key may contain a configuration for the Ubuntu + installer. + + Cloud-init verifies that an ``autoinstall`` key contains a ``version`` key + and that the installer package is present on the system. .. note:: - The installer may use the provided configuration to instrument - cloud-init on the target system. See - `the Ubuntu installer documentation `_ - for more information. + The Ubuntu installer might pass part of this configuration to cloud-init + during a later boot as part of the install process. + See `the Ubuntu installer documentation `_ + for more information. Please direct Ubuntu installer questions to + their IRC channel (#ubuntu-server on Libera). examples: - comment: | Example 1: file: cc_ubuntu_autoinstall/example1.yaml name: Ubuntu Autoinstall - title: Support Ubuntu live-server install syntax + title: Autoinstall configuration is ignored (but validated) by cloud-init. diff --git a/doc/module-docs/cc_update_etc_hosts/data.yaml b/doc/module-docs/cc_update_etc_hosts/data.yaml index c723a8a0..4ba38e4b 100644 --- a/doc/module-docs/cc_update_etc_hosts/data.yaml +++ b/doc/module-docs/cc_update_etc_hosts/data.yaml @@ -38,7 +38,7 @@ cc_update_etc_hosts: The strings ``$hostname`` and ``$fqdn`` are replaced in the template with the appropriate values, either from the config-config ``fqdn`` or - ``hostname`` if provided. When absent, the cloud metadata will be + ``hostname`` if provided. When absent, the meta-data will be checked for ``local-hostname`` which can be split into ``.``. diff --git a/doc/module-docs/cc_update_hostname/data.yaml b/doc/module-docs/cc_update_hostname/data.yaml index cfe5dc56..427fe512 100644 --- a/doc/module-docs/cc_update_hostname/data.yaml +++ b/doc/module-docs/cc_update_hostname/data.yaml @@ -27,7 +27,7 @@ cc_update_hostname: file: cc_update_hostname/example4.yaml - comment: > Example 5: Set hostname to ``external`` instead of ``external.fqdn.me`` when - cloud metadata provides the ``local-hostname``: ``external.fqdn.me``. + meta-data provides the ``local-hostname``: ``external.fqdn.me``. file: cc_update_hostname/example5.yaml - comment: > Example 6: On a machine without an ``/etc/hostname`` file, don''t create diff --git a/doc/rtd/development/contribute_docs.rst b/doc/rtd/development/contribute_docs.rst index a32dc38a..f57d7146 100644 --- a/doc/rtd/development/contribute_docs.rst +++ b/doc/rtd/development/contribute_docs.rst @@ -43,7 +43,7 @@ web browser to open `index.html` to view and navigate the site. How are the docs structured? ============================ -We use `Diataxis`_ to organise our documentation. There is more detail on the +We use `Diataxis`_ to organize our documentation. There is more detail on the layout of the ``doc`` directory in the :doc:`docs_layout` article. We also have a :doc:`style_docs` that will help you if you need to edit or diff --git a/doc/rtd/development/datasource_creation.rst b/doc/rtd/development/datasource_creation.rst index a4b13d10..10bb85f1 100644 --- a/doc/rtd/development/datasource_creation.rst +++ b/doc/rtd/development/datasource_creation.rst @@ -82,8 +82,8 @@ and/or unused may be considered for eventual removal. Adding cloud-init support ========================= -There are multiple ways to provide `user data`, `metadata`, and -`vendor data`, and each cloud solution prefers its own way. A datasource +There are multiple ways to provide `user-data`, `meta-data`, and +`vendor-data`, and each cloud solution prefers its own way. A datasource abstract base class defines a single interface to interact with the different clouds. Each cloud implementation must inherit from this base class to use this shared functionality and interface. See :file:`cloud-init/sources/__init__.py` diff --git a/doc/rtd/development/docs_layout.rst b/doc/rtd/development/docs_layout.rst index 1e4e7c76..f25807b2 100644 --- a/doc/rtd/development/docs_layout.rst +++ b/doc/rtd/development/docs_layout.rst @@ -80,7 +80,7 @@ page. This subdirectory is of most interest to anyone who wants to create or update either the content of the documentation, or the styling of it. -* The content of the documentation is organised according to the `Diataxis`_ +* The content of the documentation is organized according to the `Diataxis`_ framework and can be found in the subdirectories: ``tutorial/``, ``howto/``, ``explanation/``, and ``reference/``. diff --git a/doc/rtd/development/integration_tests.rst b/doc/rtd/development/integration_tests.rst index 2f810427..17010c05 100644 --- a/doc/rtd/development/integration_tests.rst +++ b/doc/rtd/development/integration_tests.rst @@ -17,7 +17,7 @@ Test definition =============== Tests are defined like any other ``pytest`` test. The ``user_data`` -mark can be used to supply the cloud-config user data. Platform-specific +mark can be used to supply the cloud-config user-data. Platform-specific marks can be used to limit tests to particular platforms. The ``client`` fixture can be used to interact with the launched test instance. @@ -237,6 +237,30 @@ on of: PLATFORM = 'lxd_container' +Selecting Instance Type +----------------------- + +To select a specific instance type, modify the ``INSTANCE_TYPE`` variable to be +the desired instance type. This value is cloud-specific, so refer to the +cloud's documentation for the available instance types. If you specify an +instance type, be sure to also specify respective cloud platform you are +testing against. + +.. tab-set:: + + .. tab-item:: Inline environment variable + + .. code-block:: bash + + CLOUD_INIT_PLATFORM=ec2 CLOUD_INIT_INSTANCE_TYPE='t2.micro' tox -e integration_tests + + .. tab-item:: user_settings.py file + + .. code-block:: python + + PLATFORM = 'ec2' # need to specify the cloud in order to use the instance type setting + INSTANCE_TYPE = 't2.micro' + Image selection =============== diff --git a/doc/rtd/development/internal_files.rst b/doc/rtd/development/internal_files.rst index 5da1c684..27e72360 100644 --- a/doc/rtd/development/internal_files.rst +++ b/doc/rtd/development/internal_files.rst @@ -20,8 +20,8 @@ subdirectories: The :file:`/var/lib/cloud/instance` directory is a symbolic link that points to the most recently used :file:`instance-id` directory. This folder contains -the information ``cloud-init`` received from datasources, including vendor and -user data. This can help to determine that the correct data was passed. +the information ``cloud-init`` received from datasources, including vendor-data +and user-data. This can help to determine that the correct data was passed. It also contains the :file:`datasource` file that contains the full information about which datasource was identified and used to set up the system. diff --git a/doc/rtd/development/logging.rst b/doc/rtd/development/logging.rst index fed9f35b..b543dd42 100644 --- a/doc/rtd/development/logging.rst +++ b/doc/rtd/development/logging.rst @@ -166,7 +166,7 @@ The default configuration is to emit events to the cloud-init log file at ``DEBUG`` level. Event reporting can be configured using the ``reporting`` key in cloud-config -user data. +user-data. Configuration ------------- diff --git a/doc/rtd/development/module_creation.rst b/doc/rtd/development/module_creation.rst index 3e10a1ee..f88f9623 100644 --- a/doc/rtd/development/module_creation.rst +++ b/doc/rtd/development/module_creation.rst @@ -51,7 +51,7 @@ definition in `cloud-init-schema.json`_. - ``ONCE``: Runs only on first boot. - ``PER_INSTANCE``: Runs once per instance. When exactly this happens is dependent on the datasource, but may be triggered any time there - would be a significant change to the instance metadata. An example + would be a significant change to the instance-data. An example could be an instance being moved to a different subnet. - ``activate_by_schema_keys``: Optional list of cloud-config keys that will diff --git a/doc/rtd/development/style_docs.rst b/doc/rtd/development/style_docs.rst index ea73b815..bce84463 100644 --- a/doc/rtd/development/style_docs.rst +++ b/doc/rtd/development/style_docs.rst @@ -4,10 +4,8 @@ Documentation style guide Language -------- -Where possible, text should be written in UK English. However, discretion and -common sense can both be applied. For example, where text refers to code -elements that exist in US English, the spelling of these elements should not -be changed to UK English. +Where possible, text should be written in US English. However, discretion and +common sense can both be applied. Try to be concise and to the point in your writing. It is acceptable to link to official documentation elsewhere rather than repeating content. It's also @@ -87,12 +85,18 @@ It is generally best to avoid screenshots where possible. If you need to refer to text output, you can use code blocks. For diagrams, we recommend the use of `Mermaid`_. +File names +---------- + +File names should be decorated with backticks to ensure monospace font is used +to distinguish the name from regular text. + Code blocks ----------- -Our documentation uses the Sphinx extension "sphinx-copybutton", which creates -a small button on the right-hand side of code blocks for users to copy the -code snippets we provide. +Our documentation uses the Sphinx extension ``sphinx-copybutton``, which +creates a small button on the right-hand side of code blocks for users to copy +the code snippets we provide. The copied code will strip out the prompt symbol (``$``) so that users can paste commands directly into their terminal. For user convenience, please @@ -104,30 +108,30 @@ Vertical whitespace One newline between each section helps ensure readability of the documentation source code. -Common words ------------- - -There are some common words that should follow specific usage in text: - -- **cloud-init**: Always hyphenated, and follows sentence case, so only - capitalised at the start of a sentence. -- **metadata**, **datasource**: One word. -- **user data**, **vendor data**: Two words, not to be combined or hyphenated. - -When referring to file names, which may be hyphenated, they should be decorated -with backticks to ensure monospace font is used to distinguish them from -regular text. - Acronyms -------- -Acronyms are always capitalised (e.g., JSON, YAML, QEMU, LXD) in text. +Acronyms are always capitalized (e.g., JSON, YAML, QEMU, LXD) in text. The first time an acronym is used on a page, it is best practice to introduce it by showing the expanded name followed by the acronym in parentheses. E.g., Quick EMUlator (QEMU). If the acronym is very common, or you provide a link to a documentation page that provides such details, you will not need to do this. +Common terms +------------ + +The following project terms should follow specific usage in text: + +- **cloud-init**: Always hyphenated, and follows sentence case, so only + capitalized at the start of a sentence. +- **datasource**: One word. +- **user-data**: Two words, always hyphenated. +- **vendor-data**: Two words, always hyphenated. +- **cloud-config**: Two words, always hyphenated. +- **instance-data**: Two words, always hyphenated. +- **meta-data**: Two words, always hyphenated. + .. _Read the Docs: https://readthedocs.com/ .. _Python style guide: https://devguide.python.org/documentation/markup/ .. _Mermaid: https://mermaid.js.org/ diff --git a/doc/rtd/development/summit/2017_summit.rst b/doc/rtd/development/summit/2017_summit.rst index 803154e2..d9523df3 100644 --- a/doc/rtd/development/summit/2017_summit.rst +++ b/doc/rtd/development/summit/2017_summit.rst @@ -42,7 +42,7 @@ that they gave as a part of the summit: merge request CI process and encouraged this as a way for other OSes to participate. * **Using lxd for Rapid Development and Testing**: Scott demoed setting - userdata when launching a lxd instance and how this can be used in the + user-data when launching a lxd instance and how this can be used in the development process. He also discussed lxd image remotes and types of images. Breakout Sessions @@ -52,7 +52,7 @@ In addition to the prepared demos, the summit had numerous sessions that were requested by the attendees as additional topics for discussion: * Netplan (v2 YAML) as primary format -* How to query metadata +* How to query instance-data * Version numbering * Device hot-plug * Python 3 diff --git a/doc/rtd/development/summit/2018_summit.rst b/doc/rtd/development/summit/2018_summit.rst index aa659bb8..4888aca8 100644 --- a/doc/rtd/development/summit/2018_summit.rst +++ b/doc/rtd/development/summit/2018_summit.rst @@ -45,7 +45,7 @@ that they gave as a part of the summit: and there was a discussion on ending Python 2.7 support as well. An announcement to the mailing list is coming soon. * **Instance-data.json support and cloud-init cli**: Chad demoed a standard way - of querying instance data keys to enable scripting, templating, and access + of querying instance-data keys to enable scripting, templating, and access across all clouds. * **Multipass**: Alberto from the Canonical Multipass team joined us to demo the `Multipass`_ project. Multipass is the fastest way to get a virtual diff --git a/doc/rtd/development/summit/2023_summit.rst b/doc/rtd/development/summit/2023_summit.rst index 5cb3f8a0..0f329faf 100644 --- a/doc/rtd/development/summit/2023_summit.rst +++ b/doc/rtd/development/summit/2023_summit.rst @@ -15,7 +15,7 @@ and to realign on the direction and goals of the project. The event was generously hosted by Microsoft this year at their Redmond campus in Seattle, Washington, and we are grateful to the Microsoft community members "on the ground" who coordinated with Canonical's cloud-init development team to -help organise and run the event. Big thanks go as well to the Canonical +help organize and run the event. Big thanks go as well to the Canonical community team for helping us to set up the event site, as well as for their support and guidance with all the planning involved. @@ -29,7 +29,7 @@ our newer contributors in person. The first hybrid summit ======================= -This summit was organised as a hybrid event for the first time, and despite +This summit was organized as a hybrid event for the first time, and despite some initial uncertainties about how to implement that, it worked very well. In-person attendees included developers and contributors from Microsoft, Google, Amazon, Oracle, openSUSE and we had remote presentations provided by @@ -85,8 +85,8 @@ Presentation takeaways spelunking". This aligns well with Brett’s ongoing roadmap work to raise warnings from the - CLI and some of the strict JSON schema validation on network-config and user - data/vendor data. + CLI and some of the strict JSON schema validation on network-config and + user-data/vendor-data. * Good lessons from both AlpineLinux (Dermot Bradley), who investigated SSH alternatives like dropbearSSH and tinySSH, and FreeBSD (Mina Galić), who @@ -124,7 +124,7 @@ Round-table discussions * **Shared test frameworks**: Azure intends to invest in integration testing with the cloud-init community, to develop distribution-agnostic best practices for verification of distribution releases, and boot-speed and image - health analysis. If there are ways we want to collaborate on generalised + health analysis. If there are ways we want to collaborate on generalized testing and verification of images, they may provide some development toward this cause. @@ -132,7 +132,7 @@ Breakout sessions ----------------- * **Private reviews of partner engagements** with Oracle and AWS, and Fabio - Martins, Kyler Horner, and James to prioritise ongoing work and plan for the + Martins, Kyler Horner, and James to prioritize ongoing work and plan for the future development of IPv6-only datasource support - as well as other features. @@ -166,7 +166,7 @@ Thank you! ========== This event could not have taken place without the hard work and preparation of -all our presenters, organisers, and the voices of our community members in +all our presenters, organizers, and the voices of our community members in attendance. So, thank you again to everyone who participated, and we very much hope to see you again at the next cloud-init summit! diff --git a/doc/rtd/development/testing.rst b/doc/rtd/development/testing.rst index 940ce3b2..a94641a7 100644 --- a/doc/rtd/development/testing.rst +++ b/doc/rtd/development/testing.rst @@ -51,28 +51,14 @@ Test layout * ``pytest`` tests should use bare ``assert`` statements, to take advantage of ``pytest``'s `assertion introspection`_. -``pytest`` version "gotchas" ----------------------------- - -As we still support Ubuntu 18.04 (Bionic Beaver), we can only use ``pytest`` -features that are available in v3.3.2. This is an inexhaustive list of -ways in which this may catch you out: - - * Only the following built-in fixtures are available [#fixture-list]_: - - * ``cache`` - * ``capfd`` - * ``capfdbinary`` - * ``caplog`` - * ``capsys`` - * ``capsysbinary`` - * ``doctest_namespace`` - * ``monkeypatch`` - * ``pytestconfig`` - * ``record_xml_property`` - * ``recwarn`` - * ``tmpdir_factory`` - * ``tmpdir`` +Dependency versions +------------------- + +Cloud-init supports a range of versions for each of its test dependencies, as +well as runtime dependencies. If you are unsure whether a specific feature is +supported for a particular dependency, check the ``lowest-supported`` +environment in ``tox.ini``. This can be run using ``tox -e lowest-supported``. +This runs as a Github Actions job when a pull request is submitted or updated. Mocking and assertions ---------------------- @@ -139,13 +125,6 @@ Test argument ordering * ``pytest.mark.parametrize`` * ``mock.patch`` -.. [#fixture-list] This list of fixtures (with markup) can be - reproduced by running:: - - python3 -m pytest --fixtures -q | grep "^[^ -]" | grep -v 'no tests ran in' | sort | sed 's/ \[session scope\]//g;s/.*/* ``\0``/g' - - in an ubuntu lxd container with python3-pytest installed. - .. LINKS: .. _pytest: https://docs.pytest.org/ .. _pytest fixtures: https://docs.pytest.org/en/latest/fixture.html diff --git a/doc/rtd/explanation/about-cloud-config.rst b/doc/rtd/explanation/about-cloud-config.rst index dae73a65..23cdfc03 100644 --- a/doc/rtd/explanation/about-cloud-config.rst +++ b/doc/rtd/explanation/about-cloud-config.rst @@ -3,7 +3,7 @@ About the cloud-config file *************************** -The ``#cloud-config`` file is a type of user data that cloud-init can consume +The ``#cloud-config`` file is a type of user-data that cloud-init can consume to automatically set up various aspects of the system. It is commonly referred to as **cloud config**. Using cloud config to configure your machine leverages the best practices encoded into cloud-init's modules in a distribution-agnostic @@ -73,7 +73,7 @@ you of any errors. Example cloud-config file ========================= -The following code is an example of a complete user data cloud-config file. +The following code is an example of a complete user-data cloud-config file. The :ref:`cloud-config example library ` contains further examples that can be copy/pasted and adapted to your needs. diff --git a/doc/rtd/explanation/analyze.rst b/doc/rtd/explanation/analyze.rst index 04205aec..3257a530 100644 --- a/doc/rtd/explanation/analyze.rst +++ b/doc/rtd/explanation/analyze.rst @@ -324,7 +324,7 @@ Example output: UserspaceTimestampMonotonic=989279 The ``UserspaceTimestamp`` tracks when the init system starts, which is used -as an indicator of the kernel finishing initialisation. +as an indicator of the kernel finishing initialization. Running the following command will gather the ``InactiveExitTimestamp``: diff --git a/doc/rtd/explanation/boot.rst b/doc/rtd/explanation/boot.rst index ac1f6193..e72914c3 100644 --- a/doc/rtd/explanation/boot.rst +++ b/doc/rtd/explanation/boot.rst @@ -74,7 +74,7 @@ In most cases, this stage does not do much more than that. It finds the datasource and determines the network configuration to be used. That network configuration can come from: -- **datasource**: Cloud-provided network configuration via metadata. +- **datasource**: Cloud-provided network configuration via meta-data. - **fallback**: ``Cloud-init``'s fallback networking consists of rendering the equivalent to ``dhcp on eth0``, which was historically the most popular mechanism for network configuration of a guest. @@ -116,7 +116,7 @@ Network +---------+--------+----------------------------------------------------------+ This stage requires all configured networking to be online, as it will fully -process any user data that is found. Here, processing means it will: +process any user-data that is found. Here, processing means it will: - retrieve any ``#include`` or ``#include-once`` (recursively) including http, @@ -127,7 +127,7 @@ This stage runs the ``disk_setup`` and ``mounts`` modules which may partition and format disks and configure mount points (such as in :file:`/etc/fstab`). Those modules cannot run earlier as they may receive configuration input from sources only available via the network. For example, a user may have -provided user data in a network resource that describes how local mounts +provided user-data in a network resource that describes how local mounts should be done. On some clouds, such as Azure, this stage will create filesystems to be @@ -180,7 +180,7 @@ Things that run here include: - package installations, - configuration management plugins (Ansible, Puppet, Chef, salt-minion), and -- user-defined scripts (i.e., shell scripts passed as user data). +- user-defined scripts (i.e., shell scripts passed as user-data). For scripts external to ``cloud-init`` looking to wait until ``cloud-init`` is finished, the :command:`cloud-init status --wait` subcommand can help block diff --git a/doc/rtd/explanation/configuration.rst b/doc/rtd/explanation/configuration.rst index 4f4c43b6..5eb0be1d 100644 --- a/doc/rtd/explanation/configuration.rst +++ b/doc/rtd/explanation/configuration.rst @@ -13,7 +13,7 @@ Base configuration The base configuration format uses `YAML version 1.1`_, but may be declared as jinja templates which cloud-init will render at runtime with -:ref:`instance data ` variables. +:ref:`instance-data ` variables. From lowest priority to highest, configuration sources are: @@ -23,7 +23,7 @@ From lowest priority to highest, configuration sources are: and :file:`/etc/cloud/cloud.cfg.d/*.cfg`. - **Runtime config**: Anything defined in :file:`/run/cloud-init/cloud.cfg`. - **Kernel command line**: On the kernel command line, anything found between - ``cc:`` and ``end_cc`` will be interpreted as cloud-config user data. + ``cc:`` and ``end_cc`` will be interpreted as cloud-config user-data. These four sources make up the base configuration. The contents of this configuration are defined in the @@ -32,13 +32,13 @@ configuration are defined in the .. note:: Base configuration may contain :ref:`cloud-config` which may be - overridden by vendor data and user data. + overridden by vendor-data and user-data. -Vendor and user data -==================== +Vendor-data and user-data +========================= -Added to the base configuration are :ref:`vendor data` and -:ref:`user data` which are both provided by the datasource. +Added to the base configuration are :ref:`vendor-data` and +:ref:`user-data` which are both provided by the datasource. These get fetched from the datasource and are defined at instance launch. @@ -55,29 +55,29 @@ Specifying configuration End users --------- -Pass :ref:`user data` to the cloud provider. +Pass :ref:`user-data` to the cloud provider. Every platform supporting ``cloud-init`` will provide a method of supplying -user data. If you're unsure how to do this, reference the documentation +user-data. If you're unsure how to do this, reference the documentation provided by the cloud platform you're on. Additionally, there may be related ``cloud-init`` documentation in the :ref:`datasource` section. -Once an instance has been initialised, the user data may not be edited. +Once an instance has been initialized, the user-data may not be edited. It is sourced directly from the cloud, so even if you find a local file -that contains user data, it will likely be overwritten in the next boot. +that contains user-data, it will likely be overwritten in the next boot. Distro providers ---------------- Modify the base config. This often involves submitting a PR to modify -the base `cloud.cfg template`_, which is used to customise +the base `cloud.cfg template`_, which is used to customize :file:`/etc/cloud/cloud.cfg` per distro. Additionally, a file can be added to :file:`/etc/cloud/cloud.cfg.d` to override a piece of the base configuration. Cloud providers --------------- -Pass vendor data. This is the preferred method for clouds to provide +Pass vendor-data. This is the preferred method for clouds to provide their own customisation. In some cases, it may make sense to modify the base config in the same manner as distro providers on cloud-supported images. diff --git a/doc/rtd/explanation/events.rst b/doc/rtd/explanation/events.rst index dffbb773..5d5f042f 100644 --- a/doc/rtd/explanation/events.rst +++ b/doc/rtd/explanation/events.rst @@ -6,7 +6,7 @@ Events and updates Events ====== -``Cloud-init`` will fetch and apply cloud and user data configuration +``Cloud-init`` will fetch and apply cloud and user-data configuration upon several event types. The two most common events for ``cloud-init`` are when an instance first boots and any subsequent boot thereafter (reboot). In addition to boot events, ``cloud-init`` users and vendors are interested @@ -21,12 +21,6 @@ event types: default behaviour, this option exists to prevent regressing such behaviour. - ``HOTPLUG``: Dynamic add of a system device. -Future work will likely include infrastructure and support for the following -events: - -- ``METADATA_CHANGE``: An instance's metadata has changed. -- ``USER_REQUEST``: Directed request to update. - Datasource event support ======================== @@ -39,7 +33,7 @@ running on a platform whose datasource cannot support the event. Configuring event updates ========================= -Update configuration may be specified via user data, which can be used to +Update configuration may be specified via user-data, which can be used to enable or disable handling of specific events. This configuration will be honored as long as the events are supported by the datasource. However, configuration will always be applied at first boot, regardless of the user @@ -66,9 +60,9 @@ Hotplug ======= When the ``hotplug`` event is supported by the datasource and configured in -:ref:`user data`, ``cloud-init`` will respond to the +:ref:`user-data`, ``cloud-init`` will respond to the addition or removal of network interfaces to the system. In addition to -fetching and updating the system metadata, ``cloud-init`` will also bring +fetching and updating the instance-data, ``cloud-init`` will also bring up/down the newly added interface. Example diff --git a/doc/rtd/explanation/first_boot.rst b/doc/rtd/explanation/first_boot.rst index 2348e6e2..3e4d3b69 100644 --- a/doc/rtd/explanation/first_boot.rst +++ b/doc/rtd/explanation/first_boot.rst @@ -75,7 +75,7 @@ other than manually cleaning the cache. .. [#problems] A couple of ways in which this strict reliance on the presence of a datasource has been observed to cause problems: - - If a cloud's metadata service is flaky and ``cloud-init`` cannot + - If a cloud's instance metadata service is flaky and ``cloud-init`` cannot obtain the instance ID locally on that platform, ``cloud-init``'s instance ID determination will sometimes fail to determine the current instance ID, which makes it impossible to determine if this is an diff --git a/doc/rtd/explanation/format.rst b/doc/rtd/explanation/format.rst index 7d14acb6..f999a9ec 100644 --- a/doc/rtd/explanation/format.rst +++ b/doc/rtd/explanation/format.rst @@ -1,28 +1,28 @@ .. _user_data_formats: -User data formats +User-data formats ***************** -User data is configuration data provided by a user of a cloud platform to an -instance at launch. User data can be passed to cloud-init in any of many -formats documented here. User data is combined with the other +User-data is configuration data provided by a user of a cloud platform to an +instance at launch. User-data can be passed to cloud-init in any of many +formats documented here. User-data is combined with the other :ref:`configuration sources` to create a combined configuration which modifies an instance. Configuration types =================== -User data formats can be categorized into those that directly configure the +User-data formats can be categorized into those that directly configure the instance, and those that serve as a container, template, or means to obtain or modify another configuration. Formats that directly configure the instance: - `Cloud config data`_ -- `User data script`_ +- `User-data script`_ - `Cloud boothook`_ -Formats that deal with other user data formats: +Formats that deal with other user-data formats: - `Include file`_ - `Jinja template`_ @@ -71,7 +71,7 @@ For more information, see the cloud config .. _user_data_script: -User data script +User-data script ================ Example @@ -85,13 +85,13 @@ Example Explanation ----------- -A user data script is a single script to be executed once per instance. -User data scripts are run relatively late in the boot process, during +A user-data script is a single script to be executed once per instance. +User-data scripts are run relatively late in the boot process, during cloud-init's :ref:`final stage` as part of the :ref:`cc_scripts_user` module. .. warning:: - Use of ``INSTANCE_ID`` variable within user data scripts is deprecated. + Use of ``INSTANCE_ID`` variable within user-data scripts is deprecated. Use :ref:`jinja templates` with :ref:`v1.instance_id` instead. @@ -125,7 +125,7 @@ Example of once-per-instance script Explanation ----------- -A cloud boothook is similar to a :ref:`user data script` +A cloud boothook is similar to a :ref:`user-data script` in that it is a script run on boot. When run, the environment variable ``INSTANCE_ID`` is set to the current instance ID for use within the script. @@ -157,7 +157,7 @@ Explanation ----------- An include file contains a list of URLs, one per line. Each of the URLs will -be read and their content can be any kind of user data format, both base +be read and their content can be any kind of user-data format, both base config and meta config. If an error occurs reading a file the remaining files will not be read. @@ -180,7 +180,7 @@ Example cloud-config .. _jinja-script: -Example user data script +Example user-data script ------------------------ .. code-block:: shell @@ -193,13 +193,13 @@ Explanation ----------- `Jinja templating `_ may be used for -cloud-config and user data scripts. Any -:ref:`instance-data variables` may be used +cloud-config and user-data scripts. Any +:ref:`instance-data variables` may be used as jinja template variables. Any jinja templated configuration must contain the original header along with the new jinja header above it. .. note:: - Use of Jinja templates is supported for cloud-config, user data + Use of Jinja templates is supported for cloud-config, user-data scripts, and cloud-boothooks. Jinja templates are not supported for meta configs. @@ -241,7 +241,7 @@ Explanation Using a MIME multi-part file, the user can specify more than one type of data. -For example, both a user data script and a cloud-config type could be +For example, both a user-data script and a cloud-config type could be specified. Each part must specify a valid @@ -265,14 +265,14 @@ MIME multipart message to :file:`stdout`. **MIME subcommand Examples** -Create user data containing both a cloud-config (:file:`config.yaml`) +Create user-data containing both a cloud-config (:file:`config.yaml`) and a shell script (:file:`script.sh`) .. code-block:: shell-session - $ cloud-init devel make-mime -a config.yaml:cloud-config -a script.sh:x-shellscript > userdata + $ cloud-init devel make-mime -a config.yaml:cloud-config -a script.sh:x-shellscript > user-data.mime -Create user data containing 3 shell scripts: +Create user-data containing 3 shell scripts: - :file:`always.sh` - run every boot - :file:`instance.sh` - run once per instance @@ -315,13 +315,13 @@ The format is a list of dictionaries. Required fields: * ``type``: The :ref:`Content-Type` - identifier for the type of user data in content -* ``content``: The user data configuration + identifier for the type of user-data in content +* ``content``: The user-data configuration Optional fields: * ``launch-index``: The EC2 Launch-Index (if applicable) -* ``filename``: This field is only used if using a user data format that +* ``filename``: This field is only used if using a user-data format that requires a filename in a MIME part. This is unrelated to any local system file. @@ -344,7 +344,7 @@ Explanation ----------- A part handler contains custom code for either supporting new -mime-types in multi-part user data or for overriding the existing handlers for +mime-types in multi-part user-data or for overriding the existing handlers for supported mime-types. See the :ref:`custom part handler` reference documentation @@ -357,7 +357,7 @@ Gzip compressed content Content found to be gzip compressed will be uncompressed. The uncompressed data will then be used as if it were not compressed. -This is typically useful because user data size may be limited based on +This is typically useful because user-data size may be limited based on cloud platform. .. _user_data_formats-content_types: @@ -365,22 +365,22 @@ cloud platform. Headers and content types ========================= -In order for cloud-init to recognize which user data format is being used, -the user data must contain a header. Additionally, if the user data +In order for cloud-init to recognize which user-data format is being used, +the user-data must contain a header. Additionally, if the user-data is being passed as a multi-part message, such as MIME, cloud-config-archive, or part-handler, the content-type for each part must also be set appropriately. -The table below lists the headers and content types for each user data format. +The table below lists the headers and content types for each user-data format. Note that gzip compressed content is not represented here as it gets passed as binary data and so may be processed automatically. +--------------------+-----------------------------+-------------------------+ -|User data format |Header |Content-Type | +|User-data format |Header |Content-Type | +====================+=============================+=========================+ |Cloud config data |#cloud-config |text/cloud-config | +--------------------+-----------------------------+-------------------------+ -|User data script |#! |text/x-shellscript | +|User-data script |#! |text/x-shellscript | +--------------------+-----------------------------+-------------------------+ |Cloud boothook |#cloud-boothook |text/cloud-boothook | +--------------------+-----------------------------+-------------------------+ diff --git a/doc/rtd/explanation/instancedata.rst b/doc/rtd/explanation/instancedata.rst index b8162820..9b6ede9c 100644 --- a/doc/rtd/explanation/instancedata.rst +++ b/doc/rtd/explanation/instancedata.rst @@ -1,8 +1,8 @@ -.. _instance_metadata: +.. _instance-data: -Instance metadata -***************** +Instance-data +************* .. toctree:: :maxdepth: 1 @@ -45,7 +45,7 @@ Discovery on invalid ``instance-data`` keys, paths, or invalid syntax. The :command:`query` command also publishes ``userdata`` and ``vendordata`` -keys to the root user which will contain the decoded user and vendor data +keys to the root user which will contain the decoded user-data and vendor-data provided to this instance. Non-root users referencing ``userdata`` or ``vendordata`` keys will see only redacted values. @@ -62,7 +62,7 @@ Using ``instance-data`` ``instance-data`` can be used in the following configuration types: -* :ref:`User data scripts`. +* :ref:`User-data scripts`. * :ref:`Cloud-config`. * :ref:`Base configuration`. * Command line interface via :command:`cloud-init query` or @@ -101,7 +101,7 @@ Example: Cloud config with ``instance-data`` "availability-zone": "{{ v1.availability_zone }}"}' https://example.com -Example: User data script with ``instance-data`` +Example: User-data script with ``instance-data`` ------------------------------------------------ .. code-block:: jinja @@ -157,7 +157,7 @@ Storage locations ----------------- * :file:`/run/cloud-init/instance-data.json`: world-readable JSON containing - standardised keys, sensitive keys redacted. + standardized keys, sensitive keys redacted. * :file:`/run/cloud-init/instance-data-sensitive.json`: root-readable unredacted JSON blob. * :file:`/run/cloud-init/combined-cloud-config.json`: root-readable @@ -165,7 +165,7 @@ Storage locations are applied to the :file:`/run/cloud-init/combined-cloud-config.json` config values. -.. _instance_metadata-keys: +.. _instance-data-keys: :file:`instance-data.json` top level keys ----------------------------------------- @@ -214,9 +214,9 @@ included in the ``sensitive-keys`` list which is only readable by root. ``ds`` ^^^^^^ -Datasource-specific metadata crawled for the specific cloud platform. It should -closely represent the structure of the cloud metadata crawled. The structure of -content and details provided are entirely cloud-dependent. Mileage will vary +Datasource-specific data crawled for the specific cloud platform. It should +closely represent the structure of the data crawled. The structure of content +and details provided are entirely cloud-dependent. Mileage will vary depending on what the cloud exposes. The content exposed under the ``ds`` key is currently **experimental** and expected to change slightly in the upcoming ``cloud-init`` release. @@ -238,13 +238,13 @@ underlying host ``sys_info`` key above. ``v1`` ^^^^^^ -Standardised ``cloud-init`` metadata keys, these keys are guaranteed to exist +Standardized ``cloud-init`` data keys, these keys are guaranteed to exist on all cloud platforms. They will also retain their current behaviour and format, and will be carried forward even if ``cloud-init`` introduces a new -version of standardised keys with ``v2``. +version of standardized keys with ``v2``. To cut down on keystrokes on the command line, ``cloud-init`` also provides -top-level key aliases for any standardised ``v#`` keys present. The preceding +top-level key aliases for any standardized ``v#`` keys present. The preceding ``v1`` is not required of ``v1.var_name`` These aliases will represent the value of the highest versioned standard key. For example, ``cloud_name`` value will be ``v2.cloud_name`` if both ``v1`` and ``v2`` keys are present in @@ -257,13 +257,13 @@ jinja-safe key alias. This allows for ``cloud-init`` templates to use aliased variable references which allow for jinja's dot-notation reference such as ``{{ ds.v1_0.my_safe_key }}`` instead of ``{{ ds["v1.0"]["my/safe-key"] }}``. -Standardised :file:`instance-data.json` v1 keys +Standardized :file:`instance-data.json` v1 keys ----------------------------------------------- ``v1._beta_keys`` ^^^^^^^^^^^^^^^^^ -List of standardised keys still in 'beta'. The format, intent or presence of +List of standardized keys still in 'beta'. The format, intent or presence of these keys can change. Do not consider them production-ready. Example output: @@ -367,8 +367,7 @@ Example output: ``v1.subplatform`` ^^^^^^^^^^^^^^^^^^ -Additional platform details describing the specific source or type of metadata -used. The format of subplatform will be: +Detailed platform information. Subplatform format is: `` ()`` @@ -382,7 +381,7 @@ Example output: ``v1.public_ssh_keys`` ^^^^^^^^^^^^^^^^^^^^^^ -A list of SSH keys provided to the instance by the datasource metadata. +A list of SSH keys provided to the instance by the datasource. Example output: diff --git a/doc/rtd/explanation/introduction.rst b/doc/rtd/explanation/introduction.rst index d14fe19c..4c584c13 100644 --- a/doc/rtd/explanation/introduction.rst +++ b/doc/rtd/explanation/introduction.rst @@ -4,7 +4,7 @@ Introduction to cloud-init ************************** Managing and configuring cloud instances and servers can be a complex -and time-consuming task. Cloud-init is an open source initialisation tool that +and time-consuming task. Cloud-init is an open source initialization tool that was designed to make it easier to get your systems up and running with a minimum of effort, already configured according to your needs. @@ -63,10 +63,10 @@ it will: from it. This data tells cloud-init what actions to take. This can be in the form of: - * **Metadata** about the instance, such as the machine ID, hostname and - network config, or - * **Vendor data** and/or **user data**. These take the same form, although - Vendor data is provided by the cloud vendor, and user data is provided by + * **Meta-data** instance platform data, such as the machine ID, hostname and + network config + * **Vendor-data** and/or **user-data**. These take the same form, although + vendor-data is provided by the cloud vendor, and user-data is provided by the user. These data are usually applied in the post-networking phase, and might include: @@ -85,7 +85,7 @@ During late boot In the boot stages that come after the network has been configured, cloud-init runs through the tasks that were not critical for provisioning. This is where it configures the running instance according to your needs, as specified in the -vendor data and/or user data. It will take care of: +vendor-data and/or user-data. It will take care of: * **Configuration management**: Cloud-init can interact with tools like Puppet, Ansible, or Chef to apply @@ -97,7 +97,7 @@ vendor data and/or user data. It will take care of: Cloud-init is able to create and modify user accounts, set default passwords, and configure permissions. * **Execute user scripts**: - If any custom scripts were provided in the user data, cloud-init can run + If any custom scripts were provided in the user-data, cloud-init can run them. This allows additional specified software to be installed, security settings to be applied, etc. It can also inject SSH keys into the instance’s ``authorized_keys`` file, which allows secure remote access to the machine. diff --git a/doc/rtd/explanation/kernel-command-line.rst b/doc/rtd/explanation/kernel-command-line.rst index c7f861a6..bfcb9488 100644 --- a/doc/rtd/explanation/kernel-command-line.rst +++ b/doc/rtd/explanation/kernel-command-line.rst @@ -29,7 +29,7 @@ In order to allow an ephemeral, or otherwise pristine image to receive some configuration, ``cloud-init`` can read a URL directed by the kernel command line and proceed as if its data had previously existed. -This allows for configuring a metadata service, or some other data. +This allows for configuring an instance metadata service, or some other data. When :ref:`the local stage` runs, it will check to see if ``cloud-config-url`` appears in key/value fashion in the kernel command line, diff --git a/doc/rtd/explanation/vendordata.rst b/doc/rtd/explanation/vendordata.rst index 0e5e1881..1745f9cc 100644 --- a/doc/rtd/explanation/vendordata.rst +++ b/doc/rtd/explanation/vendordata.rst @@ -1,46 +1,46 @@ -.. _vendordata: +.. _vendor-data: -Vendor data +Vendor-data *********** Overview ======== -Vendor data is data provided by the entity that launches an instance (e.g., -the cloud provider). This data can be used to customise the image to fit into +Vendor-data is data provided by the entity that launches an instance (e.g., +the cloud provider). This data can be used to customize the image to fit into the particular environment it is being run in. -Vendor data follows the same rules as user data, with the following +Vendor-data follows the same rules as user-data, with the following caveats: -1. Users have ultimate control over vendor data. They can disable its +1. Users have ultimate control over vendor-data. They can disable its execution or disable handling of specific parts of multi-part input. 2. By default it only runs on first boot. -3. Vendor data can be disabled by the user. If the use of vendor data is - required for the instance to run, then vendor data should not be used. -4. User-supplied cloud-config is merged over cloud-config from vendor data. +3. Vendor-data can be disabled by the user. If the use of vendor-data is + required for the instance to run, then vendor-data should not be used. +4. User-supplied cloud-config is merged over cloud-config from vendor-data. Further, we strongly advise vendors to ensure you protect against any action that could compromise a system. Since users trust you, please take -care to make sure that any vendor data is safe, atomic, idempotent and does +care to make sure that any vendor-data is safe, atomic, idempotent and does not put your users at risk. Input formats ============= -``Cloud-init`` will download and cache to filesystem any vendor data that it -finds. Vendor data is handled exactly like -:ref:`user data`. This means that the vendor can supply -multi-part input and have those parts acted on in the same way as with user -data. +``Cloud-init`` will download and cache to filesystem any vendor-data that it +finds. Vendor-data is handled exactly like +:ref:`user-data`. This means that the vendor can supply +multi-part input and have those parts acted on in the same way as with +user-data. The only differences are: * Vendor-data-defined scripts are stored in a different location than user-data-defined scripts (to avoid namespace collision). * The user can disable part handlers via the cloud-config settings. - For example, to disable handling of 'part-handlers' in vendor data, - the user could provide user data like this: + For example, to disable handling of 'part-handlers' in vendor-data, + the user could provide user-data like this: .. code:: yaml diff --git a/doc/rtd/howto/debug_user_data.rst b/doc/rtd/howto/debug_user_data.rst index 0c555c35..b40ee509 100644 --- a/doc/rtd/howto/debug_user_data.rst +++ b/doc/rtd/howto/debug_user_data.rst @@ -1,17 +1,17 @@ .. _check_user_data_cloud_config: -How to validate user data cloud config +How to validate user-data cloud config ====================================== -The two most common issues with cloud config user data are: +The two most common issues with cloud config user-data are: 1. Incorrectly formatted YAML 2. The first line does not start with ``#cloud-config`` -Static user data validation +Static user-data validation --------------------------- -Cloud-init is capable of validating cloud config user data directly from +Cloud-init is capable of validating cloud config user-data directly from its datasource (i.e. on a running cloud instance). To do this, you can run: .. code-block:: shell-session diff --git a/doc/rtd/howto/debugging.rst b/doc/rtd/howto/debugging.rst index 546e8dd9..9d04a1cd 100644 --- a/doc/rtd/howto/debugging.rst +++ b/doc/rtd/howto/debugging.rst @@ -66,9 +66,9 @@ Cloud-init did not run Cloud-init ran, but didn't do what I want it to =============================================== -1. If you are using cloud-init's user data +1. If you are using cloud-init's user-data :ref:`cloud config`, make sure - to :ref:`validate your user data cloud config` + to :ref:`validate your user-data cloud config` 2. Check for errors in ``cloud-init status --long`` diff --git a/doc/rtd/howto/index.rst b/doc/rtd/howto/index.rst index b6811fa3..d3bc2552 100644 --- a/doc/rtd/howto/index.rst +++ b/doc/rtd/howto/index.rst @@ -18,10 +18,10 @@ How do I...? .. toctree:: :maxdepth: 1 - Run cloud-init locally before deploying + Launch cloud-init with... Re-run cloud-init Change how often a module runs - Validate my user data + Validate my user-data Debug cloud-init Check the status of cloud-init Report a bug diff --git a/doc/rtd/howto/launch_libvirt.rst b/doc/rtd/howto/launch_libvirt.rst new file mode 100644 index 00000000..3b481ed2 --- /dev/null +++ b/doc/rtd/howto/launch_libvirt.rst @@ -0,0 +1,30 @@ +.. _launch_libvirt: + +Run cloud-init locally with libvirt +*********************************** + +`Libvirt`_ is a tool for managing virtual machines and containers. + +Create your configuration +------------------------- + +.. include:: shared/create_config.txt + +Download a cloud image +---------------------- + +.. include:: shared/download_image.txt + +Create an instance +------------------ + +.. code-block:: shell-session + + virt-install --name cloud-init-001 --memory 4000 --noreboot \ + --os-variant detect=on,name=ubuntujammy \ + --disk=size=10,backing_store="$(pwd)/jammy-server-cloudimg-amd64.img" \ + --cloud-init user-data="$(pwd)/user-data,meta-data=$(pwd)/meta-data,network-config=$(pwd)/network-config" + +.. LINKS +.. _Libvirt: https://libvirt.org/ + diff --git a/doc/rtd/howto/launch_lxd.rst b/doc/rtd/howto/launch_lxd.rst new file mode 100644 index 00000000..c3f76f22 --- /dev/null +++ b/doc/rtd/howto/launch_lxd.rst @@ -0,0 +1,74 @@ +.. _launch_lxd: + +Run cloud-init locally with LXD +******************************** + +`LXD`_ offers a streamlined user experience for using Linux system containers. + +Create your configuration +------------------------- + +In this example we will create a file called ``user-data.yaml`` containing +a basic cloud-init configuration: + +.. code-block:: shell-session + + $ cat >user-data.yaml <` about the *user-data +cloud-config* format. + +Create your configuration +------------------------- + +.. include:: shared/create_config.txt + +Launch your instance +-------------------- + +You can pass the ``user-data`` file to Multipass and launch a Bionic instance +named ``test-vm`` with the following command: + +.. code-block:: shell-session + + $ multipass launch bionic --name test-vm --cloud-init user-data + +Multipass will validate the ``user-data`` configuration file before starting +the VM. This breaks all cloud-init configuration formats except the *user-data +cloud-config*. + +.. LINKS +.. _Multipass: https://multipass.run/ + diff --git a/doc/rtd/howto/launch_qemu.rst b/doc/rtd/howto/launch_qemu.rst new file mode 100644 index 00000000..49441828 --- /dev/null +++ b/doc/rtd/howto/launch_qemu.rst @@ -0,0 +1,67 @@ +.. _launch_qemu: + +Run cloud-init locally with QEMU +******************************** + +`QEMU`_ is a general purpose computer hardware emulator, able to run virtual +machines with hardware acceleration, and to emulate the instruction sets of +different architectures than the host you are running on. + +The ``NoCloud`` datasource allows you to provide your own user-data, +vendor-data, meta-data, or network configuration directly to an instance +without running a network service. This is helpful for launching local cloud +images with QEMU. + +Create your configuration +------------------------- + +.. include:: shared/create_config.txt + +Create an ISO disk +------------------ + +This disk is used to pass the configuration files to cloud-init. Create it with +the ``genisoimage`` command: + +.. code-block:: shell-session + + genisoimage \ + -output seed.img \ + -volid cidata -rational-rock -joliet \ + user-data meta-data network-config + +Download a cloud image +---------------------- + +.. include:: shared/download_image.txt + +.. note:: + This example uses emulated CPU instructions on non-x86 hosts, so it may be + slow. To make it faster on non-x86 architectures, one can change the image + type and ``qemu-system-`` command name to match the + architecture of your host machine. + +Boot the image with the ISO attached +------------------------------------ + +Boot the cloud image with our configuration, ``seed.img``, to QEMU: + +.. code-block:: shell-session + + $ qemu-system-x86_64 -m 1024 -net nic -net user \ + -drive file=jammy-server-cloudimg-amd64.img,index=0,format=qcow2,media=disk \ + -drive file=seed.img,index=1,media=cdrom \ + -machine accel=kvm:tcg + +The now-booted image will allow for login using the password provided above. + +For additional configuration, you can provide much more detailed +configuration in the empty :file:`network-config` and :file:`meta-data` files. + +.. note:: + See the :ref:`network_config_v2` page for details on the format and config + of network configuration. To learn more about the possible values for + metadata, check out the :ref:`datasource_nocloud` page. + +.. LINKS +.. _QEMU: https://www.qemu.org/ diff --git a/doc/rtd/tutorial/wsl.rst b/doc/rtd/howto/launch_wsl.rst similarity index 82% rename from doc/rtd/tutorial/wsl.rst rename to doc/rtd/howto/launch_wsl.rst index d1bb2180..39e4dd7e 100644 --- a/doc/rtd/tutorial/wsl.rst +++ b/doc/rtd/howto/launch_wsl.rst @@ -1,26 +1,18 @@ -.. _tutorial_wsl: +.. _launch_wsl: -WSL Tutorial -************ +Using WSL with cloud-init +************************* -In this tutorial, we will customize a Windows Subsystem for Linux (WSL) +In this guide, we will customize a `Windows Subsystem for Linux`_ (WSL) instance using cloud-init on Ubuntu. -How to use this tutorial -======================== - -In this tutorial, the commands in each code block can be copied and pasted -directly into a ``PowerShell`` Window . Omit the prompt before each -command, or use the "copy code" button on the right-hand side of the block, -which will copy the command for you without the prompt. - Prerequisites ============= -This tutorial assumes you are running within a ``Windows 11`` or ``Windows +This guide assumes you are running within a ``Windows 11`` or ``Windows Server 2022`` environment. If ``wsl`` is already installed, you must be running version 2. You can check your version of ``wsl`` by running the -following command: +following command in your terminal: .. code-block:: doscon @@ -38,9 +30,8 @@ Example output: DXCore version: 10.0.25131.1002-220531-1700.rs-onecore-base2-hyp Windows version: 10.0.20348.2402 -If running this tutorial within a virtualized -environment (`including in the cloud`_), ensure that -`nested virtualization`_ is enabled. +If you follow this guide while using a virtualized environment +(`including in the cloud`_), ensure that `nested virtualization`_ is enabled. Install WSL =========== @@ -66,10 +57,10 @@ Example output: Reboot the system when prompted. -Create our user data -==================== +Create some user-data +===================== -User data is the primary way for a user to customize a cloud-init instance. +User-data is the primary way for a user to customize a cloud-init instance. Open Notepad and paste the following: .. code-block:: yaml @@ -89,13 +80,13 @@ Ensure that the file is saved with the ``.user-data`` extension and not as a ``.txt`` file. .. note:: - We are creating user data that is tied to the instance we just created, - but by changing the filename, we can create user data that applies to - multiple or all WSL instances. See + We are creating user-data that is tied to the instance we just created, + but by changing the filename, we can create user-data that applies to + multiple (or all) WSL instances. See :ref:`WSL Datasource reference page` for more information. -What is user data? +What is user-data? ================== Before moving forward, let's inspect our :file:`user-data` file. @@ -112,7 +103,7 @@ We created the following contents: permissions: '0770' The first line starts with ``#cloud-config``, which tells cloud-init -what type of user data is in the config. Cloud-config is a YAML-based +what type of user-data is in the config. Cloud-config is a YAML-based configuration type that tells cloud-init how to configure the instance being created. Multiple different format types are supported by cloud-init. For more information, see the @@ -167,7 +158,7 @@ Download the Ubuntu 24.04 WSL image. PS> Invoke-WebRequest -Uri https://cloud-images.ubuntu.com/wsl/noble/current/ubuntu-noble-wsl-amd64-wsl.rootfs.tar.gz -OutFile wsl-images\ubuntu-noble-wsl-amd64-wsl.rootfs.tar.gz -Import the image into WSL storing it in the ``wsl-images`` directory. +Import the image into WSL, storing it in the ``wsl-images`` directory. .. code-block:: doscon @@ -186,8 +177,8 @@ Start the Ubuntu WSL instance PS> wsl --distribution Ubuntu-24.04 -Setup the Ubuntu WSL instance -============================= +Set up the Ubuntu WSL instance +============================== The Ubuntu WSL instance will start, and you may be prompted for a username and password. @@ -224,12 +215,12 @@ screen similar to the following: /root/.hushlogin file. root@machine:/mnt/c/Users/me# -You should now be in a shell inside the WSL instance. +This indicates you are now in a shell inside the WSL instance. Verify that ``cloud-init`` ran successfully ------------------------------------------- -Before validating the user data, let's wait for ``cloud-init`` to complete +Before validating the user-data, let's wait for ``cloud-init`` to complete successfully: .. code-block:: shell-session @@ -254,11 +245,11 @@ Which provides the following output: wsl -Verify our user data +Verify our user-data -------------------- Now we know that ``cloud-init`` has been successfully run, we can verify that -it received the expected user data we provided earlier: +it received the expected user-data we provided earlier: .. code-block:: shell-session @@ -275,7 +266,7 @@ Which should print the following to the terminal window: path: /var/tmp/hello-world.txt permissions: '0770' -We can also assert the user data we provided is a valid cloud-config: +We can also assert the user-data we provided is a valid cloud-config: .. code-block:: shell-session @@ -287,7 +278,7 @@ Which should print the following: Valid schema user-data -Finally, let us verify that our user data was applied successfully: +Finally, let us verify that our user-data was applied successfully: .. code-block:: shell-session @@ -299,13 +290,13 @@ Which should then print: Hello from cloud-init -We can see that ``cloud-init`` has received and consumed our user data +We can see that ``cloud-init`` has received and consumed our user-data successfully! What's next? ============ -In this tutorial, we used the :ref:`Write Files module ` to +In this guide, we used the :ref:`Write Files module ` to write a file to our WSL instance. The full list of modules available can be found in our :ref:`modules documentation`. Each module contains examples of how to use it. @@ -316,7 +307,7 @@ examples of more common use cases. Cloud-init's WSL reference documentation can be found on the :ref:`WSL Datasource reference page`. - +.. _Windows Subsystem for Linux: https://learn.microsoft.com/en-us/windows/wsl/ .. _including in the cloud: https://techcommunity.microsoft.com/t5/itops-talk-blog/how-to-setup-nested-virtualization-for-azure-vm-vhd/ba-p/1115338 .. _nested virtualization: https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/nested-virtualization .. _Ubuntu 24.04: https://apps.microsoft.com/detail/9nz3klhxdjp5 diff --git a/doc/rtd/howto/launching.rst b/doc/rtd/howto/launching.rst new file mode 100644 index 00000000..4dd160e0 --- /dev/null +++ b/doc/rtd/howto/launching.rst @@ -0,0 +1,31 @@ +.. _launching: + +Launch a local instance with cloud-init +*************************************** + +It’s very likely that you will want to test your cloud-init configuration +locally before deploying it to the cloud. + +Fortunately, there are several different virtual machine (VM) and container +tools ideal for this sort of local testing. + +Due to differences across platforms, initializing and launching instances with +cloud-init can vary. Here we present instructions for various platforms, or +links to instructions where platforms have provided their preferred methods for +using cloud-init. + +* :ref:`Launch with QEMU ` +* :ref:`Launch with LXD ` +* :ref:`Launch with Multipass ` +* :ref:`Launch with libvirt ` +* :ref:`Launch with WSL ` + +.. toctree:: + :maxdepth: 2 + :hidden: + + QEMU + LXD + Multipass + Libvirt + WSL diff --git a/doc/rtd/howto/module_run_frequency.rst b/doc/rtd/howto/module_run_frequency.rst index bbe87277..167fc940 100644 --- a/doc/rtd/howto/module_run_frequency.rst +++ b/doc/rtd/howto/module_run_frequency.rst @@ -30,7 +30,7 @@ Update :file:`/etc/cloud/cloud.cfg`: - final_message - power_state_change -Then your user data could then be: +Then your user-data could then be: .. code-block:: yaml diff --git a/doc/rtd/howto/rerun_cloud_init.rst b/doc/rtd/howto/rerun_cloud_init.rst index 9af4d19e..241ef9eb 100644 --- a/doc/rtd/howto/rerun_cloud_init.rst +++ b/doc/rtd/howto/rerun_cloud_init.rst @@ -34,7 +34,7 @@ think that it hasn't run yet. It will then re-run after a reboot. Run a single cloud-init module ------------------------------ -If you are using :ref:`user data cloud-config` +If you are using :ref:`user-data cloud-config` format, you might wish to re-run just a single configuration module. Cloud-init provides the ability to run a single module in isolation and separately from boot. This command is: @@ -53,7 +53,7 @@ Example output: This subcommand is not called by the init system. It can be called manually to load the configured datasource and run a single cloud-config module once, using -the cached user data and metadata after the instance has booted. +the cached instance-data after the instance has booted. .. note:: diff --git a/doc/rtd/howto/run_cloud_init_locally.rst b/doc/rtd/howto/run_cloud_init_locally.rst deleted file mode 100644 index 2510eadd..00000000 --- a/doc/rtd/howto/run_cloud_init_locally.rst +++ /dev/null @@ -1,224 +0,0 @@ -.. _run_cloud_init_locally: - -How to run ``cloud-init`` locally -********************************* - -It's very likely that you will want to test ``cloud-init`` locally before -deploying it to the cloud. Fortunately, there are several different virtual -machine (VM) and container tools that are ideal for this sort of local -testing. - -* :ref:`boot cloud-init with QEMU ` -* :ref:`boot cloud-init with LXD ` -* :ref:`boot cloud-init with Libvirt ` -* :ref:`boot cloud-init with Multipass ` - -.. _run_with_qemu: - -QEMU -==== - -`QEMU`_ is a general purpose computer hardware emulator that is capable of -running virtual machines with hardware acceleration as well as emulating the -instruction sets of different architectures than the host that you are -running on. - -The ``NoCloud`` datasource allows users to provide their own user data, -metadata, or network configuration directly to an instance without running a -network service. This is helpful for launching local cloud images with QEMU. - -Create your configuration -------------------------- - -We will leave the :file:`network-config` and :file:`meta-data` files empty, but -populate :file:`user-data` with a cloud-init configuration. You may edit the -:file:`network-config` and :file:`meta-data` files if you have a config to -provide. - -.. code-block:: shell-session - - $ touch network-config - $ touch meta-data - $ cat >user-data <` command name to match the - architecture of your host machine. - -Boot the image with the ISO attached ------------------------------------- - -Boot the cloud image with our configuration, :file:`seed.img`, to QEMU: - -.. code-block:: shell-session - - $ qemu-system-x86_64 -m 1024 -net nic -net user \ - -drive file=jammy-server-cloudimg-amd64.img,index=0,format=qcow2,media=disk \ - -drive file=seed.img,index=1,media=cdrom \ - -machine accel=kvm:tcg - -The now-booted image will allow for login using the password provided above. - -For additional configuration, users can provide much more detailed -configuration in the empty :file:`network-config` and :file:`meta-data` files. - -.. note:: - - See the :ref:`network_config_v2` page for details on the format and config - of network configuration. To learn more about the possible values for - metadata, check out the :ref:`datasource_nocloud` page. - -.. _run_with_lxd: - -LXD -=== - -`LXD`_ offers a streamlined user experience for using Linux system containers. -With LXD, the following command initialises a container with user data: - -.. code-block:: shell-session - - $ lxc init ubuntu-daily:jammy test-container - $ lxc config set test-container user.user-data - < userdata.yaml - $ lxc start test-container - -To avoid the extra commands this can also be done at launch: - -.. code-block:: shell-session - - $ lxc launch ubuntu-daily:jammy test-container --config=user.user-data="$(cat userdata.yaml)" - -Finally, a profile can be set up with the specific data if you need to -launch this multiple times: - -.. code-block:: shell-session - - $ lxc profile create dev-user-data - $ lxc profile set dev-user-data user.user-data - < cloud-init-config.yaml - $ lxc launch ubuntu-daily:jammy test-container -p default -p dev-user-data - -LXD configuration types ------------------------ - -The above examples all show how to pass user data. To pass other types of -configuration data use the configuration options specified below: - -+----------------+---------------------------+ -| Data | Configuration option | -+================+===========================+ -| user data | cloud-init.user-data | -+----------------+---------------------------+ -| vendor data | cloud-init.vendor-data | -+----------------+---------------------------+ -| network config | cloud-init.network-config | -+----------------+---------------------------+ - -See the LXD `Instance Configuration`_ docs for more info about configuration -values or the LXD `Custom Network Configuration`_ document for more about -custom network config. - -.. _run_with_libvirt: - -Libvirt -======= - -`Libvirt`_ is a tool for managing virtual machines and containers. - -Create your configuration -------------------------- - -We will leave the :file:`network-config` and :file:`meta-data` files empty, but -populate user-data with a cloud-init configuration. You may edit the -:file:`network-config` and :file:`meta-data` files if you have a config to -provide. - -.. code-block:: shell-session - - $ touch network-config - $ touch meta-data - $ cat >user-data <user-data <`. It will generally take the form of: @@ -103,10 +103,10 @@ System info keys ---------------- These keys are used for setup of ``cloud-init`` itself, or the datasource -or distro. Anything under ``system_info`` cannot be overridden by vendor data, -user data, or any other handlers or transforms. In some cases there may be a +or distro. Anything under ``system_info`` cannot be overridden by vendor-data, +user-data, or any other handlers or transforms. In some cases there may be a ``system_info`` key used for the distro, while the same key is used outside of -``system_info`` for a user data module. +``system_info`` for a user-data module. Both keys will be processed independently. * ``system_info``: Top-level key. @@ -128,7 +128,7 @@ Both keys will be processed independently. either ``ssh`` or ``sshd``. - ``network``: Top-level key for distro-specific networking configuration. - + ``renderers``: Prioritised list of networking configurations to try + + ``renderers``: Prioritized list of networking configurations to try on this system. The first valid entry found will be used. Options are: @@ -140,7 +140,7 @@ Both keys will be processed independently. * ``netbsd`` * ``openbsd`` - + ``activators``: Prioritised list of networking tools to try to activate + + ``activators``: Prioritized list of networking tools to try to activate network on this system. The first valid entry found will be used. Options are: @@ -228,7 +228,7 @@ instance. ``datasource_pkg_list`` ^^^^^^^^^^^^^^^^^^^^^^^ -Prioritised list of python packages to search when finding a datasource. +Prioritized list of python packages to search when finding a datasource. Automatically includes ``cloudinit.sources``. .. _base_config_datasource_list: @@ -236,7 +236,7 @@ Automatically includes ``cloudinit.sources``. ``datasource_list`` ^^^^^^^^^^^^^^^^^^^ -This key contains a prioritised list of datasources that ``cloud-init`` +This key contains a prioritized list of datasources that ``cloud-init`` attempts to discover on boot. By default, this is defined in :file:`/etc/cloud/cloud.cfg.d`. @@ -270,10 +270,10 @@ Format is a dict with ``enabled`` and ``prefix`` keys: ``allow_userdata`` ^^^^^^^^^^^^^^^^^^ -A boolean value to disable the use of user data. +A boolean value to disable the use of user-data. This allows custom images to prevent users from accidentally breaking closed appliances. Setting ``allow_userdata: false`` in the configuration will disable -``cloud-init`` from processing user data. +``cloud-init`` from processing user-data. ``manual_cache_clean`` ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/rtd/reference/breaking_changes.rst b/doc/rtd/reference/breaking_changes.rst index ce54e1c9..db59b62d 100644 --- a/doc/rtd/reference/breaking_changes.rst +++ b/doc/rtd/reference/breaking_changes.rst @@ -11,6 +11,20 @@ releases. many operating system vendors patch out breaking changes in cloud-init to ensure consistent behavior on their platform. +25.1 +==== + +/usr merge +---------- + +Cloud-init's packaging code no longer installs anything to ``/lib``. Instead, +anything that was installed to ``/lib`` is now installed to ``/usr/lib``. +This shouldn't affect any systemd-based distributions as they have all +transitioned to the ``/usr`` merge. However, this could affect older +stable releases, non-systemd and non-Linux distributions. See +`this commit `_ +for more details. + 24.3 ==== diff --git a/doc/rtd/reference/cli.rst b/doc/rtd/reference/cli.rst index 704ca068..c540df3d 100644 --- a/doc/rtd/reference/cli.rst +++ b/doc/rtd/reference/cli.rst @@ -39,7 +39,7 @@ Example output: init DEPRECATED: Initialize cloud-init and perform initial modules. modules DEPRECATED: Activate modules using a given configuration key. single Manually run a single module. Useful for testing during development. - query Query standardized instance metadata from the command line. + query Query standardized instance-data from the command line. features List defined features. analyze Devel tool: Analyze cloud-init logs and data. devel Run development tools. @@ -66,8 +66,8 @@ Possible subcommands include: events. * :command:`show`: show time-ordered report of the cost of operations during each boot stage. -* :command:`boot`: show timestamps from kernel initialisation, kernel finish - initialisation, and ``cloud-init`` start. +* :command:`boot`: show timestamps from kernel initialization, kernel finish + initialization, and ``cloud-init`` start. .. _cli_clean: @@ -141,7 +141,7 @@ configuration or testing changes to the network conversion logic itself. Use ``cloud-init``'s jinja template render to process **#cloud-config** or **custom-scripts**, injecting any variables from -:file:`/run/cloud-init/instance-data.json`. It accepts a user data file +:file:`/run/cloud-init/instance-data.json`. It accepts a user-data file containing the jinja template header ``## template: jinja`` and renders that content with any :file:`instance-data.json` variables present. @@ -161,7 +161,7 @@ Query if hotplug is enabled for a given subsystem. :command:`handle` ----------------- -Respond to newly added system devices by retrieving updated system metadata +Respond to newly added system devices by retrieving updated system meta-data and bringing up/down the corresponding device. :command:`enable` @@ -243,23 +243,23 @@ run only once due to semaphores in :file:`/var/lib/cloud/`. :command:`query` ---------------- -Query standardised cloud instance metadata crawled by ``cloud-init`` and stored -in :file:`/run/cloud-init/instance-data.json`. This is a convenience -command-line interface to reference any cached configuration metadata that -``cloud-init`` crawls when booting the instance. See :ref:`instance_metadata` +Query standardized instance-data crawled by ``cloud-init`` and +stored in :file:`/run/cloud-init/instance-data.json`. This is a convenience +command-line interface to reference any cached configuration meta-data that +``cloud-init`` crawls when booting the instance. See :ref:`instance-data` for more info. -* :command:`--all`: Dump all available instance data as JSON which can be +* :command:`--all`: Dump all available instance-data as JSON which can be queried. * :command:`--instance-data`: Optional path to a different :file:`instance-data.json` file to source for queries. -* :command:`--list-keys`: List available query keys from cached instance data. +* :command:`--list-keys`: List available query keys from cached instance-data. * :command:`--format`: A string that will use jinja-template syntax to render a string replacing. * :command:``: A dot-delimited variable path into the :file:`instance-data.json` object. -Below demonstrates how to list all top-level query keys that are standardised +Below demonstrates how to list all top-level query keys that are standardized aliases: .. code-block:: shell-session @@ -286,7 +286,7 @@ Example output: v1 vendordata -Here are a few examples of how to query standardised metadata from clouds: +Here are a few examples of how to query standardized meta-data from clouds: .. code-block:: shell-session @@ -298,7 +298,7 @@ Example output: aws # or openstack, azure, gce etc. -Any standardised ``instance-data`` under a key is aliased as a top-level +Any standardized ``instance-data`` under a key is aliased as a top-level key for convenience: .. code-block:: shell-session @@ -311,7 +311,7 @@ Example output: aws # or openstack, azure, gce etc. -One can also query datasource-specific metadata on EC2, e.g.: +One can also query datasource-specific meta-data on EC2, e.g.: .. code-block:: shell-session @@ -320,9 +320,9 @@ One can also query datasource-specific metadata on EC2, e.g.: .. note:: - The standardised instance data keys under **v#** are guaranteed not to + The standardized instance data keys under **v#** are guaranteed not to change behaviour or format. If using top-level convenience aliases for any - standardised instance data keys, the most value (highest **v#**) of that key + standardized instance data keys, the most value (highest **v#**) of that key name is what is reported as the top-level value. So these aliases act as a 'latest'. @@ -352,7 +352,7 @@ Validate cloud-config files using jsonschema. * :command:`-t SCHEMA_TYPE, --schema-type SCHEMA_TYPE`: The schema type to validate --config-file against. One of: cloud-config, network-config. Default: cloud-config. -* :command:`--system`: Validate the system cloud-config user data. +* :command:`--system`: Validate the system cloud-config user-data. * :command:`-d DOCS [cc_module ...], --docs DOCS [cc_module ...]`: Print schema module docs. Choices are: "all" or "space-delimited" ``cc_names``. diff --git a/doc/rtd/reference/custom_modules/custom_part_handlers.rst b/doc/rtd/reference/custom_modules/custom_part_handlers.rst index 501dc7af..58e41beb 100644 --- a/doc/rtd/reference/custom_modules/custom_part_handlers.rst +++ b/doc/rtd/reference/custom_modules/custom_part_handlers.rst @@ -15,13 +15,13 @@ The ``handle_part`` function takes 4 arguments and returns nothing. See the example for how exactly each argument is used. To use this part handler, it must be included in a MIME multipart file as -part of the :ref:`user data`. +part of the :ref:`user-data`. Since MIME parts are processed in order, a part handler part must precede -any parts with mime-types that it is expected to handle in the same user data. +any parts with mime-types that it is expected to handle in the same user-data. ``Cloud-init`` will then call the ``handle_part`` function once before it handles any parts, once per part received, and once after all parts have been -handled. These additional calls allow for initialisation or teardown before +handled. These additional calls allow for initialization or teardown before or after receiving any parts. Example diff --git a/doc/rtd/reference/datasources.rst b/doc/rtd/reference/datasources.rst index d195b9b7..3dbb6582 100644 --- a/doc/rtd/reference/datasources.rst +++ b/doc/rtd/reference/datasources.rst @@ -4,14 +4,14 @@ Datasources *********** Datasources are sources of configuration data for ``cloud-init`` that typically -come from the user (i.e., user data) or come from the cloud that created the -configuration drive (i.e., metadata). Typical user data includes files, -YAML, and shell scripts whereas typical metadata includes server name, +come from the user (i.e., user-data) or come from the cloud that created the +configuration drive (i.e., meta-data). Typical user-data includes files, +YAML, and shell scripts whereas typical meta-data includes server name, instance id, display name, and other cloud specific details. -Any metadata processed by ``cloud-init``'s datasources is persisted as +Any meta-data processed by ``cloud-init``'s datasources is persisted as :file:`/run/cloud-init/instance-data.json`. ``Cloud-init`` provides tooling to -quickly introspect some of that data. See :ref:`instance_metadata` for more +quickly introspect some of that data. See :ref:`instance-data` for more information. How to configure which datasource to use diff --git a/doc/rtd/reference/datasources/akamai.rst b/doc/rtd/reference/datasources/akamai.rst index 1a31e61f..0377e009 100644 --- a/doc/rtd/reference/datasources/akamai.rst +++ b/doc/rtd/reference/datasources/akamai.rst @@ -3,9 +3,9 @@ Akamai ****** -The Akamai datasource provides an interface to consume metadata on the `Akamai -Connected Cloud`_. This service is available at ``169.254.169.254`` and -``fd00:a9fe:a9fe::1`` from within the instance. +The Akamai datasource provides an interface to consume instance-data on the +`Akamai Connected Cloud`_. This service is available at ``169.254.169.254`` +and ``fd00:a9fe:a9fe::1`` from within the instance. .. _Akamai Connected Cloud: https://linode.com @@ -35,7 +35,8 @@ use no changes to the defaults should be necessary: :: * ``base_urls`` - The URLs used to access the metadata service over IPv4 and IPv6 respectively. + The URLs used to access the instance metadata service over IPv4 and IPv6 + respectively. * ``paths`` @@ -53,16 +54,16 @@ use no changes to the defaults should be necessary: :: * ``allow_dhcp`` - Allows this datasource to use dhcp to find an IPv4 address to fetch metadata - with during the local stage. + Allows this datasource to use dhcp to find an IPv4 address to fetch + instance-data with during the local stage. * ``allow_ipv4`` - Allow the use of IPv4 when fetching metadata during any stage. + Allow the use of IPv4 when fetching instance-data during any stage. * ``allow_ipv6`` - Allows the use of IPv6 when fetching metadata during any stage. + Allows the use of IPv6 when fetching instance-data during any stage. * ``preferred_mac_prefixes`` @@ -76,7 +77,7 @@ Configuration Overrides In some circumstances, the Akamai platform may send configurations overrides to instances via dmi data to prevent certain behavior that may not be supported based on the instance's region or configuration. For example, if deploying an -instance in a region that does not yet support metadata, both the local and -init stages will be disabled, preventing cloud-init from attempting to fetch -metadata. Configuration overrides sent this way will appears in the +instance in a region that does not yet support instance-data, both the local +and init stages will be disabled, preventing cloud-init from attempting to +fetch instance-data. Configuration overrides sent this way will appears in the ``baseboard-serial-number`` field. diff --git a/doc/rtd/reference/datasources/aliyun.rst b/doc/rtd/reference/datasources/aliyun.rst index 5121f53d..124c1ecf 100644 --- a/doc/rtd/reference/datasources/aliyun.rst +++ b/doc/rtd/reference/datasources/aliyun.rst @@ -6,12 +6,12 @@ Alibaba Cloud (AliYun) The ``AliYun`` datasource reads data from Alibaba Cloud ECS. Support is present in ``cloud-init`` since 0.7.9. -Metadata service -================ +Instance metadata service +========================= -The Alibaba Cloud metadata service is available at the well known URL +The Alibaba Cloud instance metadata service is available at the well known URL :file:`http://100.100.100.200/`. For more information see Alibaba Cloud ECS -on `metadata`_. +on `meta-data`_. Configuration ============= @@ -33,9 +33,9 @@ An example configuration with the default values is provided below: Versions -------- -Like the EC2 metadata service, Alibaba Cloud's metadata service provides -versioned data under specific paths. As of April 2018, there are only -``2016-01-01`` and ``latest`` versions. +Like the EC2 instance metadata service, Alibaba Cloud's instance metadata +service provides versioned data under specific paths. As of April 2018, there +are only ``2016-01-01`` and ``latest`` versions. It is expected that the dated version will maintain a stable interface but ``latest`` may change content at a future date. @@ -55,10 +55,10 @@ Example output: 2016-01-01 latest -Metadata --------- +Instance Metadata Service +------------------------- -Instance metadata can be queried at +The instance metadata service can be queried at :file:`http://100.100.100.200/2016-01-01/meta-data`: .. code-block:: shell-session @@ -89,12 +89,12 @@ Example output: vpc-cidr-block vpc-id -Userdata --------- +User-data +--------- -If provided, user data will appear at +If provided, user-data will appear at :file:`http://100.100.100.200/2016-01-01/user-data`. -If no user data is provided, this will return a 404. +If no user-data is provided, this will return a 404. .. code-block:: shell-session @@ -108,4 +108,4 @@ Example output: echo "Hello World." .. LINKS -.. _metadata: https://www.alibabacloud.com/help/zh/faq-detail/49122.htm +.. _meta-data: https://www.alibabacloud.com/help/zh/faq-detail/49122.htm diff --git a/doc/rtd/reference/datasources/altcloud.rst b/doc/rtd/reference/datasources/altcloud.rst index 19233404..3b962cc2 100644 --- a/doc/rtd/reference/datasources/altcloud.rst +++ b/doc/rtd/reference/datasources/altcloud.rst @@ -3,13 +3,13 @@ AltCloud ********* -The datasource AltCloud will be used to pick up user data on `RHEVm`_ and +The datasource AltCloud will be used to pick up user-data on `RHEVm`_ and `vSphere`_. RHEVm ===== -For `RHEVm`_ v3.0 the user data is injected into the VM using floppy +For `RHEVm`_ v3.0 the user-data is injected into the VM using floppy injection via the `RHEVm`_ dashboard "Custom Properties". The format of the "Custom Properties" entry must be: :: @@ -41,7 +41,7 @@ data to it using the `Delta Cloud`_. vSphere ======= -For VMWare's `vSphere`_ the user data is injected into the VM as an ISO +For VMWare's `vSphere`_ the user-data is injected into the VM as an ISO via the CD-ROM. This can be done using the `vSphere`_ dashboard by connecting an ISO image to the CD/DVD drive. @@ -50,7 +50,7 @@ set the CD/DVD drive when creating the vSphere VM to point to an ISO on the data store. .. note:: - The ISO must contain the user data. + The ISO must contain the user-data. For example, to pass the same ``simple_script.bash`` to vSphere: diff --git a/doc/rtd/reference/datasources/azure.rst b/doc/rtd/reference/datasources/azure.rst index 8cab989f..a3c6ffa0 100644 --- a/doc/rtd/reference/datasources/azure.rst +++ b/doc/rtd/reference/datasources/azure.rst @@ -3,7 +3,7 @@ Azure ***** -This datasource finds metadata and user data from the Azure cloud platform. +This datasource finds meta-data and user-data from the Azure cloud platform. The Azure cloud platform provides initial data to an instance via an attached CD formatted in UDF. This CD contains a :file:`ovf-env.xml` file that @@ -13,14 +13,14 @@ with the "endpoint". IMDS ==== -Azure provides the `instance metadata service (IMDS)`_, which is a REST service -on ``169.254.169.254`` providing additional configuration information to the -instance. ``Cloud-init`` uses the IMDS for: +Azure provides the `instance metadata service (IMDS)`_, which is a REST +service on ``169.254.169.254`` providing additional configuration information +to the instance. ``Cloud-init`` uses the IMDS for: - Network configuration for the instance which is applied per boot. - A pre-provisioning gate which blocks instance configuration until Azure fabric is ready to provision. -- Retrieving SSH public keys. ``Cloud-init`` will first try to utilise SSH +- Retrieving SSH public keys. ``Cloud-init`` will first try to utilize SSH keys returned from IMDS, and if they are not provided from IMDS then it will fall back to using the OVF file provided from the CD-ROM. There is a large performance benefit to using IMDS for SSH key retrieval, but in order to @@ -47,7 +47,7 @@ The settings that may be configured are: configuration. Default is True. * :command:`data_dir` - Path used to read metadata files and write crawled data. + Path used to read meta-data files and write crawled data. * :command:`disk_aliases` @@ -72,17 +72,17 @@ An example configuration with the default values is provided below: ephemeral0: /dev/disk/cloud/azure_resource -User data +User-data ========= -User data is provided to ``cloud-init`` inside the :file:`ovf-env.xml` file. -``Cloud-init`` expects that user data will be provided as a base64 encoded +User-data is provided to ``cloud-init`` inside the :file:`ovf-env.xml` file. +``Cloud-init`` expects that user-data will be provided as a base64 encoded value inside the text child of an element named ``UserData`` or ``CustomData``, which is a direct child of the ``LinuxProvisioningConfigurationSet`` (a sibling to ``UserName``). If both ``UserData`` and ``CustomData`` are provided, the behaviour is -undefined on which will be selected. In the example below, user data provided +undefined on which will be selected. In the example below, user-data provided is ``'this is my userdata'``. Example: diff --git a/doc/rtd/reference/datasources/cloudcix.rst b/doc/rtd/reference/datasources/cloudcix.rst index 9bd9a083..c5fb27be 100644 --- a/doc/rtd/reference/datasources/cloudcix.rst +++ b/doc/rtd/reference/datasources/cloudcix.rst @@ -3,8 +3,8 @@ CloudCIX ======== -`CloudCIX`_ serves metadata through an internal server, accessible at -``http://169.254.169.254/v1``. The metadata and userdata can be fetched at +`CloudCIX`_ serves meta-data through an internal server, accessible at +``http://169.254.169.254/v1``. The meta-data and user-data can be fetched at the ``/metadata`` and ``/userdata`` paths respectively. CloudCIX instances are identified by the dmi product name `CloudCIX`. @@ -24,10 +24,10 @@ CloudCIX datasource has the following config options: - *retries*: The number of times the datasource should try to connect to the - metadata service -- *timeout*: How long in seconds to wait for a response from the metadata + instance metadata service +- *timeout*: How long in seconds to wait for a response from the meta-data service - *sec_between_retries*: How long in seconds to wait between consecutive - requests to the metadata service + requests to the instance metadata service _CloudCIX: https://www.cloudcix.com/ diff --git a/doc/rtd/reference/datasources/cloudsigma.rst b/doc/rtd/reference/datasources/cloudsigma.rst index 50f255ef..41e7c5a3 100644 --- a/doc/rtd/reference/datasources/cloudsigma.rst +++ b/doc/rtd/reference/datasources/cloudsigma.rst @@ -3,7 +3,7 @@ CloudSigma ********** -This datasource finds metadata and user data from the `CloudSigma`_ cloud +This datasource finds meta-data and user-data from the `CloudSigma`_ cloud platform. Data transfer occurs through a virtual serial port of the `CloudSigma`_'s VM, and the presence of a network adapter is **NOT** a requirement. See `server context`_ in their public documentation for more @@ -15,21 +15,21 @@ Setting a hostname By default, the name of the server will be applied as a hostname on the first boot. -Providing user data +Providing user-data ------------------- -You can provide user data to the VM using the dedicated `meta field`_ in the +You can provide user-data to the VM using the dedicated `meta field`_ in the `server context`_ ``cloudinit-user-data``. By default, *cloud-config* format is expected there, and the ``#cloud-config`` header can be omitted. However, since this is a raw-text field you could provide any of the valid :ref:`config formats`. -You have the option to encode your user data using Base64. In order to do that +You have the option to encode your user-data using Base64. In order to do that you have to add the ``cloudinit-user-data`` field to the ``base64_fields``. The latter is a comma-separated field with all the meta fields having Base64-encoded values. -If your user data does not need an internet connection you can create a +If your user-data does not need an internet connection you can create a `meta field`_ in the `server context`_ ``cloudinit-dsmode`` and set "local" as the value. If this field does not exist, the default value is "net". diff --git a/doc/rtd/reference/datasources/cloudstack.rst b/doc/rtd/reference/datasources/cloudstack.rst index 05eff0e5..18fd1703 100644 --- a/doc/rtd/reference/datasources/cloudstack.rst +++ b/doc/rtd/reference/datasources/cloudstack.rst @@ -3,13 +3,13 @@ CloudStack ********** -`Apache CloudStack`_ exposes user data, metadata, user password, and account +`Apache CloudStack`_ exposes user-data, meta-data, user password, and account SSH key through the ``virtual router``. The datasource obtains the ``virtual router`` address via DHCP lease information given to the instance. -For more details on metadata and user data, refer to the +For more details on meta-data and user-data, refer to the `CloudStack Administrator Guide`_. -The following URLs provide to access user data and metadata from the Virtual +The following URLs provide to access user-data and meta-data from the Virtual Machine. ``data-server.`` is a well-known hostname provided by the CloudStack ``virtual router`` that points to the next ``UserData`` server (which is usually also the ``virtual router``). @@ -18,7 +18,7 @@ usually also the ``virtual router``). http://data-server./latest/user-data http://data-server./latest/meta-data - http://data-server./latest/meta-data/{metadata type} + http://data-server./latest/meta-data/{meta-data type} If ``data-server.`` cannot be resolved, ``cloud-init`` will try to obtain the ``virtual router``'s address from the system's DHCP leases. If that fails, @@ -45,7 +45,7 @@ The settings that may be configured are: The timeout value provided to ``urlopen`` for each individual http request. This is used both when selecting a ``metadata_url`` and when crawling - the metadata service. + the instance metadata service. Default: 50 diff --git a/doc/rtd/reference/datasources/configdrive.rst b/doc/rtd/reference/datasources/configdrive.rst index 8ce5e6a2..44d50d12 100644 --- a/doc/rtd/reference/datasources/configdrive.rst +++ b/doc/rtd/reference/datasources/configdrive.rst @@ -10,13 +10,13 @@ By default, ``cloud-init`` *always* considers this source to be a fully-fledged datasource. Instead, the typical behavior is to assume it is really only present to provide networking information. ``Cloud-init`` will copy the network information, apply it to the system, and then continue on. -The "full" datasource could then be found in the EC2 metadata service. If -this is not the case then the files contained on the located drive must -provide equivalents to what the EC2 metadata service would provide (which is -typical of the version 2 support listed below). +The "full" datasource could then be found in the EC2 instance metadata service. +If this is not the case then the files contained on the located drive must +provide equivalents to what the EC2 instance metadata service would provide +(which is typical of the version 2 support listed below). .. note:: - See `the config drive extension`_ and `metadata introduction`_ in the + See `the config drive extension`_ and `meta-data introduction`_ in the public documentation for more information. .. dropdown:: Version 1 (deprecated) @@ -105,7 +105,7 @@ networking to be up before user-data actions are run. instance-id: default: iid-dsconfigdrive -This is utilised as the metadata's instance-id. It should generally +This is utilized as the meta-data's instance-id. It should generally be unique, as it is what is used to determine "is this a new instance?". ``public-keys`` @@ -120,7 +120,7 @@ If present, these keys will be used as the public keys for the instance. This value overrides the content in ``authorized_keys``. .. note:: - It is likely preferable to provide keys via user data. + It is likely preferable to provide keys via user-data. ``user-data`` ------------- @@ -130,11 +130,11 @@ instance. This value overrides the content in ``authorized_keys``. user-data: default: None -This provides ``cloud-init`` user data. See :ref:`examples ` +This provides ``cloud-init`` user-data. See :ref:`examples ` for details of what needs to be present here. .. _OpenStack: http://www.openstack.org/ -.. _metadata introduction: https://docs.openstack.org/nova/latest/user/metadata.html#config-drives +.. _meta-data introduction: https://docs.openstack.org/nova/latest/user/metadata.html#config-drives .. _python-novaclient: https://github.com/openstack/python-novaclient .. _iso9660: https://en.wikipedia.org/wiki/ISO_9660 .. _vfat: https://en.wikipedia.org/wiki/File_Allocation_Table diff --git a/doc/rtd/reference/datasources/digitalocean.rst b/doc/rtd/reference/datasources/digitalocean.rst index 583c3abf..e8a14327 100644 --- a/doc/rtd/reference/datasources/digitalocean.rst +++ b/doc/rtd/reference/datasources/digitalocean.rst @@ -7,10 +7,10 @@ DigitalOcean The `DigitalOcean`_ datasource consumes the content served from DigitalOcean's -metadata service. This metadata service serves information about the -running droplet via http over the link local address ``169.254.169.254``. The -metadata API endpoints are fully described in the DigitalOcean -`metadata documentation`_. +instance metadata service. This instance metadata service serves information +about the running droplet via http over the link local address +``169.254.169.254``. The API endpoints are fully described in the DigitalOcean +`meta-data documentation`_. Configuration ============= @@ -24,12 +24,13 @@ DigitalOcean's datasource can be configured as follows: :: * ``retries`` - Specifies the number of times to attempt connection to the metadata service. + Specifies the number of times to attempt connection to the instance metadata + service. * ``timeout`` Specifies the timeout (in seconds) to wait for a response from the - metadata service. + instance metadata service. .. _DigitalOcean: http://digitalocean.com/ -.. _metadata documentation: https://developers.digitalocean.com/metadata/ +.. _meta-data documentation: https://developers.digitalocean.com/metadata/ diff --git a/doc/rtd/reference/datasources/e24cloud.rst b/doc/rtd/reference/datasources/e24cloud.rst index e2c125db..a8ffaca9 100644 --- a/doc/rtd/reference/datasources/e24cloud.rst +++ b/doc/rtd/reference/datasources/e24cloud.rst @@ -3,8 +3,8 @@ E24Cloud ******** -`E24Cloud`_ platform provides an AWS EC2 metadata service clone. It identifies -itself to guests using the DMI system-manufacturer +`E24Cloud`_ platform provides an AWS EC2 instance metadata service clone. It +identifies itself to guests using the DMI system-manufacturer (:file:`/sys/class/dmi/id/sys_vendor`). .. _E24Cloud: https://www.e24cloud.com/en/ diff --git a/doc/rtd/reference/datasources/ec2.rst b/doc/rtd/reference/datasources/ec2.rst index d28da5bc..3d2e8e89 100644 --- a/doc/rtd/reference/datasources/ec2.rst +++ b/doc/rtd/reference/datasources/ec2.rst @@ -7,10 +7,10 @@ The EC2 datasource is the oldest and most widely used datasource that ``cloud-init`` supports. This datasource interacts with a *magic* IP provided to the instance by the cloud provider (typically this IP is ``169.254.169.254``). At this IP a http server is provided to the -instance so that the instance can make calls to get instance user data and -instance metadata. +instance so that the instance can make calls to get instance user-data and +instance-data. -Metadata is accessible via the following URL: :: +The instance metadata service is accessible via the following URL: :: GET http://169.254.169.254/2009-04-04/meta-data/ ami-id @@ -29,16 +29,16 @@ Metadata is accessible via the following URL: :: reservation-id security-groups -User data is accessible via the following URL: :: +User-data is accessible via the following URL: :: GET http://169.254.169.254/2009-04-04/user-data 1234,fred,reboot,true | 4512,jimbo, | 173,,, -Note that there are multiple EC2 Metadata versions of this data provided -to instances. ``Cloud-init`` attempts to use the most recent API version it -supports in order to get the latest API features and instance-data. If a given -API version is not exposed to the instance, those API features will be -unavailable to the instance. +Note that there are multiple EC2 instance metadata service versions of this +data provided to instances. ``Cloud-init`` attempts to use the most recent API +version it supports in order to get the latest API features and instance-data. +If a given API version is not exposed to the instance, those API features will +be unavailable to the instance. +----------------+----------------------------------------------------------+ + EC2 version | supported instance-data/feature | @@ -49,8 +49,8 @@ unavailable to the instance. +----------------+----------------------------------------------------------+ | **2016-09-02** | Required for secondary IP address support. | +----------------+----------------------------------------------------------+ -| **2009-04-04** | Minimum supports EC2 API version for metadata and | -| | user data. | +| **2009-04-04** | Minimum supports EC2 API version for meta-data and | +| | user-data. | +----------------+----------------------------------------------------------+ To see which versions are supported by your cloud provider use the following @@ -82,8 +82,8 @@ The settings that may be configured are: ``metadata_urls`` ----------------- -This list of URLs will be searched for an EC2 metadata service. The first -entry that successfully returns a 200 response for +This list of URLs will be searched for an EC2 instance metadata service. The +first entry that successfully returns a 200 response for ``//meta-data/instance-id`` will be selected. Default: [``'http://169.254.169.254'``, ``'http://[fd00:ec2::254]'``, @@ -103,7 +103,7 @@ Default: 120 The timeout value provided to ``urlopen`` for each individual http request. This is used both when selecting a ``metadata_url`` and when crawling the -metadata service. +instance metadata service. Default: 50 @@ -111,7 +111,7 @@ Default: 50 ---------------------------------- Boolean (default: True) to allow ``cloud-init`` to configure any secondary -NICs and secondary IPs described by the metadata service. All network +NICs and secondary IPs described by the instance metadata service. All network interfaces are configured with DHCP (v4) to obtain a primary IPv4 address and route. Interfaces which have a non-empty ``ipv6s`` list will also enable DHCPv6 to obtain a primary IPv6 address and route. The DHCP response (v4 and diff --git a/doc/rtd/reference/datasources/exoscale.rst b/doc/rtd/reference/datasources/exoscale.rst index f6824b75..0f0fbde0 100644 --- a/doc/rtd/reference/datasources/exoscale.rst +++ b/doc/rtd/reference/datasources/exoscale.rst @@ -3,20 +3,20 @@ Exoscale ******** -This datasource supports reading from the metadata server used on the -`Exoscale platform`_. Use of the Exoscale datasource is recommended to benefit -from new features of the Exoscale platform. +This datasource supports reading from the instance metadata server (IMDS) used +on the `Exoscale platform`_. Use of the Exoscale datasource is recommended to +benefit from new features of the Exoscale platform. -The datasource relies on the availability of a compatible metadata server +The datasource relies on the availability of a compatible IMDS (``http://169.254.169.254`` is used by default) and its companion password server, reachable at the same address (by default on port 8080). -Crawling of metadata -==================== +Crawling the datasource +======================= -The metadata service and password server are crawled slightly differently: +The IMDS and password server are crawled slightly differently: -* The "metadata service" is crawled every boot. +* The IMDS is crawled every boot. * The password server is also crawled every boot (the Exoscale datasource forces the password module to run with "frequency always"). @@ -42,7 +42,7 @@ The following settings are available and can be set for the The settings available are: -* ``metadata_url``: The URL for the metadata service. +* ``metadata_url``: The URL for the IMDS. Defaults to ``http://169.254.169.254``. @@ -51,7 +51,7 @@ The settings available are: Defaults to ``1.0``. -* ``password_server_port``: The port (on the metadata server) on which the +* ``password_server_port``: The port (on the IMDS) on which the password server listens. Defaults to ``8080``. diff --git a/doc/rtd/reference/datasources/fallback.rst b/doc/rtd/reference/datasources/fallback.rst index 98283c0b..f5198aa2 100644 --- a/doc/rtd/reference/datasources/fallback.rst +++ b/doc/rtd/reference/datasources/fallback.rst @@ -5,7 +5,7 @@ Fallback/no datasource This is the fallback datasource when no other datasource can be selected. It is the equivalent of an empty datasource, in that it provides an empty string -as user data, and an empty dictionary as metadata. +as user-data, and an empty dictionary as meta-data. It is useful for testing, as well as for occasions when you do not need an actual datasource to meet your instance requirements (i.e. you just want to diff --git a/doc/rtd/reference/datasources/gce.rst b/doc/rtd/reference/datasources/gce.rst index 5f0dc77b..1e6aa4ed 100644 --- a/doc/rtd/reference/datasources/gce.rst +++ b/doc/rtd/reference/datasources/gce.rst @@ -3,17 +3,17 @@ Google Compute Engine ********************* -The GCE datasource gets its data from the internal compute metadata server. -Metadata can be queried at the URL -:file:`http://metadata.google.internal/computeMetadata/v1/` +The GCE datasource gets its data from the internal compute meta-data server. +The instance metadata service can be queried at the URL +:file:`http://meta-data.google.internal/computeMetadata/v1/` from within an instance. For more information see the `GCE metadata docs`_. -Currently, the default project and instance level metadata keys +Currently, the default project and instance level meta-data keys ``project/attributes/sshKeys`` and ``instance/attributes/ssh-keys`` are merged to provide ``public-keys``. ``user-data`` and ``user-data-encoding`` can be provided to ``cloud-init`` by -setting those custom metadata keys for an *instance*. +setting those custom meta-data keys for an *instance*. Configuration ============= @@ -33,7 +33,8 @@ The settings that may be configured are: * ``sec_between_retries`` - The amount of wait time between retries when crawling the metadata service. + The amount of wait time between retries when crawling the instance metadata + service. Default: 1 diff --git a/doc/rtd/reference/datasources/lxd.rst b/doc/rtd/reference/datasources/lxd.rst index c983f361..b1b09b80 100644 --- a/doc/rtd/reference/datasources/lxd.rst +++ b/doc/rtd/reference/datasources/lxd.rst @@ -3,16 +3,16 @@ LXD *** -The LXD datasource allows the user to provide custom user data, -vendor data, metadata and network-config to the instance without running +The LXD datasource allows the user to provide custom user-data, +vendor-data, meta-data and network-config to the instance without running a network service (or even without having a network at all). This datasource performs HTTP GETs against the `LXD socket device`_ which is provided to each running LXD container and VM as ``/dev/lxd/sock`` and represents all -instance-metadata as versioned HTTP routes such as: +instance-meta-data as versioned HTTP routes such as: - 1.0/meta-data - - 1.0/config/user.vendor-data - - 1.0/config/user.user-data + - 1.0/config/cloud-init.vendor-data + - 1.0/config/cloud-init.user-data - 1.0/config/user. The LXD socket device ``/dev/lxd/sock`` is only present on containers and VMs @@ -28,23 +28,23 @@ The LXD datasource is detected as viable by ``ds-identify`` during the ``/sys/class/dmi/id/board_name`` matches "LXD". The LXD datasource provides ``cloud-init`` with the ability to react to -metadata, vendor data, user data and network-config changes, and to render the +meta-data, vendor-data, user-data and network-config changes, and to render the updated configuration across a system reboot. -To modify which metadata, vendor data or user data are provided to the +To modify which meta-data, vendor-data or user-data are provided to the launched container, use either LXD profiles or ``lxc launch ... -c =""`` at initial container launch, by setting one of the following keys: -- ``cloud-init.vendor-data``: YAML which overrides any metadata values. +- ``cloud-init.vendor-data``: YAML which overrides any meta-data values. - ``cloud-init.network-config``: YAML representing either :ref:`network_config_v1` or :ref:`network_config_v2` format. - ``cloud-init.user-data``: YAML which takes precedence and overrides both - metadata and vendor data values. + meta-data and vendor-data values. - ``user.``: Keys prefixed with ``user.`` are included in - :ref:`instance data` under the ``ds.config`` key. These + :ref:`instance-data` under the ``ds.config`` key. These key value pairs are used in jinja :ref:`cloud-config` - and :ref:`user data scripts`. These key-value pairs may be + and :ref:`user-data scripts`. These key-value pairs may be inspected on a launched instance using ``cloud-init query ds.config``. .. note:: @@ -83,7 +83,7 @@ Hotplug Network hotplug functionality is supported for the LXD datasource as described in the :ref:`events` documentation. As hotplug functionality relies on the -cloud-provided network metadata, the LXD datasource will only meaningfully +cloud-provided network meta-data, the LXD datasource will only meaningfully react to a hotplug event if it has the configuration necessary to respond to the change. Practically, this means that even with hotplug enabled, **the default behavior for adding a new virtual NIC will result in no change**. diff --git a/doc/rtd/reference/datasources/nocloud.rst b/doc/rtd/reference/datasources/nocloud.rst index bf32ad34..03e7654c 100644 --- a/doc/rtd/reference/datasources/nocloud.rst +++ b/doc/rtd/reference/datasources/nocloud.rst @@ -22,22 +22,24 @@ discovery configuration can be delivered to cloud-init in different ways, but is different from the configurations that cloud-init uses to configure the instance at runtime. -user data +user-data --------- -User data is a :ref:`configuration format` that allows a +User-data is a :ref:`configuration format` that allows a user to configure an instance. -metadata --------- +meta-data +--------- -The ``meta-data`` file is a YAML-formatted file. +The ``meta-data`` file is a YAML-formatted file which contains cloud-provided +information to the instance. This is required to contain an ``instance-id``, +with other cloud-specific keys available. -vendor data +vendor-data ----------- -Vendor data may be used to provide default cloud-specific configurations which -may be overriden by user data. This may be useful, for example, to configure an +Vendor-data may be used to provide default cloud-specific configurations which +may be overriden by user-data. This may be useful, for example, to configure an instance with a cloud provider's repository mirror for faster package installation. @@ -49,7 +51,7 @@ cloud-specific network configurations, or a reasonable default is set by cloud-init (typically cloud-init brings up an interface using DHCP). Since NoCloud is a generic datasource, network configuration may be set the -same way as user data, metadata, vendor data. +same way as user-data, meta-data, vendor-data. See the :ref:`network configuration` documentation for information on network configuration formats. @@ -122,7 +124,7 @@ files which are stored in :file:`/etc/cloud/cloud.cfg.d`. Configuration sources ===================== -User-data, metadata, network config, and vendor data may be sourced from one +User-data, meta-data, network config, and vendor-data may be sourced from one of several possible locations, either locally or remotely. Source 1: Local filesystem @@ -342,10 +344,10 @@ For example, you can pass this line configuration to QEMU: :: -smbios type=1,serial=ds=nocloud;s=http://10.10.0.1:8000/__dmi.chassis-serial-number__/ -This will cause NoCloud to fetch the full metadata from a URL based on +This will cause NoCloud to fetch all data from a URL based on YOUR_SERIAL_NUMBER as seen in :file:`/sys/class/dmi/id/chassis_serial_number` -(kenv on FreeBSD) from http://10.10.0.1:8000/YOUR_SERIAL_NUMBER/meta-data after -the network initialisation is complete. +(kenv on FreeBSD) from http://10.10.0.1:8000/YOUR_SERIAL_NUMBER/ after +the network initialization is complete. Example: Creating a disk @@ -364,7 +366,7 @@ sufficient disk by following the following example. 2. At this stage you have three options: - a. Create a disk to attach with some user data and metadata: + a. Create a disk to attach with some user-data and meta-data: .. code-block:: sh @@ -408,13 +410,13 @@ sufficient disk by following the following example. -drive driver=raw,file=seed.iso,if=virtio .. note:: - Note that "passw0rd" was set as password through the user data above. There + Note that "passw0rd" was set as password through the user-data above. There is no password set on these images. .. note:: The ``instance-id`` provided (``iid-local01`` above) is what is used to determine if this is "first boot". So, if you are making updates to - user data you will also have to change the ``instance-id``, or start the + user-data you will also have to change the ``instance-id``, or start the disk fresh. Example ``meta-data`` diff --git a/doc/rtd/reference/datasources/none.rst b/doc/rtd/reference/datasources/none.rst index c46472d2..66bd32d2 100644 --- a/doc/rtd/reference/datasources/none.rst +++ b/doc/rtd/reference/datasources/none.rst @@ -6,7 +6,7 @@ None The data source ``None`` may be used when no other viable datasource is present on disk. This has two primary use cases: -1. Providing user data to cloud-init from on-disk configuration when +1. Providing user-data to cloud-init from on-disk configuration when no other datasource is present. 2. As a fallback for when a datasource is otherwise intermittently unavailable. @@ -18,18 +18,18 @@ completes, a warning is logged that DataSourceNone is being used. Configuration ============= -User data and meta data may be passed to cloud-init via system +User-data and meta-data may be passed to cloud-init via system configuration in :file:`/etc/cloud/cloud.cfg` or :file:`/etc/cloud/cloud.cfg.d/*.cfg`. ``userdata_raw`` ---------------- -A **string** containing the user data (including header) to be used by +A **string** containing the user-data (including header) to be used by cloud-init. ``metadata`` ------------- +------------- The metadata to be used by cloud-init. .. _datasource_none_example: diff --git a/doc/rtd/reference/datasources/nwcs.rst b/doc/rtd/reference/datasources/nwcs.rst index 19c9ddd6..3b003619 100644 --- a/doc/rtd/reference/datasources/nwcs.rst +++ b/doc/rtd/reference/datasources/nwcs.rst @@ -4,8 +4,8 @@ NWCS **** The NWCS datasource retrieves basic configuration values from the locally -accessible metadata service. All data is served over HTTP from the address -``169.254.169.254``. +accessible instance metadata service. All data is served over HTTP from the +address ``169.254.169.254``. Configuration ============= @@ -19,10 +19,10 @@ The NWCS datasource can be configured as follows: :: timeout: 2 wait: 2 -* ``url``: The URL used to acquire the metadata configuration. +* ``url``: The URL used to acquire the meta-data configuration. * ``retries``: Determines the number of times to attempt to connect to the - metadata service. + instance metadata service. * ``timeout``: Determines the timeout (in seconds) to wait for a response from - the metadata service + the instance metadata service * ``wait``: Determines the timeout in seconds to wait before retrying after accessible failure. diff --git a/doc/rtd/reference/datasources/opennebula.rst b/doc/rtd/reference/datasources/opennebula.rst index 2ad9d5c3..4b67cc9b 100644 --- a/doc/rtd/reference/datasources/opennebula.rst +++ b/doc/rtd/reference/datasources/opennebula.rst @@ -10,10 +10,10 @@ The `OpenNebula`_ (ON) datasource supports the contextualisation disk. .. `network configuration`_ in the public documentation for .. more information. -OpenNebula's virtual machines are contextualised (parametrised) by +OpenNebula's virtual machines are contextualized (parametrized) by CD-ROM image, which contains a shell script :file:`context.sh`, with custom variables defined on virtual machine start. There are no -fixed contextualisation variables, but the datasource accepts +fixed contextualization variables, but the datasource accepts many used and recommended across the documentation. Datasource configuration @@ -106,7 +106,7 @@ One or multiple SSH keys (separated by newlines) can be specified. USER_DATA USERDATA -``Cloud-init`` user data. +``Cloud-init`` user-data. Example configuration ===================== diff --git a/doc/rtd/reference/datasources/openstack.rst b/doc/rtd/reference/datasources/openstack.rst index 7072c4ea..0038f4a2 100644 --- a/doc/rtd/reference/datasources/openstack.rst +++ b/doc/rtd/reference/datasources/openstack.rst @@ -3,7 +3,8 @@ OpenStack ********* -This datasource supports reading data from the `OpenStack Metadata Service`_. +This datasource supports reading data from the +`OpenStack Instance Metadata Service`_. Discovery ========= @@ -21,8 +22,8 @@ checks the following environment attributes as a potential OpenStack platform: ``product_name=OpenStack Nova``. * ``DMI product_name``: Either ``Openstack Nova`` or ``OpenStack Compute``. * ``DMI chassis_asset_tag`` is ``HUAWEICLOUD``, ``OpenTelekomCloud``, - ``SAP CCloud VM``, ``OpenStack Nova`` (since 19.2) or - ``OpenStack Compute`` (since 19.2). + ``SAP CCloud VM``, ``Samsung Cloud Platform``, + ``OpenStack Nova`` (since 19.2) or ``OpenStack Compute`` (since 19.2). Configuration ============= @@ -36,9 +37,9 @@ The settings that may be configured are as follows: ``metadata_urls`` ----------------- -This list of URLs will be searched for an OpenStack metadata service. The -first entry that successfully returns a 200 response for ``/openstack`` -will be selected. +This list of URLs will be searched for an OpenStack IMDS (instance +metadata service). The first entry that successfully returns a 200 response +for ``/openstack`` will be selected. Default: ['http://169.254.169.254']) @@ -56,7 +57,7 @@ Default: -1 The timeout value provided to ``urlopen`` for each individual http request. This is used both when selecting a ``metadata_url`` and when crawling the -metadata service. +instance metadata service. Default: 10 @@ -72,7 +73,7 @@ Default: 5 ------------------------ A boolean specifying whether to configure the network for the instance based -on :file:`network_data.json` provided by the metadata service. When False, +on :file:`network_data.json` provided by the IMDS. When False, only configure DHCP on the primary NIC for this instance. Default: True @@ -93,37 +94,37 @@ An example configuration with the default values is provided below: apply_network_config: True -Vendor Data +Vendor-data =========== -The OpenStack metadata server can be configured to serve up vendor data, -which is available to all instances for consumption. OpenStack vendor data is +The OpenStack IMDS can be configured to serve up vendor-data, +which is available to all instances for consumption. OpenStack vendor-data is generally a JSON object. ``Cloud-init`` will look for configuration in the ``cloud-init`` attribute -of the vendor data JSON object. ``Cloud-init`` processes this configuration -using the same handlers as user data, so any formats that work for user -data should work for vendor data. +of the vendor-data JSON object. ``Cloud-init`` processes this configuration +using the same handlers as user-data, so any formats that work for user-data +should work for vendor-data. -For example, configuring the following as vendor data in OpenStack would +For example, configuring the following as vendor-data in OpenStack would upgrade packages and install ``htop`` on all instances: .. code-block:: json {"cloud-init": "#cloud-config\npackage_upgrade: True\npackages:\n - htop"} -For more general information about how ``cloud-init`` handles vendor data, +For more general information about how ``cloud-init`` handles vendor-data, including how it can be disabled by users on instances, see our -:ref:`explanation topic`. +:ref:`explanation topic`. -OpenStack can also be configured to provide "dynamic vendordata" +OpenStack can also be configured to provide "dynamic vendor-data" which is provided by the DynamicJSON provider and appears under a -different metadata path, :file:`/vendor_data2.json`. +different IMDS path, :file:`/vendor_data2.json`. ``Cloud-init`` will look for a ``cloud-init`` at the :file:`vendor_data2` path; if found, settings are applied after (and, hence, overriding) the -settings from static vendor data. Both sets of vendor data can be overridden -by user data. +settings from static vendor-data. Both sets of vendor-data can be overridden +by user-data. .. _datasource_ironic: @@ -159,4 +160,4 @@ Example using Ubuntu + GRUB2: $ grub-mkconfig -o /boot/efi/EFI/ubuntu/grub.cfg -.. _OpenStack Metadata Service: https://docs.openstack.org/nova/latest/admin/metadata-service.html +.. _OpenStack Instance Metadata Service: https://docs.openstack.org/nova/latest/admin/metadata-service.html diff --git a/doc/rtd/reference/datasources/oracle.rst b/doc/rtd/reference/datasources/oracle.rst index 05b93fbe..44167c10 100644 --- a/doc/rtd/reference/datasources/oracle.rst +++ b/doc/rtd/reference/datasources/oracle.rst @@ -3,7 +3,7 @@ Oracle ****** -This datasource reads metadata, vendor data and user data from +This datasource reads meta-data, vendor-data and user-data from `Oracle Compute Infrastructure`_ (OCI). Oracle platform @@ -13,14 +13,14 @@ OCI provides bare metal and virtual machines. In both cases, the platform identifies itself via DMI data in the chassis asset tag with the string ``'OracleCloud.com'``. -Oracle's platform provides a metadata service that mimics the ``2013-10-17`` -version of OpenStack metadata service. Initially, support for Oracle was done -via the OpenStack datasource. +Oracle's platform provides a instance metadata service that mimics the +``2013-10-17`` version of OpenStack instance metadata service. Initially, +support for Oracle was done via the OpenStack datasource. ``Cloud-init`` has a specific datasource for Oracle in order to: a. Allow and support the future growth of the OCI platform. -b. Address small differences between OpenStack and Oracle metadata +b. Address small differences between OpenStack and Oracle meta-data implementation. Configuration @@ -34,22 +34,23 @@ configuration (in :file:`/etc/cloud/cloud.cfg` or ---------------------------- A boolean, defaulting to False. If set to True on an OCI Virtual Machine, -``cloud-init`` will fetch networking metadata from Oracle's IMDS and use it -to configure the non-primary network interface controllers in the system. If -set to True on an OCI Bare Metal Machine, it will have no effect (though this -may change in the future). +``cloud-init`` will fetch networking meta-data from Oracle's instance metadata +service and use it to configure the non-primary network interface controllers +in the system. If set to True on an OCI Bare Metal Machine, it will have no +effect (though this may change in the future). ``max_wait`` ------------ An integer, defaulting to 30. The maximum time in seconds to wait for the -metadata service to become available. If the metadata service is not -available within this time, the datasource will fail. +instance metadata service to become available. If the instance metadata service +is not available within this time, the datasource will fail. ``timeout`` ----------- + An integer, defaulting to 5. The time in seconds to wait for a response from -the metadata service before retrying. +the instance metadata service before retrying. Example configuration --------------------- diff --git a/doc/rtd/reference/datasources/rbxcloud.rst b/doc/rtd/reference/datasources/rbxcloud.rst index 44b30b3e..0a99114a 100644 --- a/doc/rtd/reference/datasources/rbxcloud.rst +++ b/doc/rtd/reference/datasources/rbxcloud.rst @@ -3,16 +3,16 @@ Rbx Cloud ********* -The Rbx datasource consumes the metadata drive available on the `HyperOne`_ -and `Rootbox`_ platforms. +The Rbx datasource consumes the instance metadata drive available on the +`HyperOne`_ and `Rootbox`_ platforms. This datasource supports network configurations, hostname, user accounts and -user metadata. +user-data. -Metadata drive -============== +Instance metadata drive +======================= -Drive metadata is a `FAT`_-formatted partition with the ``CLOUDMD`` or +This drive is a `FAT`_-formatted partition with the ``CLOUDMD`` or ``cloudmd`` label on the system disk. Its contents are refreshed each time the virtual machine is restarted, if the partition exists. For more information see `HyperOne Virtual Machine docs`_. diff --git a/doc/rtd/reference/datasources/scaleway.rst b/doc/rtd/reference/datasources/scaleway.rst index b6ee2414..7bbfa1d2 100644 --- a/doc/rtd/reference/datasources/scaleway.rst +++ b/doc/rtd/reference/datasources/scaleway.rst @@ -2,10 +2,10 @@ Scaleway ******** -`Scaleway`_ datasource uses data provided by the Scaleway metadata service -to do initial configuration of the network services. +`Scaleway`_ datasource uses data provided by the Scaleway instance metadata +service to do initial configuration of the network services. -The metadata service is reachable at the following addresses : +The instance metadata service is reachable at the following addresses : * IPv4: ``169.254.42.42`` * IPv6: ``fd00:42::42`` @@ -26,31 +26,33 @@ the following information in the `/etc/cloud.cfg.d` directory:: * ``retries`` - Controls the maximum number of attempts to reach the metadata service. + Controls the maximum number of attempts to reach the instance metadata + service. * ``timeout`` - Controls the number of seconds to wait for a response from the metadata - service for one protocol. + Controls the number of seconds to wait for a response from the instance + metadata service for one protocol. * ``max_wait`` - Controls the number of seconds to wait for a response from the metadata - service for all protocols. + Controls the number of seconds to wait for a response from the instance + metadata service for all protocols. * ``metadata_urls`` - List of additional URLs to be used in an attempt to reach the metadata - service in addition to the existing ones. + List of additional URLs to be used in an attempt to reach the instance + metadata service in addition to the existing ones. -User Data +User-data ========= -cloud-init fetches user data using the metadata service using the `/user_data` -endpoint. Scaleway's documentation provides a detailed description on how to -use `userdata`_. One can also interact with it using the `userdata api`_. +cloud-init fetches user-data using the instance metadata service using the +`/user_data` endpoint. Scaleway's documentation provides a detailed description +on how to use `user-data`_. One can also interact with it using the +`user-data api`_. .. _Scaleway: https://www.scaleway.com -.. _userdata: https://www.scaleway.com/en/docs/compute/instances/api-cli/using-cloud-init/ -.. _userdata api: https://www.scaleway.com/en/developers/api/instance/#path-user-data-list-user-data +.. _user-data: https://www.scaleway.com/en/docs/compute/instances/api-cli/using-cloud-init/ +.. _user-data api: https://www.scaleway.com/en/developers/api/instance/#path-user-data-list-user-data diff --git a/doc/rtd/reference/datasources/smartos.rst b/doc/rtd/reference/datasources/smartos.rst index fc476649..b1e28852 100644 --- a/doc/rtd/reference/datasources/smartos.rst +++ b/doc/rtd/reference/datasources/smartos.rst @@ -3,7 +3,7 @@ SmartOS Datasource ****************** -This datasource finds metadata and user data from the SmartOS virtualisation +This datasource finds meta-data and user-data from the SmartOS virtualization platform (i.e., Joyent). Please see http://smartos.org/ for information about SmartOS. @@ -11,8 +11,8 @@ Please see http://smartos.org/ for information about SmartOS. SmartOS platform ================ -The SmartOS virtualisation platform uses metadata from the instance via the -second serial console. On Linux, this is :file:`/dev/ttyS1`. The data is +The SmartOS virtualization platform uses instance-data from the instance via +the second serial console. On Linux, this is :file:`/dev/ttyS1`. The data is provided via a simple protocol: * Something queries for the data, @@ -22,20 +22,20 @@ provided via a simple protocol: New versions of the SmartOS tooling will include support for Base64-encoded data. -Metadata channels -================= +Instance metadata channels +========================== -``Cloud-init`` supports three modes of delivering user data and metadata via +``Cloud-init`` supports three modes of delivering configuration data via the flexible channels of SmartOS. -1. User data is written to :file:`/var/db/user-data`: +1. User-data is written to :file:`/var/db/user-data`: - - As per the spec, user data is for consumption by the end user, not + - As per the spec, user-data is for consumption by the end user, not provisioning tools. - ``Cloud-init`` ignores this channel, other than writing it to disk. - Removal of the ``meta-data`` key means that :file:`/var/db/user-data` gets removed. - - A backup of previous metadata is maintained as + - A backup of previous meta-data is maintained as :file:`/var/db/user-data.`. ```` is the epoch time when ``cloud-init`` ran. @@ -47,19 +47,19 @@ the flexible channels of SmartOS. - Previous versions of ``user-script`` is written to :file:`/var/lib/cloud/scripts/per-boot.backup/99_user_script..` - is the epoch time when ``cloud-init`` ran. - - When the ``user-script`` metadata key goes missing, ``user-script`` is + - When the ``user-script`` meta-data key goes missing, ``user-script`` is removed from the file system, although a backup is maintained. - If the script does not start with a shebang (i.e., it starts with #!), or it is not an executable, ``cloud-init`` will add a shebang of "#!/bin/bash". -3. ``Cloud-init`` user data is treated like on other Clouds. +3. ``Cloud-init`` user-data is treated like on other Clouds. - This channel is used for delivering ``_all_ cloud-init`` instructions. - Scripts delivered over this channel must be well formed (i.e., they must have a shebang). -``Cloud-init`` supports reading the traditional metadata fields supported by +``Cloud-init`` supports reading the traditional meta-data fields supported by the SmartOS tools. These are: * ``root_authorized_keys`` @@ -110,7 +110,7 @@ Alternatively you can use the JSON patch method: ] The default cloud-config includes "script-per-boot". ``Cloud-init`` will still -ingest and write the user data, but will not execute it when you disable +ingest and write the user-data, but will not execute it when you disable the per-boot script handling. The cloud-config needs to be delivered over the ``cloud-init:user-data`` @@ -139,14 +139,14 @@ This list can be changed through the This means that ``user-script``, ``user-data`` and other values can be Base64 encoded. Since ``cloud-init`` can only guess whether or not something -is truly Base64 encoded, the following metadata keys are hints as to whether +is truly Base64 encoded, the following meta-data keys are hints as to whether or not to Base64 decode something: * ``base64_all``: Except for excluded keys, attempt to Base64 decode the values. If the value fails to decode properly, it will be returned in its text. * ``base64_keys``: A comma-delimited list of which keys are Base64 encoded. -* ``b64-``: For any key, if an entry exists in the metadata for +* ``b64-``: For any key, if an entry exists in the meta-data for ``'b64-'``, then ``'b64-'`` is expected to be a plain-text boolean indicating whether or not its value is encoded. * ``no_base64_decode``: This is a configuration setting diff --git a/doc/rtd/reference/datasources/upcloud.rst b/doc/rtd/reference/datasources/upcloud.rst index 21b95922..e481e6af 100644 --- a/doc/rtd/reference/datasources/upcloud.rst +++ b/doc/rtd/reference/datasources/upcloud.rst @@ -3,20 +3,20 @@ UpCloud ******* -The `UpCloud`_ datasource consumes information from UpCloud's `metadata -service`_. This metadata service serves information about the -running server via HTTP over the address ``169.254.169.254`` available in -every DHCP-configured interface. The metadata API endpoints are fully -described in `UpCloud API documentation`_. +The `UpCloud`_ datasource consumes information from UpCloud's +`instance metadata service`_. This instance metadata service serves information +about the running server via HTTP over the address ``169.254.169.254`` +available in every DHCP-configured interface. The meta-data API endpoints are +fully described in `UpCloud API documentation`_. -Providing user data +Providing user-data =================== -When creating a server, user data is provided by specifying it as +When creating a server, user-data is provided by specifying it as ``user_data`` in the API or via the server creation tool in the control panel. -User data is immutable during the server's lifetime, and can be removed by +User-data is immutable during the server's lifetime, and can be removed by deleting the server. .. _UpCloud: https://upcloud.com/ -.. _metadata service: https://upcloud.com/community/tutorials/upcloud-metadata-service/ +.. _instance metadata service: https://upcloud.com/community/tutorials/upcloud-metadata-service/ .. _UpCloud API documentation: https://developers.upcloud.com/1.3/8-servers/#metadata-service diff --git a/doc/rtd/reference/datasources/vmware.rst b/doc/rtd/reference/datasources/vmware.rst index cea24a4a..b7c3edf0 100644 --- a/doc/rtd/reference/datasources/vmware.rst +++ b/doc/rtd/reference/datasources/vmware.rst @@ -35,10 +35,10 @@ Datasource configuration ------------------------ * ``allow_raw_data``: true (enable) or false (disable) the VMware customization - using ``cloud-init`` metadata and user data directly. Since vSphere 7.0 + using ``cloud-init`` meta-data and user-data directly. Since vSphere 7.0 Update 3 version, users can create a Linux customization specification with - minimal ``cloud-init`` metadata and user data, and apply this specification - to a virtual machine. This datasource will parse the metadata and user data + minimal ``cloud-init`` meta-data and user-data, and apply this specification + to a virtual machine. This datasource will parse the meta-data and user-data and configure the virtual machine with them. See `Guest customization using cloud-init`_ for more information. @@ -118,8 +118,8 @@ VMware Tools configuration options. GuestInfo keys ============== -One method of providing meta, user, and vendor data is by setting the following -key/value pairs on a VM's ``extraConfig`` `property`_: +One method of providing meta-data, user-data, and vendor-data is by setting the +following key/value pairs on a VM's ``extraConfig`` `property`_: .. list-table:: :header-rows: 1 @@ -127,15 +127,15 @@ key/value pairs on a VM's ``extraConfig`` `property`_: * - Property - Description * - ``guestinfo.metadata`` - - A YAML or JSON document containing the ``cloud-init`` metadata. + - A YAML or JSON document containing the ``cloud-init`` meta-data. * - ``guestinfo.metadata.encoding`` - The encoding type for ``guestinfo.metadata``. * - ``guestinfo.userdata`` - - A YAML document containing the ``cloud-init`` user data. + - A YAML document containing the ``cloud-init`` user-data. * - ``guestinfo.userdata.encoding`` - The encoding type for ``guestinfo.userdata``. * - ``guestinfo.vendordata`` - - A YAML document containing the ``cloud-init`` vendor data. + - A YAML document containing the ``cloud-init`` vendor-data. * - ``guestinfo.vendordata.encoding`` - The encoding type for ``guestinfo.vendordata``. @@ -177,11 +177,11 @@ Instance data and lazy networks One of the hallmarks of ``cloud-init`` is :ref:`its use of instance-data and JINJA queries ` -- the -ability to write queries in user and vendor data that reference runtime +ability to write queries in user-data and vendor-data that reference runtime information present in :file:`/run/cloud-init/instance-data.json`. This works -well when the metadata provides all of the information up front, such as the +well when the meta-data provides all of the information up front, such as the network configuration. For systems that rely on DHCP, however, this -information may not be available when the metadata is persisted to disk. +information may not be available when the meta-data is persisted to disk. This datasource ensures that even if the instance is using DHCP to configure networking, the same details about the configured network are available in @@ -259,10 +259,10 @@ The above command will result in output similar to the below JSON: Redacting sensitive information (GuestInfo keys transport only) --------------------------------------------------------------- -Sometimes the ``cloud-init`` user data might contain sensitive information, +Sometimes the ``cloud-init`` user-data might contain sensitive information, and it may be desirable to have the ``guestinfo.userdata`` key (or other ``guestinfo`` keys) redacted as soon as its data is read by the datasource. -This is possible by adding the following to the metadata: +This is possible by adding the following to the meta-data: .. code-block:: yaml @@ -270,7 +270,7 @@ This is possible by adding the following to the metadata: - userdata - vendordata -When the above snippet is added to the metadata, the datasource will iterate +When the above snippet is added to the meta-data, the datasource will iterate over the elements in the ``redact`` array and clear each of the keys. For example, when the ``guestinfo`` transport is used, the above snippet will cause the following commands to be executed: @@ -312,8 +312,8 @@ Sometimes ``cloud-init`` may bring up the network, but it will not finish coming online before the datasource's ``setup`` function is called, resulting in a :file:`/var/run/cloud-init/instance-data.json` file that does not have the correct network information. It is possible to instruct the datasource to wait -until an IPv4 or IPv6 address is available before writing the instance data -with the following metadata properties: +until an IPv4 or IPv6 address is available before writing the instance-data +with the following meta-data properties: .. code-block:: yaml @@ -331,8 +331,8 @@ Walkthrough of GuestInfo keys transport The following series of steps is a demonstration of how to configure a VM with this datasource using the GuestInfo keys transport: -#. Create the metadata file for the VM. Save the following YAML to a file named - :file:`metadata.yaml`\: +#. Create the meta-data file for the VM. Save the following YAML to a file named + :file:`meta-data.yaml`\: .. code-block:: yaml @@ -346,7 +346,7 @@ this datasource using the GuestInfo keys transport: name: ens* dhcp4: yes -#. Create the userdata file :file:`userdata.yaml`\: +#. Create the user-data file :file:`user-data.yaml`\: .. code-block:: yaml @@ -399,15 +399,15 @@ this datasource using the GuestInfo keys transport: govc vm.power -off "${VM}" -#. Export the environment variables that contain the ``cloud-init`` metadata - and user data: +#. Export the environment variables that contain the ``cloud-init`` meta-data + and user-data: .. code-block:: shell - export METADATA=$(gzip -c9 /dev/null || base64; }) \ - USERDATA=$(gzip -c9 /dev/null || base64; }) + export METADATA=$(gzip -c9 /dev/null || base64; }) \ + USERDATA=$(gzip -c9 /dev/null || base64; }) -#. Assign the metadata and user data to the VM: +#. Assign the meta-data and user-data to the VM: .. code-block:: shell @@ -434,7 +434,7 @@ this datasource using the GuestInfo keys transport: If all went according to plan, the CentOS box is: -* Locked down, allowing SSH access only for the user in the user data. +* Locked down, allowing SSH access only for the user in the user-data. * Configured for a dynamic IP address via DHCP. * Has a hostname of ``cloud-vm``. @@ -444,34 +444,34 @@ Examples of common configurations Setting the hostname -------------------- -The hostname is set by way of the metadata key ``local-hostname``. +The hostname is set by way of the meta-data key ``local-hostname``. Setting the instance ID ----------------------- -The instance ID may be set by way of the metadata key ``instance-id``. However, -if this value is absent then the instance ID is read from the file +The instance ID may be set by way of the meta-data key ``instance-id``. +However, if this value is absent then the instance ID is read from the file :file:`/sys/class/dmi/id/product_uuid`. Providing public SSH keys ------------------------- -The public SSH keys may be set by way of the metadata key ``public-keys-data``. -Each newline-terminated string will be interpreted as a separate SSH public -key, which will be placed in distro's default user's +The public SSH keys may be set by way of the meta-data key +``public-keys-data``. Each newline-terminated string will be interpreted as a +separate SSH public key, which will be placed in distro's default user's :file:`~/.ssh/authorized_keys`. If the value is empty or absent, then nothing will be written to :file:`~/.ssh/authorized_keys`. Configuring the network ----------------------- -The network is configured by setting the metadata key ``network`` with a value +The network is configured by setting the meta-data key ``network`` with a value consistent with Network Config :ref:`Version 1 ` or :ref:`Version 2 `, depending on the Linux distro's version of ``cloud-init``. -The metadata key ``network.encoding`` may be used to indicate the format of -the metadata key ``network``. Valid encodings are ``base64`` and +The meta-data key ``network.encoding`` may be used to indicate the format of +the meta-data key ``network``. Valid encodings are ``base64`` and ``gzip+base64``. diff --git a/doc/rtd/reference/datasources/vultr.rst b/doc/rtd/reference/datasources/vultr.rst index 1115c29e..2f94cfa6 100644 --- a/doc/rtd/reference/datasources/vultr.rst +++ b/doc/rtd/reference/datasources/vultr.rst @@ -4,9 +4,9 @@ Vultr ***** The `Vultr`_ datasource retrieves basic configuration values from the locally -accessible metadata service. All data is served over HTTP from the address -``169.254.169.254``. The endpoints are documented in the -`metadata service documentation`_. +accessible instance metadata service. All data is served over HTTP from the +address ``169.254.169.254``. The endpoints are documented in the +`instance metadata service documentation`_. Configuration ============= @@ -20,13 +20,13 @@ Vultr's datasource can be configured as follows: :: timeout: 2 wait: 2 -* ``url``: The URL used to acquire the metadata configuration. +* ``url``: The URL used to acquire the meta-data configuration. * ``retries``: Determines the number of times to attempt to connect to the - metadata service. + instance metadata service. * ``timeout``: Determines the timeout (in seconds) to wait for a response from - the metadata service. + the instance metadata service. * ``wait``: Determines the timeout (in seconds) to wait before retrying after accessible failure. .. _Vultr: https://www.vultr.com/ -.. _metadata service documentation: https://www.vultr.com/metadata/ +.. _instance metadata service documentation: https://www.vultr.com/metadata/ diff --git a/doc/rtd/reference/datasources/wsl.rst b/doc/rtd/reference/datasources/wsl.rst index 6c11bfe1..54200548 100644 --- a/doc/rtd/reference/datasources/wsl.rst +++ b/doc/rtd/reference/datasources/wsl.rst @@ -14,7 +14,7 @@ Requirements ============== 1. **WSL interoperability must be enabled**. The datasource needs to execute - some Windows binaries to compute the possible locations of the user data + some Windows binaries to compute the possible locations of the user-data files. 2. **WSL automount must be enabled**. The datasource needs to access files in @@ -39,18 +39,18 @@ For more information about how to configure WSL, .. _wsl_user_data_configuration: -User data configuration +User-data configuration ======================== The WSL datasource relies exclusively on the Windows filesystem as the provider -of user data. Access to those files is provided by WSL itself unless disabled +of user-data. Access to those files is provided by WSL itself unless disabled by the user, thus the datasource doesn't require any special component running on the Windows host to provide such data. -User data can be supplied in any +User-data can be supplied in any :ref:`format supported by cloud-init`, such as YAML cloud-config files or shell scripts. At runtime, the WSL datasource looks for -user data in the following locations inside the Windows host filesystem, in the +user-data in the following locations inside the Windows host filesystem, in the order specified below. The WSL datasource will be enabled if cloud-init discovers at least one of the applicable config files described below. @@ -76,7 +76,7 @@ following paths: Then, if a file from (1) is not found, optional user-provided configuration will be looked for in the following order: -1. ``%USERPROFILE%\.cloud-init\.user-data`` holds user data for a +1. ``%USERPROFILE%\.cloud-init\.user-data`` holds user-data for a specific instance configuration. The datasource resolves the name attributed by WSL to the instance being initialized and looks for this file before any of the subsequent alternatives. Example: ``sid-mlkit.user-data`` matches an @@ -112,30 +112,30 @@ configurations from previous steps were found. .. note:: Some users may have configured case sensitivity for file names on Windows. - Note that user data files will still be matched case-insensitively. If there + Note that user-data files will still be matched case-insensitively. If there are both `InstanceName.user-data` and `instancename.user-data`, which one will be chosen is arbitrary and should not be relied on. Thus it's recommended to avoid that scenario to prevent confusion. -Since WSL instances are scoped by the Windows user, having the user data files +Since WSL instances are scoped by the Windows user, having the user-data files inside the ``%USERPROFILE%`` directory (typically ``C:\Users\``) ensures that WSL instance initialization won't be subject to naming conflicts if the Windows host is shared by multiple users. -Vendor and metadata -=================== +Vendor-data and meta-data +========================= -The current implementation doesn't allow supplying vendor data. -The reasoning is that vendor data adds layering, thus complexity, for no real -benefit to the user. Supplying vendor data could be relevant to WSL itself, if +The current implementation doesn't allow supplying vendor-data. +The reasoning is that vendor-data adds layering, thus complexity, for no real +benefit to the user. Supplying vendor-data could be relevant to WSL itself, if the subsystem was aware of cloud-init and intended to leverage it, which is not the case to the best of our knowledge at the time of this writing. -Most of what ``metadata`` is intended for is not applicable under WSL, such as -setting a hostname. Yet, the knowledge of ``metadata.instance-id`` is vital for -cloud-init. So, this datasource provides a default value but also supports -optionally sourcing metadata from a per-instance specific configuration file: +Most of what ``meta-data`` is intended for is not applicable under WSL, such as +setting a hostname. Yet, the knowledge of ``meta-data.instance-id`` is vital +for cloud-init. So, this datasource provides a default value but also supports +optionally sourcing meta-data from a per-instance specific configuration file: ``%USERPROFILE%\.cloud-init\.meta-data``. If that file exists, it is a YAML-formatted file minimally providing a value for instance ID such as: ``instance-id: x-y-z``. Advanced users looking to share @@ -154,9 +154,9 @@ files, please check the following restrictions: * File paths in an include file must be Linux absolute paths. - Users may be surprised with that requirement since the user data files are + Users may be surprised with that requirement since the user-data files are inside the Windows file system. But remember that cloud-init is still running - inside a Linux instance, and the files referenced in the include user data + inside a Linux instance, and the files referenced in the include user-data file will be read by cloud-init, thus they must be represented with paths understandable inside the Linux instance. Most users will find their Windows system drive mounted as `/mnt/c`, so let's consider that assumption in the diff --git a/doc/rtd/reference/datasources/zstack.rst b/doc/rtd/reference/datasources/zstack.rst index e1fefd21..76ea2152 100644 --- a/doc/rtd/reference/datasources/zstack.rst +++ b/doc/rtd/reference/datasources/zstack.rst @@ -3,8 +3,8 @@ ZStack ****** -ZStack platform provides an AWS EC2 metadata service, but with different -datasource identity. More information about ZStack can be found at +ZStack platform provides an AWS EC2 instance metadata service, but with +different datasource identity. More information about ZStack can be found at `ZStack`_. Discovery @@ -14,19 +14,19 @@ To determine whether a VM is running on the ZStack platform, ``cloud-init`` checks DMI information via ``dmidecode -s chassis-asset-tag``. If the output ends with ``.zstack.io``, it's running on the ZStack platform. -Metadata --------- +Instance Metadata Service +------------------------- -The same way as with EC2, instance metadata can be queried at: :: +The same way as with EC2, instance-data can be queried at: :: GET http://169.254.169.254/2009-04-04/meta-data/ instance-id local-hostname -User data +User-data --------- -The same way as with EC2, instance user data can be queried at: :: +The same way as with EC2, instance user-data can be queried at: :: GET http://169.254.169.254/2009-04-04/user-data/ meta_data.json diff --git a/doc/rtd/reference/examples_library.rst b/doc/rtd/reference/examples_library.rst index 43a34766..d2ba0aaa 100644 --- a/doc/rtd/reference/examples_library.rst +++ b/doc/rtd/reference/examples_library.rst @@ -4,7 +4,7 @@ Cloud config examples library ***************************** .. note:: - This page is an index to all the cloud config YAML examples, organised by + This page is an index to all the cloud config YAML examples, organized by operation or process. If you prefer to use a single-page summary containing every cloud config yaml example, refer to the :ref:`all examples page `. diff --git a/doc/rtd/reference/merging.rst b/doc/rtd/reference/merging.rst index 097892e2..11ae99db 100644 --- a/doc/rtd/reference/merging.rst +++ b/doc/rtd/reference/merging.rst @@ -1,10 +1,10 @@ .. _merging_user_data: -Merging user data sections +Merging user-data sections ************************** -The ability to merge user data sections allows a way to specify how -cloud-config YAML "dictionaries" provided as user data are handled when there +The ability to merge user-data sections allows a way to specify how +cloud-config YAML "dictionaries" provided as user-data are handled when there are multiple YAML files to be merged together (e.g., when performing an #include). @@ -100,7 +100,7 @@ Custom 3rd party mergers can be defined, for more info visit How to activate =============== -There are a few ways to activate the merging algorithms, and to customise them +There are a few ways to activate the merging algorithms, and to customize them for your own usage. 1. The first way involves the usage of MIME messages in ``cloud-init`` to @@ -166,15 +166,15 @@ merge with a cloud-config dictionary coming after it. Other uses ========== -In addition to being used for merging user data sections, the default merging +In addition to being used for merging user-data sections, the default merging algorithm for merging :file:`'conf.d'` YAML files (which form an initial YAML config for ``cloud-init``) was also changed to use this mechanism, to take -advantage of the full benefits (and customisation) here as well. Other places -that used the previous merging are also, similarly, now extensible (metadata +advantage of the full benefits (and customization) here as well. Other places +that used the previous merging are also, similarly, now extensible (meta-data merging, for example). Note, however, that merge algorithms are not used *across* configuration types. -As was the case before merging was implemented, user data will overwrite +As was the case before merging was implemented, user-data will overwrite :file:`'conf.d'` configuration without merging. Example cloud-config diff --git a/doc/rtd/reference/network-config-format-v1.rst b/doc/rtd/reference/network-config-format-v1.rst index 236c813c..1a1bf58a 100644 --- a/doc/rtd/reference/network-config-format-v1.rst +++ b/doc/rtd/reference/network-config-format-v1.rst @@ -3,7 +3,7 @@ Networking config Version 1 *************************** -This network configuration format lets users customise their instance's +This network configuration format lets users customize their instance's networking interfaces by assigning subnet configuration, virtual device creation (bonds, bridges, VLANs) routes and DNS configuration. @@ -231,6 +231,8 @@ Type ``vlan`` requires the following keys: - ``name``: Set the name of the VLAN - ``vlan_link``: Specify the underlying link via its ``name``. - ``vlan_id``: Specify the VLAN numeric id. +- ``mac_address``: Optional, specify VLAN subinterface MAC address. If not + set MAC address from physical interface is used. The following optional keys are supported: @@ -319,6 +321,7 @@ Subnet types are one of the following: - ``ipv6_dhcpv6-stateful``: Configure this interface with ``dhcp6``. - ``ipv6_dhcpv6-stateless``: Configure this interface with SLAAC and DHCP. - ``ipv6_slaac``: Configure address with SLAAC. +- ``manual`` : Manual configure this interface. When making use of ``dhcp`` or either of the ``ipv6_dhcpv6`` types, no additional configuration is needed in the subnet dictionary. diff --git a/doc/rtd/reference/network-config-format-v2.rst b/doc/rtd/reference/network-config-format-v2.rst index b8792fce..3bb0b0d4 100644 --- a/doc/rtd/reference/network-config-format-v2.rst +++ b/doc/rtd/reference/network-config-format-v2.rst @@ -65,8 +65,8 @@ currently being defined. There are two physically/structurally different classes of device definitions, and the ID field has a different interpretation for each: -Physical devices (e.g., ethernet, wifi) ---------------------------------------- +Physical devices (e.g., ethernet, Wi-Fi) +---------------------------------------- These can dynamically come and go between reboots and even during runtime (hotplugging). In the generic case, they can be selected by ``match:`` @@ -241,7 +241,7 @@ Example: :: Add static addresses to the interface in addition to the ones received through DHCP or RA. Each sequence entry is in CIDR notation, i.e., of the -form ``addr/prefixlen``. ``addr`` is an IPv4 or IPv6 address as recognised +form ``addr/prefixlen``. ``addr`` is an IPv4 or IPv6 address as recognized by ``inet_pton(3)`` and ``prefixlen`` the number of bits of the subnet. Example: ``addresses: [192.168.14.2/24, 2001:1::1/64]`` @@ -252,7 +252,7 @@ Example: ``addresses: [192.168.14.2/24, 2001:1::1/64]`` Deprecated, see `Netplan#default-routes`_. Set default gateway for IPv4/6, for manual address configuration. This requires setting ``addresses`` too. Gateway IPs must be in a form -recognised by ``inet_pton(3)`` +recognized by ``inet_pton(3)`` Example for IPv4: ``gateway4: 172.16.0.1`` Example for IPv6: ``gateway6: 2001:4::1`` diff --git a/doc/rtd/reference/network-config.rst b/doc/rtd/reference/network-config.rst index 61a12167..46d4d977 100644 --- a/doc/rtd/reference/network-config.rst +++ b/doc/rtd/reference/network-config.rst @@ -22,7 +22,7 @@ processed: - :file:`/run/cloud-init/network-config.json`: world-readable JSON containing the selected source network-config JSON used by cloud-init network renderers. -User data cannot change an instance's network configuration. In the absence +User-data cannot change an instance's network configuration. In the absence of network configuration in any of the above sources, ``cloud-init`` will write out a network configuration that will issue a DHCP request on a "first" network interface. @@ -67,9 +67,9 @@ networking configuration. Disabling network activation ============================ -Some datasources may not be initialised until after the network has been +Some datasources may not be initialized until after the network has been brought up. In this case, ``cloud-init`` will attempt to bring up the -interfaces specified by the datasource metadata using a network activator +interfaces specified by the datasource meta-data using a network activator discovered by `cloudinit.net.activators.select_activator`_. This behaviour can be disabled in the ``cloud-init`` configuration dictionary, @@ -125,11 +125,11 @@ The following datasources optionally provide network configuration: - :ref:`datasource_config_drive` - - `OpenStack Metadata Service Network`_ + - `OpenStack Instance Metadata Service Network`_ - :ref:`datasource_digital_ocean` - - `DigitalOcean JSON metadata`_ + - `DigitalOcean JSON meta-data`_ - :ref:`datasource_lxd` @@ -142,19 +142,19 @@ The following datasources optionally provide network configuration: - :ref:`datasource_openstack` - - `OpenStack Metadata Service Network`_ + - `OpenStack Instance Metadata Service Network`_ - :ref:`datasource_smartos` - - `SmartOS JSON Metadata`_ + - `SmartOS JSON Instance Metadata`_ - :ref:`datasource_upcloud` - - `UpCloud JSON metadata`_ + - `UpCloud JSON meta-data`_ - :ref:`datasource_vultr` - - `Vultr JSON metadata`_ + - `Vultr JSON meta-data`_ For more information on network configuration formats: @@ -320,10 +320,10 @@ Example output: .. _LXD: https://documentation.ubuntu.com/lxd/en/latest/cloud-init/#how-to-specify-network-configuration-data .. _NetworkManager: https://networkmanager.dev .. _Netplan: https://netplan.io/ -.. _DigitalOcean JSON metadata: https://developers.digitalocean.com/documentation/metadata/ -.. _OpenStack Metadata Service Network: https://specs.openstack.org/openstack/nova-specs/specs/liberty/implemented/metadata-service-network-info.html -.. _SmartOS JSON Metadata: https://eng.joyent.com/mdata/datadict.html -.. _UpCloud JSON metadata: https://developers.upcloud.com/1.3/8-servers/#metadata-service -.. _Vultr JSON metadata: https://www.vultr.com/metadata/ +.. _DigitalOcean JSON meta-data: https://developers.digitalocean.com/documentation/metadata/ +.. _OpenStack Instance Metadata Service Network: https://specs.openstack.org/openstack/nova-specs/specs/liberty/implemented/metadata-service-network-info.html +.. _SmartOS JSON Instance Metadata: https://eng.joyent.com/mdata/datadict.html +.. _UpCloud JSON meta-data: https://developers.upcloud.com/1.3/8-servers/#metadata-service +.. _Vultr JSON meta-data: https://www.vultr.com/metadata/ .. _cloudinit.net.activators.select_activator: https://github.com/canonical/cloud-init/blob/main/cloudinit/net/activators.py#L249 .. _FreeBSD.start_services: https://github.com/canonical/cloud-init/blob/main/cloudinit/net/freebsd.py#L46 diff --git a/doc/rtd/reference/performance_analysis.rst b/doc/rtd/reference/performance_analysis.rst index bbedb443..8c76e5c8 100644 --- a/doc/rtd/reference/performance_analysis.rst +++ b/doc/rtd/reference/performance_analysis.rst @@ -21,7 +21,7 @@ options are responsible. These subcommands default to reading :command:`analyze show` ^^^^^^^^^^^^^^^^^^^^^^^ -Parse and organise :file:`cloud-init.log` events by stage and include each +Parse and organize :file:`cloud-init.log` events by stage and include each sub-stage granularity with time delta reports. .. code-block:: shell-session diff --git a/doc/rtd/reference/user_files.rst b/doc/rtd/reference/user_files.rst index abbeb694..38b2b800 100644 --- a/doc/rtd/reference/user_files.rst +++ b/doc/rtd/reference/user_files.rst @@ -61,7 +61,7 @@ Configuration files - :file:`/etc/cloud/cloud.cfg` - :file:`/etc/cloud/cloud.cfg.d/*.cfg` -These files can define the modules that run during instance initialisation, +These files can define the modules that run during instance initialization, the datasources to evaluate on boot, as well as other settings. See the :ref:`configuration sources explanation` and diff --git a/doc/rtd/reference/yaml_examples/apt.rst b/doc/rtd/reference/yaml_examples/apt.rst index 106a734c..4e2f514d 100644 --- a/doc/rtd/reference/yaml_examples/apt.rst +++ b/doc/rtd/reference/yaml_examples/apt.rst @@ -44,7 +44,7 @@ irrespective of this setting. Specify mirrors =============== -* Default: auto select based on cloud metadata in EC2, the default is +* Default: auto select based on instance-data in EC2, the default is ``.archive.ubuntu.com``. One can either specify a URI to use as a mirror with the ``uri`` key, or a list @@ -57,7 +57,7 @@ DataSource. In EC2, that means using ``.ec2.archive.ubuntu.com``. If no mirror is provided by the DataSource, but ``search_dns`` is true, then search for DNS names ``-mirror`` in each of: -- FQDN of this host per cloud metadata +- FQDN of this host per cloud instance-data - localdomain - no domain (which would search domains listed in ``/etc/resolv.conf``) diff --git a/doc/rtd/reference/yaml_examples/datasources.rst b/doc/rtd/reference/yaml_examples/datasources.rst index dc3f3d1a..43a0a338 100644 --- a/doc/rtd/reference/yaml_examples/datasources.rst +++ b/doc/rtd/reference/yaml_examples/datasources.rst @@ -7,12 +7,12 @@ These examples show datasource configuration options for various datasources. The options shown are as follows: -* ``timeout``: The timeout value for a request at metadata service +* ``timeout``: The timeout value for a request at instance metadata service * ``max_wait``: The length of time to wait (in seconds) before giving up on - the metadata service. The actual total wait could be up to: + the instance metadata service. The actual total wait could be up to: ``len(resolvable_metadata_urls)*timeout`` -* ``metadata_url``: List of URLs to check for metadata services. There are no - default values for this field. +* ``metadata_url``: List of URLs to check for instance metadata services. There + are no default values for this field. EC2 === diff --git a/doc/rtd/reference/yaml_examples/disable_ec2_metadata.rst b/doc/rtd/reference/yaml_examples/disable_ec2_metadata.rst index e4047c11..34064bad 100644 --- a/doc/rtd/reference/yaml_examples/disable_ec2_metadata.rst +++ b/doc/rtd/reference/yaml_examples/disable_ec2_metadata.rst @@ -1,13 +1,13 @@ .. _cce-disable-ec2-metadata: -Disable AWS EC2 metadata -************************ +Disable AWS EC2 IMDS +******************** The default value for this module is ``false``. Setting it to ``true`` disables -the IPv4 routes to EC2 metadata. +the IPv4 routes to EC2 IMDS. For more details, refer to the -:ref:`disable EC2 metadata module ` schema. +:ref:`disable EC2 IMDS module ` schema. .. literalinclude:: ../../../module-docs/cc_disable_ec2_metadata/example1.yaml :language: yaml diff --git a/doc/rtd/reference/yaml_examples/launch_index.rst b/doc/rtd/reference/yaml_examples/launch_index.rst index 4012398a..cdfb2b61 100644 --- a/doc/rtd/reference/yaml_examples/launch_index.rst +++ b/doc/rtd/reference/yaml_examples/launch_index.rst @@ -6,7 +6,7 @@ Amazon EC2 launch index This configuration syntax can be provided to have a given set of cloud config data show up on a certain launch index (and not other launches). This is done by providing a key here which acts as a filter on the instance's -user data. When this key is absent (or non-integer) then the content of this +user-data. When this key is absent (or non-integer) then the content of this file will always be used for all launch-indexes (i.e. the default behavior). .. code-block:: yaml diff --git a/doc/rtd/reference/yaml_examples/mcollective.rst b/doc/rtd/reference/yaml_examples/mcollective.rst index 7ba5f705..879f16b6 100644 --- a/doc/rtd/reference/yaml_examples/mcollective.rst +++ b/doc/rtd/reference/yaml_examples/mcollective.rst @@ -8,7 +8,7 @@ For a full list of keys, refer to the :ref:`MCollective module ` schema. .. warning:: - The EC2 metadata service is a network service, and thus is readable by + The EC2 instance metadata service is a network service, and thus is readable by non-root users on the system (i.e. ``ec2metadata --user-data``). If you want security against this, use ``include-once`` + SSL URLs. diff --git a/doc/rtd/reference/yaml_examples/scripts.rst b/doc/rtd/reference/yaml_examples/scripts.rst index 2a441235..37df7d23 100644 --- a/doc/rtd/reference/yaml_examples/scripts.rst +++ b/doc/rtd/reference/yaml_examples/scripts.rst @@ -1,10 +1,10 @@ .. _cce-scripts: -Control vendor data use +Control vendor-data use *********************** -The use of :ref:`vendor data ` can be controlled by the user. -Vendor data can be used (or disabled) with an optional prefix. +The use of :ref:`vendor-data ` can be controlled by the user. +Vendor-data can be used (or disabled) with an optional prefix. For a full list of keys, refer to the :ref:`scripts vendor module ` docs. @@ -26,7 +26,7 @@ Example 2 Example 3 --------- -With this example, vendor data will not be processed. +With this example, vendor-data will not be processed. .. literalinclude:: ../../../module-docs/cc_scripts_vendor/example3.yaml :language: yaml diff --git a/doc/rtd/reference/yaml_examples/update_etc_hosts.rst b/doc/rtd/reference/yaml_examples/update_etc_hosts.rst index c1384aac..e32b2108 100644 --- a/doc/rtd/reference/yaml_examples/update_etc_hosts.rst +++ b/doc/rtd/reference/yaml_examples/update_etc_hosts.rst @@ -27,7 +27,7 @@ The strings ``$hostname`` and ``$fqdn`` are replaced in the template with the appropriate values -- either from the ``config-config`` ``fqdn``, or ``hostname`` if provided. -When absent, the cloud metadata will be checked for ``local-hostname`` which +When absent, the meta-data will be checked for ``local-hostname`` which can be split into ``.``. To make your modifications persist across a reboot, you must modify diff --git a/doc/rtd/reference/yaml_examples/update_hostname.rst b/doc/rtd/reference/yaml_examples/update_hostname.rst index c3a0597f..82f94367 100644 --- a/doc/rtd/reference/yaml_examples/update_hostname.rst +++ b/doc/rtd/reference/yaml_examples/update_hostname.rst @@ -42,11 +42,11 @@ This example sets the hostname to ``external.fqdn.me`` instead of ``myhost``. :language: yaml :linenos: -Override cloud metadata -======================= +Override meta-data +================== -Set the hostname to ``external`` instead of ``external.fqdn.me`` when cloud -metadata provides the ``local-hostname``: ``external.fqdn.me``. +Set the hostname to ``external`` instead of ``external.fqdn.me`` when +meta-data provides the ``local-hostname``: ``external.fqdn.me``. .. literalinclude:: ../../../module-docs/cc_update_hostname/example5.yaml :language: yaml diff --git a/doc/rtd/spelling_word_list.txt b/doc/rtd/spelling_word_list.txt index 0b2ba2d3..4da0b890 100644 --- a/doc/rtd/spelling_word_list.txt +++ b/doc/rtd/spelling_word_list.txt @@ -2,26 +2,19 @@ akamai alibaba almalinux ami -analyze ansible apk apport ar arg args -artifacts authkeys autoinstaller autospecced -avaliable aways aws -backend -backends baseurl -behavior bigstep -boolean bootcmd boothook boothooks @@ -32,15 +25,13 @@ cd centos chown chrony -cleanup -cloudinit +cloud-init cloudlinux cloudplatform conf config configdrive configs -copybutton cpu csr datasource @@ -48,6 +39,7 @@ datasources deadbeef debconf debian +dev devops dhcp dicts @@ -70,8 +62,6 @@ dss eal ec ecdsa -ed -edwardo errored es esm @@ -81,28 +71,20 @@ ethernet eurolinux execve exoscale -fabio faillog -favor -favorite fips firstboot -flavors flexibile fqdn freebsd freenode -fs-freq +fs fstab -galic gce -gotchas gpart growpart gz hacktoberfest -hetzner -honored hostname hpc ids @@ -120,7 +102,6 @@ kenv keygen keytypes kyler -labeled lastlog libvirt linux @@ -149,7 +130,6 @@ netplan networkd nistp nocloud -nonexistent ntp ntpd ntpdate @@ -171,8 +151,6 @@ pem pid pipelining pki -playbook -plugins postinstall poweroff ppc @@ -194,12 +172,8 @@ rbx rc rd readinessprobes -redhat -referesh regex -renderes repodata -repositoy resizefs resolv restructuredtext @@ -223,7 +197,6 @@ smtp snapd softlayer somedir -spelunking sr sshd ssk @@ -241,28 +214,20 @@ syslogd systemd tcp teardown -th timeframe timesyncd -timezone tinyssh tlb tmp tmpfiles tracebacks -transactional ua ubuntu udev udp un -unconfigured -unentitled -unredacted -unrendered url urls -userdata userspace usr util @@ -270,7 +235,6 @@ validator var vcloud ve -vendordata veth vfstype virtuozzo @@ -278,13 +242,10 @@ vm vpc vsphere vultr -walkthrough webserver wg wgx -whitepapers -whitespace -wifi +wi-fi wireguard xor yakkety diff --git a/doc/rtd/tutorial/index.rst b/doc/rtd/tutorial/index.rst index 58c2cce2..bcb5b0b2 100644 --- a/doc/rtd/tutorial/index.rst +++ b/doc/rtd/tutorial/index.rst @@ -3,47 +3,35 @@ Tutorials ********* -This section contains step-by-step tutorials to help you get started with -``cloud-init``. We hope our tutorials make as few assumptions as possible and -are accessible to anyone with an interest in ``cloud-init``. They should be a -great place to start learning about ``cloud-init``, how it works, and what it's -capable of. +Our step-by-step tutorials will help you learn about cloud-init and what it can +do. ------ +New user tutorial +================= -Core tutorial -============= +If you are completely new to cloud-init and would like a more thorough +introduction, we suggest starting with the +:ref:`new user tutorial `. -This tutorial, which we recommend if you are completely new to ``cloud-init``, -uses the QEMU emulator to introduce you to all of the key concepts, tools, -processes and operations that you will need to get started. +This tutorial uses the QEMU emulator to introduce you to all of the key +concepts, tools, processes and operations that you will need to use cloud-init +successfully. -.. toctree:: - :maxdepth: 1 +Further tutorials +================= - qemu.rst +This tutorial is recommended if you have some familiarity with cloud-init's key +concepts already. It uses LXD containers to show more of cloud-init's +capabilities. -Quick-start tutorial -==================== +* :ref:`Part 1: quick deployment ` -This tutorial is recommended if you have some familiarity with ``cloud-init`` -or the concepts around it, and are looking to get started as quickly as -possible. Here, you will use an LXD container to deploy a ``cloud-init`` -user data script. + Here we deploy a cloud-init user-data script into a LXD container. It + can also be used as a quick-start guide. .. toctree:: :maxdepth: 1 + :hidden: + qemu.rst lxd.rst - -WSL tutorial -============ - -This tutorial is for learning to use ``cloud-init`` within a ``WSL`` -environment. You will use a ``cloud-init`` user data script to customize a -``WSL`` instance. - -.. toctree:: - :maxdepth: 1 - - wsl.rst diff --git a/doc/rtd/tutorial/lxd.rst b/doc/rtd/tutorial/lxd.rst index 8bde79c8..50dded79 100644 --- a/doc/rtd/tutorial/lxd.rst +++ b/doc/rtd/tutorial/lxd.rst @@ -3,29 +3,18 @@ Quick-start tutorial with LXD ***************************** -In this tutorial, we will create our first ``cloud-init`` user data script -and deploy it into an `LXD`_ container. +In this tutorial, we will create our first cloud-init user-data script and +deploy it into a `LXD`_ container. Why LXD? ======== We'll be using LXD for this tutorial because it provides first class support -for ``cloud-init`` user data, as well as ``systemd`` support. Because it is -container based, it allows us to quickly test and iterate upon our user data +for cloud-init user-data, as well as ``systemd`` support. Because it is +container based, it allows us to quickly test and iterate upon our user-data definition. -How to use this tutorial -======================== - -In this tutorial, the commands in each code block can be copied and pasted -directly into the terminal. Omit the prompt (``$``) before each command, or -use the "copy code" button on the right-hand side of the block, which will copy -the command for you without the prompt. - -Each code block is preceded by a description of what the command does, and -followed by an example of the type of output you should expect to see. - -Install and initialise LXD +Install and initialize LXD ========================== If you already have LXD set up, you can skip this section. Otherwise, let's @@ -38,7 +27,7 @@ install LXD: If you don't have snap, you can install LXD using one of the `other installation options`_. -Now we need to initialise LXD. The minimal configuration will be enough for +Now we need to initialize LXD. The minimal configuration will be enough for the purposes of this tutorial. If you need to, you can always change the configuration at a later time. @@ -46,11 +35,12 @@ configuration at a later time. $ lxd init --minimal -Define our user data +Define our user-data ==================== -Now that LXD is set up, we can define our user data. Create the -following file on your local filesystem at :file:`/tmp/my-user-data`: +Now that LXD is set up, we can define our user-data. Create a file on your +local filesystem at :file:`/tmp/my-user-data` and populate it with this +content: .. code-block:: yaml @@ -58,23 +48,23 @@ following file on your local filesystem at :file:`/tmp/my-user-data`: runcmd: - echo 'Hello, World!' > /var/tmp/hello-world.txt -Here, we are defining our ``cloud-init`` user data in the -:ref:`#cloud-config` format, using the +Here, we are defining our cloud-init user-data in the +:ref:`#cloud-config ` format, using the :ref:`runcmd module ` to define a command to run. When applied, it will write ``Hello, World!`` to :file:`/var/tmp/hello-world.txt` (as we shall see later!). -Launch a LXD container with our user data +Launch a LXD container with our user-data ========================================= -Now that we have LXD set up and our user data defined, we can launch an -instance with our user data: +Now that we have LXD set up and our user-data defined, we can launch an +instance with our user-data: .. code-block:: shell-session $ lxc launch ubuntu:focal my-test --config=user.user-data="$(cat /tmp/my-user-data)" -Verify that ``cloud-init`` ran successfully +Verify that cloud-init ran successfully ------------------------------------------- After launching the container, we should be able to connect to our instance @@ -86,7 +76,7 @@ using: You should now be in a shell inside the LXD instance. -Before validating the user data, let's wait for ``cloud-init`` to complete +Before validating the user-data, let's wait for cloud-init to complete successfully: .. code-block:: shell-session @@ -99,11 +89,11 @@ Which provides the following output: status: done -Verify our user data +Verify our user-data -------------------- -Now we know that ``cloud-init`` has been successfully run, we can verify that -it received the expected user data we provided earlier: +Now we know that cloud-init ran successfully, we can verify that it +received the expected user-data we provided earlier: .. code-block:: shell-session @@ -117,7 +107,7 @@ Which should print the following to the terminal window: runcmd: - echo 'Hello, World!' > /var/tmp/hello-world.txt -We can also assert the user data we provided is a valid cloud-config: +We can also assert the user-data we provided is a valid cloud-config: .. code-block:: shell-session @@ -129,7 +119,7 @@ Which should print the following: Valid schema user-data -Finally, let us verify that our user data was applied successfully: +Finally, let us verify that our user-data was applied successfully: .. code-block:: shell-session @@ -141,13 +131,13 @@ Which should then print: Hello, World! -We can see that ``cloud-init`` has received and consumed our user data +We can see that cloud-init has received and consumed our user-data successfully! -Tear down -========= +Completion and next steps +========================= -Exit the container shell (by typing :command:`exit` or pressing :kbd:`ctrl-d`). +Exit the container shell (by typing :command:`exit` or pressing :kbd:`Ctrl-D`). Once we have exited the container, we can stop the container using: .. code-block:: shell-session @@ -160,9 +150,6 @@ We can then remove the container completely using: $ lxc rm my-test -What's next? -============ - In this tutorial, we used the :ref:`runcmd module ` to execute a shell command. The full list of modules available can be found in our :ref:`modules documentation`. diff --git a/doc/rtd/tutorial/qemu.rst b/doc/rtd/tutorial/qemu.rst index caa79cd3..22032dd6 100644 --- a/doc/rtd/tutorial/qemu.rst +++ b/doc/rtd/tutorial/qemu.rst @@ -1,7 +1,7 @@ .. _tutorial_qemu: -Core tutorial with QEMU -*********************** +New user tutorial with QEMU +*************************** .. toctree:: :titlesonly: @@ -10,73 +10,55 @@ Core tutorial with QEMU qemu-debugging.rst In this tutorial, we will launch an Ubuntu cloud image in a virtual machine -that uses ``cloud-init`` to pre-configure the system during boot. +that uses cloud-init to pre-configure the system during boot. The goal of this tutorial is to provide a minimal demonstration of -``cloud-init``, which you can then use as a development environment to test -your ``cloud-init`` configurations locally before launching to the cloud. +cloud-init, which you can then use as a development environment to test +your cloud-init configuration locally before launching it to the cloud. Why QEMU? ========= `QEMU`_ is a cross-platform emulator capable of running performant virtual -machines. QEMU is used at the core of a broad range of production operating -system deployments and open source software projects (including libvirt, LXD, -and vagrant) and is capable of running Windows, Linux, and Unix guest operating -systems. While QEMU is flexibile and feature-rich, we are using it because of -the broad support it has due to its broad adoption and ability to run on -\*nix-derived operating systems. +machines. QEMU is used at the core of a range of production operating system +deployments and open source software projects (including libvirt, LXD, +and vagrant). It is capable of running Windows, Linux, and Unix guest operating +systems. While QEMU is flexibile and feature-rich, we are using it because it +is widely supported and able to run on \*nix-derived operating systems. -How to use this tutorial -======================== +If you do not already have QEMU installed, you can install it by running the +following command in Ubuntu: -In this tutorial, the commands in each code block can be copied and pasted -directly into the terminal. Omit the prompt (``$``) before each command, or -use the "copy code" button on the right-hand side of the block, which will copy -the command for you without the prompt. - -Each code block is preceded by a description of what the command does, and -followed by an example of the type of output you should expect to see. - -Install QEMU -============ - -.. code-block:: sh +.. code-block:: bash $ sudo apt install qemu-system-x86 -If you are not using Ubuntu, you can visit QEMU's `install instructions`_ for -additional information. - -Create a temporary directory -============================ +If you are not using Ubuntu, you can visit QEMU's `install instructions`_ to +see details for your system. -This directory will store our cloud image and configuration files for -:ref:`user data`, :ref:`metadata`, and -:ref:`vendor data`. - -You should run all commands from this temporary directory. If you run the -commands from anywhere else, your virtual machine will not be configured. +Download a cloud image +====================== -Let's create a temporary directory and make it our current working directory -with :command:`cd`: +First, we'll set up a temporary directory that will store both our cloud image +and the configuration files we'll create in the next section. Let's also make +it our current working directory: -.. code-block:: sh +.. code-block:: bash $ mkdir temp $ cd temp -Download a cloud image -====================== +We will run all the commands from this temporary directory. If we run the +commands from anywhere else, our virtual machine will not be configured. -Cloud images typically come with ``cloud-init`` pre-installed and configured to -run on first boot. You will not need to worry about installing ``cloud-init`` +Cloud images typically come with cloud-init pre-installed and configured to +run on first boot. We don't need to worry about installing cloud-init for now, since we are not manually creating our own image in this tutorial. -In our case, we want to select the latest Ubuntu LTS_. Let's download the +In our case, we want to select the latest `Ubuntu LTS`_. Let's download the server image using :command:`wget`: -.. code-block:: sh +.. code-block:: bash $ wget https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img @@ -86,18 +68,32 @@ server image using :command:`wget`: type and :spelling:ignore:`qemu-system-` command name to match the architecture of your host machine. -Define our user data -==================== +Define the configuration data files +=================================== + +When we launch an instance using cloud-init, we pass different types of +configuration data files to it. Cloud-init uses these as a blueprint for how to +configure the virtual machine instance. There are three major types: + +* :ref:`user-data ` is provided by the user, and cloud-init + recognizes many different formats. +* :ref:`vendor-data ` is provided by the cloud provider. +* :ref:`meta-data ` contains the platform data, including + things like machine ID, hostname, etc. + +There is a specific user-data format called "*cloud-config*" that is probably +the most commonly used, so we will create an example of this (and examples of +both vendor-data and meta-data files), then pass them all to cloud-init. -Now we need to create our :file:`user-data` file. This user data cloud-config -sets the password of the default user, and sets that password to never expire. -For more details you can refer to the -:ref:`Set Passwords module page`. +Let's create our :file:`user-data` file first. The user-data *cloud-config* +is a YAML-formatted file, and in this example it sets the password of the +default user, and sets that password to never expire. For more details you can +refer to the :ref:`Set Passwords module page`. -Run the following command, which creates a file named :file:`user-data` -containing our configuration data. +Run the following command to create the user-data file (named +:file:`user-data`) containing our configuration data. -.. code-block:: sh +.. code-block:: bash $ cat << EOF > user-data #cloud-config @@ -107,14 +103,11 @@ containing our configuration data. EOF -What is user data? -================== +Before moving forward, let's first inspect our :file:`user-data` file. -Before moving forward, let's inspect our :file:`user-data` file. +.. code-block:: bash -.. code-block:: sh - - $ cat user-data + cat user-data You should see the following contents: @@ -125,40 +118,31 @@ You should see the following contents: chpasswd: expire: False -The first line starts with ``#cloud-config``, which tells ``cloud-init`` -what type of user data is in the config. Cloud-config is a YAML-based -configuration type that tells ``cloud-init`` how to configure the virtual -machine instance. Multiple different format types are supported by -``cloud-init``. For more information, see the -:ref:`documentation describing different formats`. - -The second line, ``password: password``, as per -:ref:`the Users and Groups module docs`, sets the default -user's password to ``password``. +* The first line starts with ``#cloud-config``, which tells cloud-init what + type of user-data is in the config file. -The third and fourth lines direct ``cloud-init`` to not require a password -reset on first login. +* The second line, ``password: password`` sets the default user's password to + ``password``, as per the :ref:`Users and Groups ` + module documentation. -Define our metadata -=================== +* The third and fourth lines tell cloud-init not to require a password reset + on first login. Now let's run the following command, which creates a file named -:file:`meta-data` containing configuration data. +:file:`meta-data` containing the instance ID we want to associate to the +virtual machine instance. -.. code-block:: sh +.. code-block:: bash $ cat << EOF > meta-data instance-id: someid/somehostname EOF -Define our vendor data -====================== - -Now we will create the empty file :file:`vendor-data` in our temporary +Next, let's create an empty file called :file:`vendor-data` in our temporary directory. This will speed up the retry wait time. -.. code-block:: sh +.. code-block:: bash $ touch vendor-data @@ -166,44 +150,38 @@ directory. This will speed up the retry wait time. Start an ad hoc IMDS webserver ============================== -Open up a second terminal window, change to your temporary directory and then -start the built-in Python webserver: - -.. code-block:: sh +Instance Metadata Service (IMDS) is a service used by most cloud providers +as a way to expose information to virtual machine instances. This service is +the primary mechanism for some clouds to expose cloud-init configuration data +to the instance. - $ cd temp - $ python3 -m http.server --directory . +The IMDS uses a private http webserver to provide instance-data to each running +instance. During early boot, cloud-init sets up network access and queries this +webserver to gather configuration data. This allows cloud-init to configure +the operating system while it boots. -What is an IMDS? ----------------- - -Instance Metadata Service (IMDS) is a service provided by most cloud providers -as a means of providing information to virtual machine instances. This service -is used by cloud providers to expose information to a virtual machine. This -service is used for many different things, and is the primary mechanism for -some clouds to expose ``cloud-init`` configuration data to the instance. +In this tutorial we are emulating this workflow using QEMU and a simple Python +webserver. This workflow is suitable for developing and testing cloud-init +configurations before deploying to a cloud. -How does ``cloud-init`` use the IMDS? -------------------------------------- +Open up a second terminal window, and in that window, run the following +commands to change to the temporary directory and then start the built-in +Python webserver: -The IMDS uses a private http webserver to provide metadata to each operating -system instance. During early boot, ``cloud-init`` sets up network access and -queries this webserver to gather configuration data. This allows ``cloud-init`` -to configure your operating system while it boots. +.. code-block:: bash -In this tutorial we are emulating this workflow using QEMU and a simple Python -webserver. This workflow is suitable for developing and testing -``cloud-init`` configurations prior to cloud deployments. + $ cd temp + $ python3 -m http.server --directory . -Launch a virtual machine with our user data -=========================================== +Launch a VM with our user-data +=============================== -Switch back to your original terminal, and run the following command so we can -launch our virtual machine. By default, QEMU will print the kernel logs and +Switch back to your original terminal, and run the following command to launch +our virtual machine. By default, QEMU will print the kernel logs and ``systemd`` logs to the terminal while the operating system boots. This may take a few moments to complete. -.. code-block:: sh +.. code-block:: bash $ qemu-system-x86_64 \ -net nic \ @@ -218,9 +196,6 @@ take a few moments to complete. If the output stopped scrolling but you don't see a prompt yet, press :kbd:`Enter` to get to the login prompt. -How is QEMU configured for ``cloud-init``? ------------------------------------------- - When launching QEMU, our machine configuration is specified on the command line. Many things may be configured: memory size, graphical output, networking information, hard drives and more. @@ -228,19 +203,19 @@ information, hard drives and more. Let us examine the final two lines of our previous command. The first of them, :command:`-hda jammy-server-cloudimg-amd64.img`, tells QEMU to use the cloud image as a virtual hard drive. This will cause the virtual machine to -boot Ubuntu, which already has ``cloud-init`` installed. +boot Ubuntu, which already has cloud-init installed. -The second line tells ``cloud-init`` where it can find user data, using the -:ref:`NoCloud datasource`. During boot, ``cloud-init`` +The second line tells cloud-init where it can find user-data, using the +:ref:`NoCloud datasource`. During boot, cloud-init checks the ``SMBIOS`` serial number for ``ds=nocloud``. If found, -``cloud-init`` will use the specified URL to source its user data config files. +cloud-init will use the specified URL to source its user-data config files. In this case, we use the default gateway of the virtual machine (``10.0.2.2``) and default port number of the Python webserver (``8000``), so that -``cloud-init`` will, inside the virtual machine, query the server running on +cloud-init will, inside the virtual machine, query the server running on host. -Verify that ``cloud-init`` ran successfully +Verify that cloud-init ran successfully =========================================== After launching the virtual machine, we should be able to connect to our @@ -254,13 +229,10 @@ If you can log in using the configured password, it worked! If you couldn't log in, see :ref:`this page for debug information`. -Check ``cloud-init`` status -=========================== +Let's now check cloud-init's status. Run the following command, which will +allow us to check if cloud-init has finished running: -Run the following command, which will allow us to check if ``cloud-init`` has -finished running: - -.. code-block:: sh +.. code-block:: bash $ cloud-init status --wait @@ -269,20 +241,17 @@ If you see ``status: done`` in the output, it succeeded! If you see a failed status, you'll want to check :file:`/var/log/cloud-init.log` for warning/error messages. -Tear down -========= +Completion and next steps +========================= -In our main terminal, let's exit the QEMU shell using :kbd:`ctrl-a x` (that's -:kbd:`ctrl` and :kbd:`a` simultaneously, followed by :kbd:`x`). +In our main terminal, let's exit the QEMU shell using :kbd:`Ctrl-A X` (that's +:kbd:`Ctrl` and :kbd:`A` simultaneously, followed by :kbd:`X`). In the second terminal, where the Python webserver is running, we can stop the -server using (:kbd:`ctrl-c`). - -What's next? -============ +server using (:kbd:`Ctrl-C`). In this tutorial, we configured the default user's password and ran -``cloud-init`` inside our QEMU virtual machine. +cloud-init inside our QEMU virtual machine. The full list of modules available can be found in :ref:`our modules documentation`. @@ -293,4 +262,4 @@ examples of more common use cases. .. _QEMU: https://www.qemu.org .. _install instructions: https://www.qemu.org/download/#linux -.. _LTS: https://wiki.ubuntu.com/Releases +.. _Ubuntu LTS: https://wiki.ubuntu.com/Releases diff --git a/doc/userdata.txt b/doc/userdata.txt index c13a418e..5fa431a1 100644 --- a/doc/userdata.txt +++ b/doc/userdata.txt @@ -5,7 +5,7 @@ way or another. In EC2, the data is provided by the user via the '--user-data' or 'user-data-file' argument to ec2-run-instances. The EC2 cloud makes the -data available to the instance via its meta-data service at +data available to the instance via its instance metadata service at http://169.254.169.254/latest/user-data cloud-init can read this input and act on it in different ways. diff --git a/packages/debian/control.in b/packages/debian/control.in index fb1cffc7..87403fe9 100644 --- a/packages/debian/control.in +++ b/packages/debian/control.in @@ -13,8 +13,9 @@ Depends: ${misc:Depends}, ${python3:Depends}, iproute2, python3-debconf +Breaks: cloud-init-base Recommends: eatmydata, sudo, software-properties-common, gdisk Suggests: ssh-import-id, openssh-server Description: Init scripts for cloud instances - Cloud instances need special scripts to run during initialisation + Cloud instances need special scripts to run during initialization to retrieve and install ssh keys and to let the user run various scripts. diff --git a/pyproject.toml b/pyproject.toml index 55b3c3bb..3ae24bfc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,9 +53,6 @@ module = [ "cloudinit.config.cc_ca_certs", "cloudinit.config.cc_growpart", "cloudinit.config.cc_ntp", - "cloudinit.config.cc_power_state_change", - "cloudinit.config.cc_rsyslog", - "cloudinit.config.cc_ubuntu_pro", "cloudinit.config.modules", "cloudinit.distros", "cloudinit.distros.alpine", @@ -134,19 +131,14 @@ module = [ "tests.unittests.config.test_cc_mcollective", "tests.unittests.config.test_cc_mounts", "tests.unittests.config.test_cc_phone_home", - "tests.unittests.config.test_cc_power_state_change", "tests.unittests.config.test_cc_puppet", "tests.unittests.config.test_cc_resizefs", "tests.unittests.config.test_cc_resolv_conf", "tests.unittests.config.test_cc_rh_subscription", - "tests.unittests.config.test_cc_rsyslog", "tests.unittests.config.test_cc_runcmd", - "tests.unittests.config.test_cc_snap", "tests.unittests.config.test_cc_ssh", - "tests.unittests.config.test_cc_timezone", "tests.unittests.config.test_cc_ubuntu_autoinstall", "tests.unittests.config.test_cc_ubuntu_drivers", - "tests.unittests.config.test_cc_ubuntu_pro", "tests.unittests.config.test_cc_update_etc_hosts", "tests.unittests.config.test_cc_users_groups", "tests.unittests.config.test_cc_wireguard", @@ -165,7 +157,6 @@ module = [ "tests.unittests.helpers", "tests.unittests.net.test_dhcp", "tests.unittests.net.test_init", - "tests.unittests.net.test_network_state", "tests.unittests.net.test_networkd", "tests.unittests.runs.test_merge_run", "tests.unittests.runs.test_simple_run", @@ -185,7 +176,6 @@ module = [ "tests.unittests.sources.test_gce", "tests.unittests.sources.test_init", "tests.unittests.sources.test_lxd", - "tests.unittests.sources.test_maas", "tests.unittests.sources.test_nocloud", "tests.unittests.sources.test_opennebula", "tests.unittests.sources.test_openstack", @@ -196,7 +186,6 @@ module = [ "tests.unittests.sources.test_smartos", "tests.unittests.sources.test_upcloud", "tests.unittests.sources.test_vultr", - "tests.unittests.sources.test_wsl", "tests.unittests.sources.vmware.test_vmware_config_file", "tests.unittests.test__init__", "tests.unittests.test_apport", diff --git a/setup.py b/setup.py index 9ca4a8a2..30f735ae 100644 --- a/setup.py +++ b/setup.py @@ -242,13 +242,11 @@ def finalize_options(self): if self.init_system and isinstance(self.init_system, str): self.init_system = self.init_system.split(",") - if len(self.init_system) == 0 and not platform.system().endswith( - "BSD" - ): + if not self.init_system and not platform.system().endswith("BSD"): self.init_system = ["systemd"] bad = [f for f in self.init_system if f not in INITSYS_TYPES] - if len(bad) != 0: + if bad: raise DistutilsError("Invalid --init-system: %s" % ",".join(bad)) for system in self.init_system: @@ -329,7 +327,7 @@ def finalize_options(self): setuptools.setup( name="cloud-init", version=get_version(), - description="Cloud instance initialisation magic", + description="Cloud instance initialization magic", author="Scott Moser", author_email="scott.moser@canonical.com", url="http://launchpad.net/cloud-init/", diff --git a/setup_utils.py b/setup_utils.py index 0ff75810..e0476c07 100644 --- a/setup_utils.py +++ b/setup_utils.py @@ -16,11 +16,11 @@ def pkg_config_read(library: str, var: str) -> str: fallbacks = { "systemd": { "systemdsystemconfdir": "/etc/systemd/system", - "systemdsystemunitdir": "/lib/systemd/system", - "systemdsystemgeneratordir": "/lib/systemd/system-generators", + "systemdsystemunitdir": "/usr/lib/systemd/system", + "systemdsystemgeneratordir": "/usr/lib/systemd/system-generators", }, "udev": { - "udevdir": "/lib/udev", + "udevdir": "/usr/lib/udev", }, } cmd = ["pkg-config", f"--variable={var}", library] diff --git a/templates/sources.list.debian.deb822.tmpl b/templates/sources.list.debian.deb822.tmpl index 6d15096c..bb286e66 100644 --- a/templates/sources.list.debian.deb822.tmpl +++ b/templates/sources.list.debian.deb822.tmpl @@ -24,11 +24,11 @@ Types: deb deb-src URIs: {{mirror}} Suites: {{codename}} {{codename}}-updates {{codename}}-backports Components: main -Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg +Signed-By: {{primary_key | default('/usr/share/keyrings/debian-archive-keyring.gpg', true)}} ## Major bug fix updates produced after the final release of the distribution. Types: deb deb-src URIs: {{security}} Suites: {{codename}}{% if codename in ('buster', 'stretch') %}/updates{% else %}-security{% endif %} Components: main -Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg +Signed-By: {{security_key | default(primary_key, true) | default('/usr/share/keyrings/debian-archive-keyring.gpg', true)}} diff --git a/templates/sources.list.ubuntu.deb822.tmpl b/templates/sources.list.ubuntu.deb822.tmpl index 8202dcbe..0f5a16cf 100644 --- a/templates/sources.list.ubuntu.deb822.tmpl +++ b/templates/sources.list.ubuntu.deb822.tmpl @@ -45,7 +45,7 @@ Types: deb URIs: {{mirror}} Suites: {{codename}} {{codename}}-updates {{codename}}-backports Components: main universe restricted multiverse -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +Signed-By: {{primary_key | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true)}} ## Ubuntu security updates. Aside from URIs and Suites, ## this should mirror your choices in the previous section. @@ -53,4 +53,4 @@ Types: deb URIs: {{security}} Suites: {{codename}}-security Components: main universe restricted multiverse -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +Signed-By: {{security_key | default(primary_key, true) | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true)}} diff --git a/test-requirements.txt b/test-requirements.txt index c6c32cae..c71dddee 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,7 +10,12 @@ pytest!=7.3.2 pytest-cov pytest-mock +pytest-xdist setuptools jsonschema responses passlib + +# This one is currently used only by the CloudSigma and SmartOS datasources. +# If these datasources are removed, this is no longer needed. +pyserial diff --git a/tests/integration_tests/assets/dropins/cc_custom_module_24_1.py b/tests/integration_tests/assets/dropins/cc_custom_module_24_1.py new file mode 100644 index 00000000..7eda1c0b --- /dev/null +++ b/tests/integration_tests/assets/dropins/cc_custom_module_24_1.py @@ -0,0 +1,42 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""This was the canonical example module from 24.1 + +Ensure cloud-init still supports it.""" + +import logging + +from cloudinit.cloud import Cloud +from cloudinit.config import Config +from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.distros import ALL_DISTROS +from cloudinit.settings import PER_INSTANCE + +MODULE_DESCRIPTION = """\ +Description that will be used in module documentation. + +This will likely take multiple lines. +""" + +LOG = logging.getLogger(__name__) + +meta: MetaSchema = { + "id": "cc_example", + "name": "Example Module", + "title": "Shows how to create a module", + "description": MODULE_DESCRIPTION, + "distros": [ALL_DISTROS], + "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["example_key, example_other_key"], + "examples": [ + "example_key: example_value", + "example_other_key: ['value', 2]", + ], +} # type: ignore + +__doc__ = get_meta_doc(meta) + + +def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: + LOG.debug(f"Hi from module {name}") # noqa: G004 + # Add one more line for easier testing + print("Hello from module") diff --git a/tests/integration_tests/bugs/test_gh671.py b/tests/integration_tests/bugs/test_gh671.py index 3b84b85c..583a28a6 100644 --- a/tests/integration_tests/bugs/test_gh671.py +++ b/tests/integration_tests/bugs/test_gh671.py @@ -21,7 +21,7 @@ def _check_password(instance, unhashed_password): @pytest.mark.skipif(PLATFORM != "azure", reason="Test is Azure specific") -def test_update_default_password(setup_image, session_cloud: IntegrationCloud): +def test_update_default_password(session_cloud: IntegrationCloud): os_profile = { "os_profile": { "admin_password": "", diff --git a/tests/integration_tests/bugs/test_lp1835584.py b/tests/integration_tests/bugs/test_lp1835584.py index f44edca8..1f4c8517 100644 --- a/tests/integration_tests/bugs/test_lp1835584.py +++ b/tests/integration_tests/bugs/test_lp1835584.py @@ -92,8 +92,5 @@ def test_azure_kernel_upgrade_case_insensitive_uuid( ) } ) as instance: - # We can't use setup_image fixture here because we want to avoid - # taking a snapshot or cleaning the booted machine after cloud-init - # upgrade. instance.install_new_cloud_init(source, clean=False) _check_iid_insensitive_across_kernel_upgrade(instance) diff --git a/tests/integration_tests/bugs/test_lp1901011.py b/tests/integration_tests/bugs/test_lp1901011.py index 4a25c602..c9f71e6d 100644 --- a/tests/integration_tests/bugs/test_lp1901011.py +++ b/tests/integration_tests/bugs/test_lp1901011.py @@ -20,7 +20,7 @@ ], ) def test_ephemeral( - instance_type, is_ephemeral, session_cloud: IntegrationCloud, setup_image + instance_type, is_ephemeral, session_cloud: IntegrationCloud ): if is_ephemeral: expected_log = ( diff --git a/tests/integration_tests/bugs/test_lp1910835.py b/tests/integration_tests/bugs/test_lp1910835.py index ff8390f7..da5fb217 100644 --- a/tests/integration_tests/bugs/test_lp1910835.py +++ b/tests/integration_tests/bugs/test_lp1910835.py @@ -29,7 +29,7 @@ @pytest.mark.skipif(PLATFORM != "azure", reason="Test is Azure specific") -def test_crlf_in_azure_metadata_ssh_keys(session_cloud, setup_image): +def test_crlf_in_azure_metadata_ssh_keys(session_cloud): authorized_keys_path = "/home/{}/.ssh/authorized_keys".format( session_cloud.cloud_instance.username ) diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index f7dc1463..dead2c1f 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -7,7 +7,7 @@ import string from abc import ABC, abstractmethod from copy import deepcopy -from typing import Type +from typing import Optional, Type from uuid import UUID from pycloudlib import ( @@ -23,7 +23,7 @@ ) from pycloudlib.cloud import ImageType from pycloudlib.ec2.instance import EC2Instance -from pycloudlib.lxd.cloud import _BaseLXD +from pycloudlib.lxd.cloud import BaseCloud, _BaseLXD from pycloudlib.lxd.instance import BaseInstance, LXDInstance import cloudinit @@ -65,7 +65,7 @@ def __init__( self.settings = settings self.cloud_instance = self._get_cloud_instance() self.initial_image_id = self._get_initial_image() - self.snapshot_id = None + self.snapshot_id: Optional[str] = None @property def image_id(self): @@ -83,7 +83,7 @@ def emit_settings_to_log(self) -> None: ) @abstractmethod - def _get_cloud_instance(self): + def _get_cloud_instance(self) -> BaseCloud: raise NotImplementedError def _get_initial_image(self, **kwargs) -> str: @@ -132,6 +132,10 @@ def launch( "user_data": user_data, "username": DISTRO_TO_USERNAME[CURRENT_RELEASE.os], } + if self.settings.INSTANCE_TYPE: + default_launch_kwargs["instance_type"] = ( + self.settings.INSTANCE_TYPE + ) launch_kwargs = {**default_launch_kwargs, **launch_kwargs} display_launch_kwargs = deepcopy(launch_kwargs) if display_launch_kwargs.get("user_data") is not None: @@ -182,7 +186,7 @@ def snapshot(self, instance): def delete_snapshot(self): if self.snapshot_id: - if self.settings.KEEP_IMAGE: # type: ignore + if self.settings.KEEP_IMAGE: log.info( "NOT deleting snapshot image created for this testrun " "because KEEP_IMAGE is True: %s", @@ -198,6 +202,7 @@ def delete_snapshot(self): class Ec2Cloud(IntegrationCloud): datasource = "ec2" + cloud_instance: EC2 def _get_cloud_instance(self) -> EC2: return EC2(tag="ec2-integration-test") @@ -226,6 +231,7 @@ def _perform_launch( class GceCloud(IntegrationCloud): datasource = "gce" + cloud_instance: GCE def _get_cloud_instance(self) -> GCE: return GCE( @@ -263,6 +269,7 @@ def destroy(self): class OciCloud(IntegrationCloud): datasource = "oci" + cloud_instance: OCI def _get_cloud_instance(self) -> OCI: return OCI( @@ -382,6 +389,7 @@ def _get_or_set_profile_list(self, release) -> list: class OpenstackCloud(IntegrationCloud): datasource = "openstack" + cloud_instance: Openstack def _get_cloud_instance(self): return Openstack( @@ -414,7 +422,7 @@ def _get_cloud_instance(self) -> IBM: class QemuCloud(IntegrationCloud): datasource = "qemu" - cloud_instance = Qemu + cloud_instance: Qemu def _get_cloud_instance(self): return Qemu(tag="qemu-integration-test") diff --git a/tests/integration_tests/cmd/test_status.py b/tests/integration_tests/cmd/test_status.py index de4222e2..f318dedb 100644 --- a/tests/integration_tests/cmd/test_status.py +++ b/tests/integration_tests/cmd/test_status.py @@ -34,7 +34,7 @@ def retry_read_from_file(client: IntegrationInstance, path: str): PLATFORM != "lxd_container", reason="Test is LXD specific", ) -def test_wait_when_no_datasource(session_cloud: IntegrationCloud, setup_image): +def test_wait_when_no_datasource(session_cloud: IntegrationCloud): """Ensure that when no datasource is found, we get status: disabled LP: #1966085 diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index b62dae82..cba33601 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -15,6 +15,7 @@ from pycloudlib.cloud import ImageType from pycloudlib.lxd.instance import LXDInstance +import tests.integration_tests.reaper as reaper from tests.integration_tests import integration_settings from tests.integration_tests.clouds import ( AzureCloud, @@ -38,6 +39,11 @@ log.addHandler(logging.StreamHandler(sys.stdout)) log.setLevel(logging.INFO) +# set log level INFO instead of DEBUG for boto3 and botocore +# to prevent 1000s of lines of DEBUG log spam that occur during some tests +logging.getLogger("botocore").setLevel(logging.INFO) +logging.getLogger("boto3").setLevel(logging.INFO) + platforms: Dict[str, Type[IntegrationCloud]] = { "ec2": Ec2Cloud, "gce": GceCloud, @@ -74,8 +80,22 @@ def disable_subp_usage(request): pass +_SESSION_CLOUD: IntegrationCloud +REAPER: reaper._Reaper + + @pytest.fixture(scope="session") def session_cloud() -> Generator[IntegrationCloud, None, None]: + """a shared session is created in pytest_sessionstart() + + yield this shared session + """ + global _SESSION_CLOUD + yield _SESSION_CLOUD + + +def get_session_cloud() -> IntegrationCloud: + """get_session_cloud() creates a session from configuration""" if integration_settings.PLATFORM not in platforms.keys(): raise ValueError( f"{integration_settings.PLATFORM} is an invalid PLATFORM " @@ -91,8 +111,7 @@ def session_cloud() -> Generator[IntegrationCloud, None, None]: ) cloud = platforms[integration_settings.PLATFORM](image_type=image_type) cloud.emit_settings_to_log() - yield cloud - cloud.destroy() + return cloud def get_validated_source( @@ -118,12 +137,8 @@ def get_validated_source( raise ValueError(f"Invalid value for CLOUD_INIT_SOURCE setting: {source}") -@pytest.fixture(scope="session") -def setup_image(session_cloud: IntegrationCloud, request): - """Setup the target environment with the correct version of cloud-init. - - So we can launch instances / run tests with the correct image - """ +def setup_image(session_cloud: IntegrationCloud) -> None: + """create image with correct version of cloud-init, then make a snapshot""" source = get_validated_source(session_cloud) if not ( source.installs_new_version() @@ -141,7 +156,8 @@ def setup_image(session_cloud: IntegrationCloud, request): and integration_settings.INCLUDE_COVERAGE ): log.error( - "Invalid configuration, cannot enable both profile and coverage." + "Invalid configuration, cannot enable both profile and " + "coverage." ) raise ValueError() if integration_settings.INCLUDE_COVERAGE: @@ -158,11 +174,6 @@ def setup_image(session_cloud: IntegrationCloud, request): client.destroy() log.info("Done with environment setup") - # For some reason a yield here raises a - # ValueError: setup_image did not yield a value - # during setup so use a finalizer instead. - request.addfinalizer(session_cloud.delete_snapshot) - def _collect_logs(instance: IntegrationInstance, log_dir: Path): instance.execute( @@ -311,13 +322,16 @@ def _client( local_launch_kwargs["lxd_setup"] = lxd_setup with session_cloud.launch( - user_data=user_data, launch_kwargs=launch_kwargs, **local_launch_kwargs + user_data=user_data, + launch_kwargs=launch_kwargs, + **local_launch_kwargs, ) as instance: if lxd_use_exec is not None and isinstance( instance.instance, LXDInstance ): # Existing instances are not affected by the launch kwargs, so - # ensure it here; we still need the launch kwarg so waiting works + # ensure it here; we still need the launch kwarg so waiting + # works instance.instance.execute_via_ssh = False previous_failures = request.session.testsfailed yield instance @@ -325,10 +339,12 @@ def _client( _collect_artifacts(instance, request.node.nodeid, test_failed) # conflicting requirements: # - pytest thinks that it can cleanup loggers after tests run - # - pycloudlib thinks that at garbage collection is a good place to tear - # down sftp connections - # After the final test runs, pytest might clean up loggers which will cause - # paramiko to barf when it logs that the connection is being closed. + # - pycloudlib thinks that at garbage collection is a good place to + # tear down sftp connections + # + # After the final test runs, pytest might clean up loggers which will + # cause paramiko to barf when it logs that the connection is being + # closed. # # Manually run __del__() to prevent this teardown mess. instance.instance.__del__() @@ -336,7 +352,7 @@ def _client( @pytest.fixture def client( # pylint: disable=W0135 - request, fixture_utils, session_cloud, setup_image + request, fixture_utils, session_cloud ) -> Iterator[IntegrationInstance]: """Provide a client that runs for every test.""" with _client(request, fixture_utils, session_cloud) as client: @@ -345,7 +361,7 @@ def client( # pylint: disable=W0135 @pytest.fixture(scope="module") def module_client( # pylint: disable=W0135 - request, fixture_utils, session_cloud, setup_image + request, fixture_utils, session_cloud ) -> Iterator[IntegrationInstance]: """Provide a client that runs once per module.""" with _client(request, fixture_utils, session_cloud) as client: @@ -354,7 +370,7 @@ def module_client( # pylint: disable=W0135 @pytest.fixture(scope="class") def class_client( # pylint: disable=W0135 - request, fixture_utils, session_cloud, setup_image + request, fixture_utils, session_cloud ) -> Iterator[IntegrationInstance]: """Provide a client that runs once per class.""" with _client(request, fixture_utils, session_cloud) as client: @@ -459,8 +475,61 @@ def _generate_profile_report() -> None: log.info(command, "final.stats") +# https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_sessionstart +def pytest_sessionstart(session) -> None: + """do session setup""" + global _SESSION_CLOUD + global REAPER + log.info("starting session") + try: + _SESSION_CLOUD = get_session_cloud() + setup_image(_SESSION_CLOUD) + REAPER = reaper._Reaper() + REAPER.start() + except Exception as e: + if _SESSION_CLOUD: + # if a _SESSION_CLOUD was allocated, clean it up + if _SESSION_CLOUD.snapshot_id: + # if a snapshot id was set, then snapshot succeeded, teardown + _SESSION_CLOUD.delete_snapshot() + _SESSION_CLOUD.destroy() + pytest.exit( + f"{type(e).__name__} in session setup: {str(e)}", returncode=2 + ) + log.info("started session") + + def pytest_sessionfinish(session, exitstatus) -> None: - if integration_settings.INCLUDE_COVERAGE: - _generate_coverage_report() - elif integration_settings.INCLUDE_PROFILE: - _generate_profile_report() + """do session teardown""" + global REAPER + log.info("finishing session") + try: + if integration_settings.INCLUDE_COVERAGE: + _generate_coverage_report() + elif integration_settings.INCLUDE_PROFILE: + _generate_profile_report() + except Exception as e: + log.warning("Could not generate report during teardown: %s", e) + try: + _SESSION_CLOUD.delete_snapshot() + except Exception as e: + log.warning( + "Could not delete snapshot. Leaked snapshot id %s: %s", + _SESSION_CLOUD.snapshot_id, + e, + ) + try: + REAPER.stop() + except Exception as e: + log.warning( + "Could not tear down instance reaper thread: %s(%s)", + type(e).__name__, + e, + ) + try: + _SESSION_CLOUD.destroy() + except Exception as e: + log.warning( + "Could not destroy session cloud: %s(%s)", type(e).__name__, e + ) + log.info("finish session") diff --git a/tests/integration_tests/datasources/test_azure.py b/tests/integration_tests/datasources/test_azure.py index c1d36abe..b32bfe41 100644 --- a/tests/integration_tests/datasources/test_azure.py +++ b/tests/integration_tests/datasources/test_azure.py @@ -78,9 +78,7 @@ def parse_resolvectl_dns(output: str) -> dict: @pytest.mark.skipif( CURRENT_RELEASE < BIONIC, reason="Easier to test on Bionic+" ) -def test_azure_multi_nic_setup( - setup_image, session_cloud: IntegrationCloud -) -> None: +def test_azure_multi_nic_setup(session_cloud: IntegrationCloud) -> None: """Integration test for https://warthogs.atlassian.net/browse/CPC-3999. Azure should have the primary NIC only route to DNS. diff --git a/tests/integration_tests/decorators.py b/tests/integration_tests/decorators.py index 54a1943a..885b4f46 100644 --- a/tests/integration_tests/decorators.py +++ b/tests/integration_tests/decorators.py @@ -27,7 +27,7 @@ def wrapper(*args, **kwargs): time.sleep(delay) else: if last_error: - raise last_error + raise TimeoutError from last_error return retval return wrapper diff --git a/tests/integration_tests/dropins/test_custom_modules.py b/tests/integration_tests/dropins/test_custom_modules.py new file mode 100644 index 00000000..a9a67866 --- /dev/null +++ b/tests/integration_tests/dropins/test_custom_modules.py @@ -0,0 +1,28 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import pytest + +from tests.integration_tests import releases +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.releases import IS_UBUNTU +from tests.integration_tests.util import ASSETS_DIR + + +@pytest.mark.skipif( + not IS_UBUNTU, reason="module dir tested is ubuntu-specific" +) +def test_custom_module_24_1(client: IntegrationInstance): + """Ensure that modifications to cloud-init don't break old custom modules. + + 24.1 had documentation that differs from current best practices. We want + to ensure modules created from this documentation still work: + https://docs.cloud-init.io/en/24.1/development/module_creation.html + """ + client.push_file( + ASSETS_DIR / "dropins/cc_custom_module_24_1.py", + "/usr/lib/python3/dist-packages/cloudinit/config/cc_custom_module_24_1.py", + ) + output = client.execute("cloud-init single --name cc_custom_module_24_1") + if releases.CURRENT_RELEASE >= releases.PLUCKY: + assert "The 'get_meta_doc()' function is deprecated" in output + assert "Hello from module" in output diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index a5ce8f1e..d745219e 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -14,7 +14,7 @@ from pycloudlib.result import Result from tests.helpers import cloud_init_project_dir -from tests.integration_tests import integration_settings +from tests.integration_tests import conftest, integration_settings from tests.integration_tests.decorators import retry from tests.integration_tests.util import ASSETS_DIR @@ -268,9 +268,14 @@ def install_deb(self): # to install missing dependency errors due to stale cache. self.execute("apt update") # Use apt install instead of dpkg -i to pull in any changed pkg deps - assert self.execute( - f"apt install {remote_path} --yes --allow-downgrades" - ).ok + apt_result = self.execute( + f"apt install -qy {remote_path} --allow-downgrades" + ) + if not apt_result.ok: + raise RuntimeError( + f"Failed to install {deb_name}: stdout: {apt_result.stdout}. " + f"stderr: {apt_result.stderr}" + ) @retry(tries=30, delay=1) def upgrade_cloud_init(self, pkg: str): @@ -325,6 +330,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): if not self.settings.KEEP_INSTANCE: - self.destroy() + conftest.REAPER.reap(self) else: log.info("Keeping Instance, public ip: %s", self.ip()) diff --git a/tests/integration_tests/modules/test_ansible.py b/tests/integration_tests/modules/test_ansible.py index 974f86fb..bf74cba2 100644 --- a/tests/integration_tests/modules/test_ansible.py +++ b/tests/integration_tests/modules/test_ansible.py @@ -286,6 +286,7 @@ def _test_ansible_pull_from_local_server(my_client): @pytest.mark.user_data( USER_DATA + INSTALL_METHOD.format(package="ansible-core", method="pip") ) +@pytest.mark.skip("This test is currently broken and needs to be fixed") def test_ansible_pull_pip(client: IntegrationInstance): push_and_enable_systemd_unit(client, "repo_server.service", REPO_SERVER) push_and_enable_systemd_unit(client, "repo_waiter.service", REPO_WAITER) @@ -303,6 +304,7 @@ def test_ansible_pull_pip(client: IntegrationInstance): @pytest.mark.user_data( USER_DATA + INSTALL_METHOD.format(package="ansible", method="distro") ) +@pytest.mark.skip("This test is currently broken and needs to be fixed") def test_ansible_pull_distro(client): push_and_enable_systemd_unit(client, "repo_server.service", REPO_SERVER) push_and_enable_systemd_unit(client, "repo_waiter.service", REPO_WAITER) diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index 963d82fd..61336ad5 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. """Series of integration tests covering apt functionality.""" +import logging import re from textwrap import dedent @@ -9,7 +10,10 @@ from cloudinit.util import is_true from tests.integration_tests.clouds import IntegrationCloud from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.integration_settings import PLATFORM +from tests.integration_tests.integration_settings import ( + KEEP_INSTANCE, + PLATFORM, +) from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, MANTIC from tests.integration_tests.util import ( get_feature_flag_value, @@ -17,6 +21,8 @@ verify_clean_log, ) +logger = logging.getLogger(__name__) + DEB822_SOURCES_FILE = "/etc/apt/sources.list.d/ubuntu.sources" ORIG_SOURCES_FILE = "/etc/apt/sources.list" GET_TEMPDIR = "python3 -c 'import tempfile;print(tempfile.mkdtemp());'" @@ -477,12 +483,65 @@ def test_apt_proxy(client: IntegrationInstance): """ +def _do_oci_customization(cloud_config: str): + """ + Add a snap disable command to the cloud-config for OCI. + + This is necessary to disable the oracle-cloud-agent snap on Oracle Cloud + in order to prevent it from interfering with apt during tests. + """ + addition = " - snap disable oracle-cloud-agent" + if PLATFORM == "oci": + logger.info( + "Running on Oracle Cloud, adding snap disable command to " + "cloud-config to disable the oracle-cloud-agent snap." + ) + return cloud_config.replace("runcmd:", f"runcmd:\n{addition}") + return cloud_config + + @pytest.mark.skipif(not IS_UBUNTU, reason="Apt usage") -def test_install_missing_deps(setup_image, session_cloud: IntegrationCloud): +def test_install_missing_deps(session_cloud: IntegrationCloud): + """ + Test the installation of missing dependencies using apt on an Ubuntu + system. This test is divided into two stages: + Stage 1 (Remove 'gpg' package): + - Launch an instance with user-data that removes the 'gpg' package. + - If on Oracle Cloud, add a command to the user-data to disable the + oracle-cloud-agent snap to prevent it from interfering with apt. + - Verify that the cloud-init log is clean and the boot process is clean. + - Verify that 'gpg' is actually uninstalled using dpkg. + - Create a snapshot of the instance after 'gpg' has been removed. + - If KEEP_INSTANCE is False, destroy the instance after snapshotting. + Stage 2 (re-install 'gpg' package with user-data): + - Launch a new instance from the snapshot created in Stage 1 with + user-data that installs any missing recommended dependencies. + - Verify that the cloud-init log is clean and the boot process is clean. + - Check the cloud-init log to ensure that 'gpg' and its dependencies are + installed successfully. + - Double check that 'gpg' is actually installed using dpkg. + """ # Two stage install: First stage: remove gpg noninteractively from image - instance1 = session_cloud.launch(user_data=REMOVE_GPG_USERDATA) + instance1 = session_cloud.launch( + user_data=_do_oci_customization(REMOVE_GPG_USERDATA) + ) + + # look for r"un gpg" using regex ('un' means uninstalled) + dpkg_output = instance1.execute("dpkg -l gpg") + assert re.search(r"un\s+gpg", dpkg_output.stdout), ( + "gpg package is still installed. it should have been removed by " + "the user-data." + ) + snapshot_id = instance1.snapshot() - instance1.destroy() + if not KEEP_INSTANCE: + logger.info("Destroying instance1 after snapshotting.") + instance1.destroy() + else: + logger.info( + "Not destroying instance1 after snapshotting because KEEP_INSTANCE" + " is True." + ) # Second stage: provide active apt user-data which will install missing gpg with session_cloud.launch( user_data=INSTALL_ANY_MISSING_RECOMMENDED_DEPENDENCIES, @@ -492,3 +551,7 @@ def test_install_missing_deps(setup_image, session_cloud: IntegrationCloud): verify_clean_log(log) verify_clean_boot(minimal_client) assert re.search(RE_GPG_SW_PROPERTIES_INSTALLED, log) + gpg_installed = re.search( + r"ii\s+gpg", minimal_client.execute("dpkg -l gpg").stdout + ) + assert gpg_installed is not None, "gpg package is not installed." diff --git a/tests/integration_tests/modules/test_boothook.py b/tests/integration_tests/modules/test_boothook.py index 6d8a176a..29ed471b 100644 --- a/tests/integration_tests/modules/test_boothook.py +++ b/tests/integration_tests/modules/test_boothook.py @@ -56,6 +56,9 @@ def test_boothook_waits_for_network( ): """Test boothook handling waits for network before running.""" client = class_client - assert network_wait_logged( - client.read_from_file("/var/log/cloud-init.log") - ) == features.MANUAL_NETWORK_WAIT + assert ( + network_wait_logged( + client.read_from_file("/var/log/cloud-init.log") + ) + == features.MANUAL_NETWORK_WAIT + ) diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 9c525840..c3458a5a 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -19,6 +19,7 @@ import cloudinit.config from cloudinit import lifecycle from cloudinit.util import is_true +from tests.integration_tests.clouds import Ec2Cloud from tests.integration_tests.decorators import retry from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import ( @@ -484,7 +485,11 @@ def test_instance_json_lxd_vm(self, class_client: IntegrationInstance): assert v1_data["region"] is None @pytest.mark.skipif(PLATFORM != "ec2", reason="Test is ec2 specific") - def test_instance_json_ec2(self, class_client: IntegrationInstance): + def test_instance_json_ec2( + self, + class_client: IntegrationInstance, + session_cloud: Ec2Cloud, + ): client = class_client instance_json_file = client.read_from_file( "/run/cloud-init/instance-data.json" @@ -507,7 +512,7 @@ def test_instance_json_ec2(self, class_client: IntegrationInstance): ) assert v1_data["instance_id"] == client.instance.name assert v1_data["local_hostname"].startswith("ip-") - assert v1_data["region"] == client.cloud.cloud_instance.region + assert v1_data["region"] == session_cloud.cloud_instance.region @pytest.mark.skipif(PLATFORM != "gce", reason="Test is GCE specific") def test_instance_json_gce(self, class_client: IntegrationInstance): diff --git a/tests/integration_tests/modules/test_disk_setup.py b/tests/integration_tests/modules/test_disk_setup.py index 27a70d32..71a19eba 100644 --- a/tests/integration_tests/modules/test_disk_setup.py +++ b/tests/integration_tests/modules/test_disk_setup.py @@ -8,7 +8,12 @@ from cloudinit.subp import subp from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM -from tests.integration_tests.releases import CURRENT_RELEASE, FOCAL, IS_UBUNTU +from tests.integration_tests.releases import ( + CURRENT_RELEASE, + FOCAL, + IS_UBUNTU, + NOBLE, +) from tests.integration_tests.util import verify_clean_boot, verify_clean_log DISK_PATH = "/tmp/test_disk_setup_{}".format(uuid4()) @@ -24,8 +29,7 @@ def setup_and_mount_lxd_disk(instance: LXDInstance): @pytest.fixture def create_disk(): - # 640k should be enough for anybody - subp("dd if=/dev/zero of={} bs=1k count=640".format(DISK_PATH).split()) + subp("dd if=/dev/zero of={} bs=64k count=40".format(DISK_PATH).split()) yield os.remove(DISK_PATH) @@ -227,3 +231,42 @@ def test_disk_setup_no_partprobe( self._verify_first_disk_setup(client, log) assert "partprobe" not in log + + +def setup_lxd_disk_with_fs(instance: LXDInstance): + subp(["mkfs.ext4", DISK_PATH]) + subp( + f"lxc config device add {instance.name} test-disk-setup-disk " + f"disk source={DISK_PATH}".split() + ) + + +@pytest.mark.lxd_setup.with_args(setup_lxd_disk_with_fs) +@pytest.mark.skipif(not IS_UBUNTU, reason="Only ever tested on Ubuntu") +@pytest.mark.skipif( + PLATFORM != "lxd_vm", reason="Test requires additional mounted device" +) +def test_required_mounts(create_disk, client: IntegrationInstance): + """Ensure /var is mounted before used. + + GH-6001 + LP: #2097441 + """ + client.execute( + 'echo "/dev/sdb /var auto defaults,nofail 0 2" >> /etc/fstab' + ) + client.execute("cloud-init clean --logs") + client.restart() + + service = ( + "cloud-init-local.service" + if CURRENT_RELEASE <= NOBLE + else "cloud-init-main.service" + ) + + deps = client.execute( + f"systemctl list-dependencies --all {service}".split() + ) + assert "var.mount" in deps, "Exepected 'var.mount' to be a dependency" + + verify_clean_boot(client) diff --git a/tests/integration_tests/modules/test_frequency_override.py b/tests/integration_tests/modules/test_frequency_override.py index e7cd2036..a78dacd4 100644 --- a/tests/integration_tests/modules/test_frequency_override.py +++ b/tests/integration_tests/modules/test_frequency_override.py @@ -1,6 +1,7 @@ import pytest from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE USER_DATA = """\ @@ -18,7 +19,8 @@ def test_frequency_override(client: IntegrationInstance): in client.read_from_file("/var/log/cloud-init.log") ) assert client.read_from_file("/var/tmp/hi").strip().count("hi") == 1 - if CURRENT_RELEASE.os == "ubuntu": + # This workaround is not needed for OCI, so just skip it + if CURRENT_RELEASE.os == "ubuntu" and PLATFORM != "oci": if CURRENT_RELEASE.series in ("focal", "jammy", "lunar", "mantic"): # Stable series will block on snapd.seeded.service and create a # semaphore file diff --git a/tests/integration_tests/modules/test_growpart.py b/tests/integration_tests/modules/test_growpart.py index ebd2d8d1..0b0ed425 100644 --- a/tests/integration_tests/modules/test_growpart.py +++ b/tests/integration_tests/modules/test_growpart.py @@ -62,7 +62,7 @@ def test_grow_part(self, client: IntegrationInstance): log = client.read_from_file("/var/log/cloud-init.log") assert ( "cc_growpart.py[INFO]: '/dev/sdb1' resized:" - " changed (/dev/sdb1) from" in log + " changed (/dev/sdb1)" in log ) lsblk = json.loads(client.execute("lsblk --json")) diff --git a/tests/integration_tests/modules/test_hotplug.py b/tests/integration_tests/modules/test_hotplug.py index 908dfc6d..4c531491 100644 --- a/tests/integration_tests/modules/test_hotplug.py +++ b/tests/integration_tests/modules/test_hotplug.py @@ -309,9 +309,14 @@ def test_multi_nic_hotplug(client: IntegrationInstance): verify_clean_boot(client) +# TODO: support early hotplug +# +# This test usually passes without the `wait_for_cloud_init()` +# but sometimes the hotplug event races with the end of cloud-init +# so occasionally fails. For now, document this shortcoming and +# wait for cloud-init to complete before testing the behavior. @pytest.mark.skipif(PLATFORM != "ec2", reason="test is ec2 specific") -@pytest.mark.skip(reason="IMDS race, see GH-5373. Unskip when fixed.") -def test_multi_nic_hotplug_vpc(setup_image, session_cloud: IntegrationCloud): +def test_multi_nic_hotplug_vpc(session_cloud: IntegrationCloud): """Tests that additional secondary NICs are routable from local networks after the hotplug hook is executed when network updates are configured on the HOTPLUG event.""" diff --git a/tests/integration_tests/modules/test_lxd.py b/tests/integration_tests/modules/test_lxd.py index 44814e67..930167cb 100644 --- a/tests/integration_tests/modules/test_lxd.py +++ b/tests/integration_tests/modules/test_lxd.py @@ -247,7 +247,7 @@ def test_storage_btrfs(client): @pytest.mark.skipif( CURRENT_RELEASE < FOCAL, reason="tested on Focal and later" ) -def test_storage_preseed_btrfs(setup_image, session_cloud: IntegrationCloud): +def test_storage_preseed_btrfs(session_cloud: IntegrationCloud): # TODO: If test is marked as not bionic, why is there a bionic section? if CURRENT_RELEASE.series in ("bionic",): nictype = "nictype: bridged" @@ -311,7 +311,7 @@ def test_storage_zfs(client): @pytest.mark.skipif( CURRENT_RELEASE < FOCAL, reason="Tested on focal and later" ) -def test_storage_preseed_zfs(setup_image, session_cloud: IntegrationCloud): +def test_storage_preseed_zfs(session_cloud: IntegrationCloud): # TODO: If test is marked as not bionic, why is there a bionic section? if CURRENT_RELEASE.series in ("bionic",): nictype = "nictype: bridged" diff --git a/tests/integration_tests/modules/test_ubuntu_drivers.py b/tests/integration_tests/modules/test_ubuntu_drivers.py index ffa19ac1..16856ab7 100644 --- a/tests/integration_tests/modules/test_ubuntu_drivers.py +++ b/tests/integration_tests/modules/test_ubuntu_drivers.py @@ -1,11 +1,17 @@ +import logging import re import pytest -from tests.integration_tests.clouds import IntegrationCloud +from tests.integration_tests.clouds import ( + IntegrationCloud, + IntegrationInstance, +) from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.util import verify_clean_boot, verify_clean_log +logger = logging.getLogger(__name__) + USER_DATA = """\ #cloud-config drivers: @@ -13,27 +19,55 @@ license-accepted: true """ -# NOTE(VM.GPU2.1 is not in all availability_domains: use qIZq:US-ASHBURN-AD-1) - -@pytest.mark.adhoc # Expensive instance type @pytest.mark.skipif(PLATFORM != "oci", reason="Test is OCI specific") def test_ubuntu_drivers_installed(session_cloud: IntegrationCloud): - with session_cloud.launch( - launch_kwargs={"instance_type": "VM.GPU2.1"}, user_data=USER_DATA - ) as client: - log = client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log) - verify_clean_boot(client) - assert 1 == log.count( - "Installing and activating NVIDIA drivers " - "(nvidia/license-accepted=True, version=latest)" - ) - result = client.execute("dpkg -l | grep nvidia") - assert result.ok, "No nvidia packages found" - assert re.search( - r"ii\s+linux-modules-nvidia-\d+-server", result.stdout - ), ( - f"Did not find specific nvidia drivers packages in:" - f" {result.stdout}" + """ + Test the installation of NVIDIA drivers on an OCI instance. + + This test checks that the ubuntu-drivers module installs NVIDIA drivers + as expected on an OCI instance. + + This test launches its own instance so that it can ensure that a GPU + instance type is used. The "VM.GPU.A10.1" instance type is used because + it is the most widely available GPU instance type on OCI. + + Additionally, in case there is limited availability of GPU instances, this + test launches an instance outside the normal context manager used in other + tests. This is so that if the instance fails to launch, the test can be + marked as xfail rather than just failing. + + Test Steps: + 1. Launch a GPU instance with the user data that installs NVIDIA drivers. + 2. Verify that the cloud-init log is clean. + 3. Verify that the instance boots cleanly. + 4. Verify that the NVIDIA drivers are installed as expected. + """ + try: + client = session_cloud.launch( + launch_kwargs={"instance_type": "VM.GPU.A10.1"}, + user_data=USER_DATA, ) + except Exception as e: + pytest.xfail(f"Instance launch failed: {e}") + + try: + _do_test(client) + finally: + client.destroy() + + +def _do_test(client: IntegrationInstance): + """Performs above test steps on the given client.""" + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + verify_clean_boot(client) + assert 1 == log.count( + "Installing and activating NVIDIA drivers " + "(nvidia/license-accepted=True, version=latest)" + ) + result = client.execute("dpkg -l | grep nvidia") + assert result.ok, "No nvidia packages found" + assert re.search( + r"ii\s+nvidia.*-\d+-server", result.stdout + ), f"Did not find specific nvidia driver packages in: {result.stdout}" diff --git a/tests/integration_tests/reaper.py b/tests/integration_tests/reaper.py new file mode 100644 index 00000000..308dbbfe --- /dev/null +++ b/tests/integration_tests/reaper.py @@ -0,0 +1,203 @@ +"""Defines _Reaper, which destroys instances in a background thread + +This class is intended to be a singleton which is instantiated on session setup +and cleaned on session teardown. Any instances submitted to the reaper are +destroyed. Instances that refuse to be destroyed due to external library errors +or flaky infrastructure are tracked, retried and upon test session completion +are reported to the end user as a test warning. +""" + +from __future__ import annotations # required for Python 3.8 + +import logging +import queue +import threading +import warnings +from typing import Final, List, Optional + +from tests.integration_tests.instances import IntegrationInstance + +LOG = logging.getLogger() + + +class _Reaper: + def __init__(self, timeout: float = 30.0): + # self.timeout sets the amount of time to sleep before retrying + self.timeout = timeout + # self.wake_reaper tells the reaper to wake up. + # + # A lock is used for synchronization. This means that notify() will + # block if + # the reaper is currently awake. + # + # It is set by: + # - signal interrupt indicating cleanup + # - session completion indicating cleanup + # - reaped instance indicating work to be done + self.wake_reaper: Final[threading.Condition] = threading.Condition() + + # self.exit_reaper tells the reaper loop to tear down, called once at + # end of tests + self.exit_reaper: Final[threading.Event] = threading.Event() + + # List of instances which temporarily escaped death + # The primary porpose of the reaper is to coax these instance towards + # eventual demise and report their insubordination on shutdown. + self.undead_ledger: Final[List[IntegrationInstance]] = [] + + # Queue of newly reaped instances + self.reaped_instances: Final[queue.Queue[IntegrationInstance]] = ( + queue.Queue() + ) + + # Thread object, handle used to re-join the thread + self.reaper_thread: Optional[threading.Thread] = None + + # Count the dead + self.counter = 0 + + def reap(self, instance: IntegrationInstance): + """reap() submits an instance to the reaper thread. + + An instance that is passed to the reaper must not be used again. It may + not be dead yet, but it has no place among the living. + """ + LOG.info("Reaper: receiving %s", instance.instance.id) + + self.reaped_instances.put(instance) + with self.wake_reaper: + self.wake_reaper.notify() + LOG.info("Reaper: awakened to reap") + + def start(self): + """Spawn the reaper background thread.""" + LOG.info("Reaper: starting") + self.reaper_thread = threading.Thread( + target=self._reaper_loop, name="reaper" + ) + self.reaper_thread.start() + + def stop(self): + """Stop the reaper background thread and wait for completion.""" + LOG.info("Reaper: stopping") + self.exit_reaper.set() + with self.wake_reaper: + self.wake_reaper.notify() + LOG.info("Reaper: awakened to reap") + if self.reaper_thread and self.reaper_thread.is_alive(): + self.reaper_thread.join() + LOG.info("Reaper: stopped") + + def _destroy(self, instance: IntegrationInstance) -> bool: + """destroy() destroys an instance and returns True on success.""" + try: + LOG.info("Reaper: destroying %s", instance.instance.id) + instance.destroy() + self.counter += 1 + return True + except Exception as e: + LOG.warning( + "Error while tearing down instance %s: %s ", instance, e + ) + return False + + def _reaper_loop(self) -> None: + """reaper_loop() manages all instances that have been reaped + + tasks: + - destroy newly reaped instances + - manage a ledger undead instances + - periodically attempt to kill undead instances + - die when instructed to + - ensure that every reaped instance is destroyed at least once before + reaper dies + """ + LOG.info("Reaper: exalted in life, to assist others in death") + while True: + # nap until woken or timeout + with self.wake_reaper: + self.wake_reaper.wait(timeout=self.timeout) + if self._do_reap(): + break + LOG.info("Reaper: exited") + + def _do_reap(self) -> bool: + """_do_reap does a single pass of the reaper loop + + return True if the loop should exit + """ + + new_undead_instances: List[IntegrationInstance] = [] + + # first destroy all newly reaped instances + while not self.reaped_instances.empty(): + instance = self.reaped_instances.get_nowait() + success = self._destroy(instance) + if not success: + LOG.warning( + "Reaper: failed to destroy %s", + instance.instance.id, + ) + # failure to delete, add to the ledger + new_undead_instances.append(instance) + else: + LOG.info("Reaper: destroyed %s", instance.instance.id) + + # every instance has tried at least once and the reaper has been + # instructed to tear down - so do it + if self.exit_reaper.is_set(): + if not self.reaped_instances.empty(): + # race: an instance was added to the queue after iteration + # completed. Destroy the latest instance. + self._update_undead_ledger(new_undead_instances) + return False + self._update_undead_ledger(new_undead_instances) + LOG.info("Reaper: exiting") + if self.undead_ledger: + # undead instances exist - unclean teardown + LOG.info( + "Reaper: the faults of incompetent abilities will be " + "consigned to oblivion, as myself must soon be to the " + "mansions of rest." + ) + warnings.warn(f"Test instance(s) leaked: {self.undead_ledger}") + else: + LOG.info("Reaper: duties complete, my turn to rest") + LOG.info( + "Reaper: reaped %s/%s instances", + self.counter, + self.counter + len(self.undead_ledger), + ) + return True + + # attempt to destroy all instances which previously refused to destroy + for instance in self.undead_ledger: + if self.exit_reaper.is_set() and self.reaped_instances.empty(): + # don't retry instances if the exit_reaper Event is set + break + if self._destroy(instance): + self.undead_ledger.remove(instance) + LOG.info("Reaper: destroyed %s (undead)", instance.instance.id) + + self._update_undead_ledger(new_undead_instances) + return False + + def _update_undead_ledger( + self, new_undead_instances: List[IntegrationInstance] + ): + """update the ledger with newly undead instances""" + if new_undead_instances: + if self.undead_ledger: + LOG.info( + "Reaper: instance(s) not ready to die %s, will now join " + "the ranks of the undead: %s", + new_undead_instances, + self.undead_ledger, + ) + else: + LOG.info( + "Reaper: instance(s) not ready to die %s", + new_undead_instances, + ) + self.undead_ledger.extend(new_undead_instances) + return False diff --git a/tests/integration_tests/test_networking.py b/tests/integration_tests/test_networking.py index 53a7b8c5..4f4a6816 100644 --- a/tests/integration_tests/test_networking.py +++ b/tests/integration_tests/test_networking.py @@ -8,7 +8,7 @@ from cloudinit.subp import subp from tests.integration_tests import random_mac_address -from tests.integration_tests.clouds import IntegrationCloud +from tests.integration_tests.clouds import Ec2Cloud, IntegrationCloud from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import ( @@ -182,9 +182,7 @@ def test_applied(self, client: IntegrationInstance): pytest.param(NET_V2_MATCH_CONFIG, id="v2"), ), ) -def test_netplan_rendering( - net_config, session_cloud: IntegrationCloud, setup_image -): +def test_netplan_rendering(net_config, session_cloud: IntegrationCloud): mac_addr = random_mac_address() launch_kwargs = { "config_dict": { @@ -222,9 +220,7 @@ def test_netplan_rendering( reason="Test requires custom networking provided by LXD", ) @pytest.mark.parametrize("net_config", (NET_V1_NAME_TOO_LONG,)) -def test_schema_warnings( - net_config, session_cloud: IntegrationCloud, setup_image -): +def test_schema_warnings(net_config, session_cloud: IntegrationCloud): # TODO: This test takes a lot more time than it needs to. # The default launch wait will wait until cloud-init done, but the # init network stage will wait 2 minutes for network timeout. @@ -266,9 +262,7 @@ def test_schema_warnings( PLATFORM not in ("lxd_vm", "lxd_container"), reason="Test requires lxc exec feature due to broken network config", ) -def test_invalid_network_v2_netplan( - session_cloud: IntegrationCloud, setup_image -): +def test_invalid_network_v2_netplan(session_cloud: IntegrationCloud): mac_addr = random_mac_address() if PLATFORM == "lxd_vm": @@ -316,7 +310,7 @@ def test_invalid_network_v2_netplan( @pytest.mark.skipif(PLATFORM != "ec2", reason="test is ec2 specific") -def test_ec2_multi_nic_reboot(setup_image, session_cloud: IntegrationCloud): +def test_ec2_multi_nic_reboot(session_cloud: IntegrationCloud): """Tests that additional secondary NICs and secondary IPs on them are routable from non-local networks after a reboot event when network updates are configured on every boot.""" @@ -347,7 +341,7 @@ def test_ec2_multi_nic_reboot(setup_image, session_cloud: IntegrationCloud): @pytest.mark.adhoc # costly instance not available in all regions / azs @pytest.mark.skipif(PLATFORM != "ec2", reason="test is ec2 specific") -def test_ec2_multi_network_cards(setup_image, session_cloud: IntegrationCloud): +def test_ec2_multi_network_cards(session_cloud: Ec2Cloud): """ Tests that with an interface type with multiple network cards (non unique device indexes). diff --git a/tests/integration_tests/test_reaper.py b/tests/integration_tests/test_reaper.py new file mode 100644 index 00000000..d34b2a19 --- /dev/null +++ b/tests/integration_tests/test_reaper.py @@ -0,0 +1,141 @@ +"""reaper self-test""" + +import logging +import time +import warnings +from unittest import mock + +import pytest + +from tests.integration_tests import reaper +from tests.integration_tests.instances import IntegrationInstance + +LOG = logging.Logger(__name__) + + +class MockInstance(IntegrationInstance): + # because of instance id printing + instance = mock.Mock() + + def __init__(self, times_refused): + self.times_refused = times_refused + self.call_count = 0 + + # assert that destruction succeeded + self.stopped = False + + def destroy(self): + """destroy() only succeeds after failing N=times_refused times""" + if self.call_count == self.times_refused: + self.stopped = True + return + self.call_count += 1 + raise RuntimeError("I object!") + + +@pytest.mark.ci +class TestReaper: + def test_start_stop(self): + """basic setup teardown""" + + instance = MockInstance(0) + r = reaper._Reaper() + # start / stop + r.start() + r.stop() + # start / reap / stop + r.start() + r.reap(instance) + r.stop() + + # start / stop + r.start() + r.stop() + assert instance.stopped + + def test_basic_reap(self): + """basic setup teardown""" + + i_1 = MockInstance(0) + r = reaper._Reaper() + r.start() + r.reap(i_1) + r.stop() + assert i_1.stopped + + def test_unreaped_instance(self): + """a single warning should print for any number of leaked instances""" + + i_1 = MockInstance(64) + i_2 = MockInstance(64) + r = reaper._Reaper() + r.start() + r.reap(i_1) + r.reap(i_2) + with warnings.catch_warnings(record=True) as w: + r.stop() + assert len(w) == 1 + + def test_stubborn_reap(self): + """verify that stubborn instances are cleaned""" + + sleep_time = 0.000_001 + sleep_total = 0.0 + instances = [ + MockInstance(0), + MockInstance(3), + MockInstance(6), + MockInstance(9), + MockInstance(12), + MockInstance(9), + MockInstance(6), + MockInstance(3), + MockInstance(0), + ] + + # forcibly disallow sleeping, to avoid wasted time during tests + r = reaper._Reaper(timeout=0.0) + r.start() + for i in instances: + r.reap(i) + + # this should really take no time at all, waiting 1s should be plenty + # of time for the reaper to reap it when not sleeping + while sleep_total < 1.0: + # are any still undead? + any_undead = False + for i in instances: + if not i.stopped: + any_undead = True + break + if not any_undead: + # test passed + # Advance to GO, collect $400 + break + # sleep then recheck, incremental backoff + sleep_total += sleep_time + sleep_time *= 2 + time.sleep(sleep_time) + r.stop() + for i in instances: + assert i.stopped, ( + f"Reaper didn't reap stubborn instance {i} in {sleep_total}s. " + "Something appears to be broken in the reaper logic or test." + ) + + def test_start_stop_multiple(self): + """reap lots of instances + + obedient ones + """ + num = 64 + instances = [] + r = reaper._Reaper() + r.start() + for _ in range(num): + i = MockInstance(0) + instances.append(i) + r.reap(i) + r.stop() + for i in instances: + assert i.stopped diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index f5f51dc4..0aabbca9 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -143,13 +143,17 @@ def append_or_create_list( ignore_warnings, "Could not match supplied host pattern, ignoring:", ) - elif "oracle" == PLATFORM: + elif "oci" == PLATFORM: # LP: #1842752 ignore_errors = append_or_create_list( ignore_warnings, "Stderr: RTNETLINK answers: File exists" ) if isinstance(ignore_tracebacks, list): ignore_tracebacks.append("Stderr: RTNETLINK answers: File exists") + # Ubuntu lxd storage + ignore_warnings = append_or_create_list( + ignore_warnings, "thinpool by default on Ubuntu due to LP #1982780" + ) # LP: #1833446 ignore_warnings = append_or_create_list( ignore_warnings, diff --git a/tests/unittests/config/test_apt_configure_sources_list_v3.py b/tests/unittests/config/test_apt_configure_sources_list_v3.py index 5ba6dac8..cdb26c3b 100644 --- a/tests/unittests/config/test_apt_configure_sources_list_v3.py +++ b/tests/unittests/config/test_apt_configure_sources_list_v3.py @@ -150,6 +150,46 @@ Components: main restricted """ +EXAMPLE_CUSTOM_KEY_TMPL_DEB822 = """\ +## template:jinja +# Generated by cloud-init +Types: deb deb-src +URIs: {{mirror}} +Suites: {{codename}} {{codename}}-updates +Components: main restricted +Signed-By: {{ + primary_key + | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true) +}} + +# Security section +Types: deb deb-src +URIs: {{security}} +Suites: {{codename}}-security +Components: main restricted +Signed-By: {{ + security_key + | default(primary_key, true) + | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true) +}} +""" + +EXPECTED_PM_BASE_CUSTOM_KEYS = """\ +# Generated by cloud-init +Types: deb deb-src +URIs: http://local.ubuntu.com/ +Suites: fakerel fakerel-updates +Components: main restricted +""" + +EXPECTED_SM_BASE_CUSTOM_KEYS = """\ +# Security section +Types: deb deb-src +URIs: http://local.ubuntu.com/ +Suites: fakerel-security +Components: main restricted +""" + @pytest.mark.usefixtures("fake_filesystem") class TestAptSourceConfigSourceList: @@ -332,3 +372,94 @@ def test_apt_v3_srcl_custom_deb822_feature_aware( sources_file = tmpdir.join(apt_file) assert expected == sources_file.read() assert 0o644 == stat.S_IMODE(sources_file.stat().mode) + + @pytest.mark.parametrize( + "distro,pm,pmkey,sm,smkey", + ( + pytest.param( + "ubuntu", + "http://local.ubuntu.com/", + "fakekey 4321", + "http://local.ubuntu.com/", + "fakekey 1234", + ), + pytest.param( + "ubuntu", + "http://local.ubuntu.com/", + "fakekey 4321", + None, + None, + ), + pytest.param( + "ubuntu", "http://local.ubuntu.com/", None, None, None + ), + pytest.param( + "ubuntu", + "http://local.ubuntu.com/", + None, + "http://local.ubuntu.com/", + "fakekey 1234", + ), + ), + ) + def test_apt_v3_srcl_deb822_custom_psm_keys( + self, + distro, + pm, + pmkey, + sm, + smkey, + mocker, + tmpdir, + ): + """test_apt_v3_srcl_deb822_custom_psm_keys - Test the ability to + specify raw GPG keys alongside primary and security mirrors such + that the keys are both added to the trusted.gpg.d directory + also the ubuntu.sources template + """ + + self.deb822 = mocker.patch.object( + cc_apt_configure.features, "APT_DEB822_SOURCE_LIST_FILE", True + ) + + tmpl_file = f"/etc/cloud/templates/sources.list.{distro}.deb822.tmpl" + tmpl_content = EXAMPLE_CUSTOM_KEY_TMPL_DEB822 + util.write_file(tmpl_file, tmpl_content) + + # Base config + cfg = { + "preserve_sources_list": False, + "primary": [{"arches": ["default"], "uri": pm}], + } + + # Add defined variables to the config + if pmkey: + cfg["primary"][0]["key"] = pmkey + if sm: + cfg["security"] = [{"arches": ["default"], "uri": sm}] + if smkey: + cfg["security"][0]["key"] = smkey + + mycloud = get_cloud(distro) + cc_apt_configure.handle("test", {"apt": cfg}, mycloud, None) + + apt_file = f"/etc/apt/sources.list.d/{distro}.sources" + sources_file = tmpdir.join(apt_file) + + default_keyring = f"/usr/share/keyrings/{distro}-archive-keyring.gpg" + trusted_keys_dir = "/etc/apt/trusted.gpg.d/" + primary_keyring = f"{trusted_keys_dir}primary.gpg" + security_keyring = f"{trusted_keys_dir}security.gpg" + + primary_keyring_path = primary_keyring if pmkey else default_keyring + primary_signature = f"Signed-By: {primary_keyring_path}" + expected_pm = f"{EXPECTED_PM_BASE_CUSTOM_KEYS}{primary_signature}" + + fallback_keyring = primary_keyring if pmkey else default_keyring + security_keyring_path = security_keyring if smkey else fallback_keyring + + security_signature = f"Signed-By: {security_keyring_path}" + expected_sm = f"{EXPECTED_SM_BASE_CUSTOM_KEYS}{security_signature}" + + expected = f"{expected_pm}\n\n{expected_sm}\n" + assert expected == sources_file.read() diff --git a/tests/unittests/config/test_cc_ansible.py b/tests/unittests/config/test_cc_ansible.py index 5bb00fdc..37aa40fd 100644 --- a/tests/unittests/config/test_cc_ansible.py +++ b/tests/unittests/config/test_cc_ansible.py @@ -323,12 +323,16 @@ def test_required_keys(self, cfg, exception, mocker): def test_deps_not_installed(self, m_which): """assert exception raised if package not installed""" with raises(ValueError): - cc_ansible.AnsiblePullDistro(get_cloud().distro).check_deps() + cc_ansible.AnsiblePullDistro( + get_cloud().distro, "root" + ).check_deps() @mock.patch(M_PATH + "subp.which", return_value=True) def test_deps(self, m_which): """assert exception not raised if package installed""" - cc_ansible.AnsiblePullDistro(get_cloud().distro).check_deps() + cc_ansible.AnsiblePullDistro( + get_cloud().distro, "ansible" + ).check_deps() @mark.serial @mock.patch(M_PATH + "subp.subp", return_value=("stdout", "stderr")) @@ -390,7 +394,7 @@ def test_ansible_pull(self, m_subp1, m_subp2, m_which, cfg, expected): ansible_pull = ( cc_ansible.AnsiblePullPip(distro, "ansible") if pull_type == "pip" - else cc_ansible.AnsiblePullDistro(distro) + else cc_ansible.AnsiblePullDistro(distro, "") ) cc_ansible.run_ansible_pull( ansible_pull, deepcopy(cfg["ansible"]["pull"]) @@ -415,7 +419,7 @@ def test_do_not_run(self, m_validate): def test_parse_version_distro(self, m_subp): """Verify that the expected version is returned""" assert cc_ansible.AnsiblePullDistro( - get_cloud().distro + get_cloud().distro, "" ).get_version() == lifecycle.Version(2, 10, 8) @mock.patch("cloudinit.subp.subp", side_effect=[(pip_version, "")]) diff --git a/tests/unittests/config/test_cc_apt_configure.py b/tests/unittests/config/test_cc_apt_configure.py index 7b4ce012..3651355e 100644 --- a/tests/unittests/config/test_cc_apt_configure.py +++ b/tests/unittests/config/test_cc_apt_configure.py @@ -328,7 +328,7 @@ def test_remove_source( Components: main restricted universe multiverse Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg""" } - cc_apt.generate_sources_list(cfg, "noble", {}, cloud) + cc_apt.generate_sources_list(cfg, "noble", {}, cloud, {}) if expected_content is None: assert not sources_file.exists() assert f"Removing {sources_file} to favor deb822" in caplog.text diff --git a/tests/unittests/config/test_cc_chef.py b/tests/unittests/config/test_cc_chef.py index eea21ad6..8ba4aa0e 100644 --- a/tests/unittests/config/test_cc_chef.py +++ b/tests/unittests/config/test_cc_chef.py @@ -17,27 +17,28 @@ from tests.helpers import cloud_init_project_dir from tests.unittests.helpers import ( SCHEMA_EMPTY_ERROR, - FilesystemMockingTestCase, - ResponsesTestCase, mock, skipIf, skipUnlessJsonSchema, ) from tests.unittests.util import MockDistro, get_cloud -CLIENT_TEMPL = cloud_init_project_dir("templates/chef_client.rb.tmpl") +try: + client_path = cloud_init_project_dir("templates/chef_client.rb.tmpl") + with open(client_path) as stream: + CLIENT_TEMPL = stream.read() +except FileNotFoundError: + CLIENT_TEMPL = "" -class TestInstallChefOmnibus(ResponsesTestCase): - def setUp(self): - super(TestInstallChefOmnibus, self).setUp() - self.new_root = self.tmp_dir() +class TestInstallChefOmnibus: + @responses.activate @mock.patch("cloudinit.config.cc_chef.OMNIBUS_URL", cc_chef.OMNIBUS_URL) def test_install_chef_from_omnibus_runs_chef_url_content(self): """install_chef_from_omnibus calls subp_blob_in_tempfile.""" response = b'#!/bin/bash\necho "Hi Mom"' - self.responses.add( + responses.add( responses.GET, cc_chef.OMNIBUS_URL, body=response, status=200 ) ret = (None, None) # stdout, stderr but capture=False @@ -49,28 +50,25 @@ def test_install_chef_from_omnibus_runs_chef_url_content(self): cc_chef.install_chef_from_omnibus(distro=distro) # admittedly whitebox, but assuming subp_blob_in_tempfile works # this should be fine. - self.assertEqual( - [ - mock.call( - blob=response, - args=[], - basename="chef-omnibus-install", - capture=False, - distro=distro, - ) - ], - m_subp_blob.call_args_list, - ) + assert [ + mock.call( + blob=response, + args=[], + basename="chef-omnibus-install", + capture=False, + distro=distro, + ) + ] == m_subp_blob.call_args_list @mock.patch("cloudinit.config.cc_chef.url_helper.readurl") @mock.patch("cloudinit.config.cc_chef.subp_blob_in_tempfile") - def test_install_chef_from_omnibus_retries_url(self, m_subp_blob, m_rdurl): + def test_install_chef_from_omnibus_retries_url( + self, m_subp_blob, m_rdurl, tmpdir + ): """install_chef_from_omnibus retries OMNIBUS_URL upon failure.""" class FakeURLResponse: - contents = '#!/bin/bash\necho "Hi Mom" > {0}/chef.out'.format( - self.new_root - ) + contents = f'#!/bin/bash\necho "Hi Mom" > {tmpdir}/chef.out' m_rdurl.return_value = FakeURLResponse() @@ -80,10 +78,13 @@ class FakeURLResponse: "retries": cc_chef.OMNIBUS_URL_RETRIES, "url": cc_chef.OMNIBUS_URL, } - self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) + assert expected_kwargs == m_rdurl.call_args_list[0][1] cc_chef.install_chef_from_omnibus(retries=10, distro=distro) expected_kwargs = {"retries": 10, "url": cc_chef.OMNIBUS_URL} - self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) + cc_chef.install_chef_from_omnibus( + retries=10, distro=distro, omnibus_version="2.0" + ) + assert expected_kwargs == m_rdurl.call_args_list[1][1] expected_subp_kwargs = { "args": ["-v", "2.0"], "basename": "chef-omnibus-install", @@ -91,17 +92,18 @@ class FakeURLResponse: "capture": False, "distro": distro, } - self.assertCountEqual( - expected_subp_kwargs, m_subp_blob.call_args_list[0][1] - ) + assert expected_subp_kwargs == m_subp_blob.call_args_list[2][1] + @responses.activate @mock.patch("cloudinit.config.cc_chef.OMNIBUS_URL", cc_chef.OMNIBUS_URL) @mock.patch("cloudinit.config.cc_chef.subp_blob_in_tempfile") - def test_install_chef_from_omnibus_has_omnibus_version(self, m_subp_blob): + def test_install_chef_from_omnibus_has_omnibus_version( + self, m_subp_blob, tmpdir + ): """install_chef_from_omnibus provides version arg to OMNIBUS_URL.""" - chef_outfile = self.tmp_path("chef.out", self.new_root) + chef_outfile = tmpdir / "chef.out" response = '#!/bin/bash\necho "Hi Mom" > {0}'.format(chef_outfile) - self.responses.add(responses.GET, cc_chef.OMNIBUS_URL, body=response) + responses.add(responses.GET, cc_chef.OMNIBUS_URL, body=response) distro = mock.Mock() cc_chef.install_chef_from_omnibus(distro=distro, omnibus_version="2.0") @@ -109,33 +111,27 @@ def test_install_chef_from_omnibus_has_omnibus_version(self, m_subp_blob): expected_kwargs = { "args": ["-v", "2.0"], "basename": "chef-omnibus-install", - "blob": response, + "blob": response.encode("utf-8"), "capture": False, "distro": distro, } - self.assertCountEqual(expected_kwargs, called_kwargs) + assert expected_kwargs == called_kwargs -class TestChef(FilesystemMockingTestCase): - def setUp(self): - super(TestChef, self).setUp() - self.tmp = self.tmp_dir() +@pytest.mark.usefixtures("fake_filesystem") +class TestChef: def test_no_config(self): - self.patchUtils(self.tmp) - self.patchOS(self.tmp) - + """No chef directories are created on when no chef config provided""" cfg = {} cc_chef.handle("chef", cfg, get_cloud(), []) for d in cc_chef.CHEF_DIRS: - self.assertFalse(os.path.isdir(d)) + assert not os.path.isdir(d) - @skipIf( - not os.path.isfile(CLIENT_TEMPL), CLIENT_TEMPL + " is not available" - ) + @skipIf(not CLIENT_TEMPL, "templates/chef_client.rb.tmpl is not available") def test_basic_config(self): """ - test basic config looks sane + test basic config looks correct # This should create a file of the format... # Created by cloud-init v. 0.7.6 on Sat, 11 Oct 2014 23:57:21 +0000 @@ -150,17 +146,15 @@ def test_basic_config(self): environment "_default" node_name "iid-datasource-none" json_attribs "/etc/chef/firstboot.json" - file_cache_path "/var/cache/chef" - file_backup_path "/var/backups/chef" + file_cache_path "/var/chef/cache" + file_backup_path "/var/chef/backup" pid_file "/var/run/chef/client.pid" Chef::Log::Formatter.show_time = true encrypted_data_bag_secret "/etc/chef/encrypted_data_bag_secret" """ - tpl_file = util.load_text_file(CLIENT_TEMPL) - self.patchUtils(self.tmp) - self.patchOS(self.tmp) - - util.write_file("/etc/cloud/templates/chef_client.rb.tmpl", tpl_file) + util.write_file( + "/etc/cloud/templates/chef_client.rb.tmpl", CLIENT_TEMPL + ) cfg = { "chef": { "chef_license": "accept", @@ -175,7 +169,7 @@ def test_basic_config(self): } cc_chef.handle("chef", cfg, get_cloud(), []) for d in cc_chef.CHEF_DIRS: - self.assertTrue(os.path.isdir(d)) + assert os.path.isdir(d) c = util.load_text_file(cc_chef.CHEF_RB_PATH) # the content of these keys is not expected to be rendered to tmpl @@ -183,21 +177,18 @@ def test_basic_config(self): for k, v in cfg["chef"].items(): if k in unrendered_keys: continue - self.assertIn(v, c) + assert v in c for k, v in cc_chef.CHEF_RB_TPL_DEFAULTS.items(): if k in unrendered_keys: continue # the value from the cfg overrides that in the default val = cfg["chef"].get(k, v) if isinstance(val, str): - self.assertIn(val, c) + assert val in c c = util.load_text_file(cc_chef.CHEF_FB_PATH) - self.assertEqual({}, json.loads(c)) + assert {} == json.loads(c) def test_firstboot_json(self): - self.patchUtils(self.tmp) - self.patchOS(self.tmp) - cfg = { "chef": { "server_url": "localhost", @@ -210,23 +201,76 @@ def test_firstboot_json(self): } cc_chef.handle("chef", cfg, get_cloud(), []) c = util.load_text_file(cc_chef.CHEF_FB_PATH) - self.assertEqual( - { + assert { + "run_list": ["a", "b", "c"], + "c": "d", + } == json.loads(c) + + @pytest.mark.parametrize( + "file_content, shutil_moves, expected_msg_count", + ( + pytest.param({}, [], {}, id="no_migration_when_dirs_empty"), + pytest.param( + {"/var/cache/chef/cache.1": "cache1"}, + [mock.call("/var/cache/chef/cache.1", "/var/chef/cache")], + {"Moving /var/cache/chef/cache.1 to /var/chef/cache": 1}, + id="migration_when_old_cache_dir_present", + ), + pytest.param( + {"/var/backups/chef/backup.1": "backup1"}, + [mock.call("/var/backups/chef/backup.1", "/var/chef/backup")], + {"Moving /var/backups/chef/backup.1 to /var/chef/backup": 1}, + id="migration_when_old_backups_dir_present", + ), + pytest.param( + { + "/var/backups/chef/backup.1": "backup1", + "/var/chef/backup/backup.1": "backup1", + }, + [], + { + "Ignoring migration of /var/backups/chef/backup.1." + " File already exists in /var/chef/backup": 1 + }, + id="migration_skips_when_migrated_file_present", + ), + ), + ) + def test_migrate_chef_config_dirs( + self, file_content, shutil_moves, expected_msg_count, caplog + ): + """When present, old backup and cache dirs migrated to defaults""" + cfg = { + "chef": { + "server_url": "localhost", + "validation_name": "bob", "run_list": ["a", "b", "c"], - "c": "d", + "initial_attributes": { + "c": "d", + }, }, - json.loads(c), + } + for file_path in file_content: + util.ensure_dir(os.path.dirname(file_path)) + util.write_file(file_path, file_content[file_path]) + util.write_file( + "/etc/cloud/templates/chef_client.rb.tmpl", CLIENT_TEMPL ) - - @skipIf( - not os.path.isfile(CLIENT_TEMPL), CLIENT_TEMPL + " is not available" - ) + with mock.patch("cloudinit.config.cc_chef.shutil.move") as m_shutil: + cc_chef.handle("chef", cfg, get_cloud(), []) + assert m_shutil.call_args_list == shutil_moves + if len(file_content) == 0: + # no files to migrate, so we don't expect any messages + assert "Moving" not in caplog.text + for expected_msg, count in expected_msg_count.items(): + assert caplog.text.count(expected_msg) == count + + @skipIf(not CLIENT_TEMPL, "templates/chef_client.rb.tmpl is not available") def test_template_deletes(self): - tpl_file = util.load_text_file(CLIENT_TEMPL) - self.patchUtils(self.tmp) - self.patchOS(self.tmp) - util.write_file("/etc/cloud/templates/chef_client.rb.tmpl", tpl_file) + util.write_file( + "/etc/cloud/templates/chef_client.rb.tmpl", CLIENT_TEMPL + ) cfg = { "chef": { "server_url": "localhost", @@ -237,19 +281,15 @@ def test_template_deletes(self): } cc_chef.handle("chef", cfg, get_cloud(), []) c = util.load_text_file(cc_chef.CHEF_RB_PATH) - self.assertNotIn("json_attribs", c) - self.assertNotIn("Formatter.show_time", c) + assert "json_attribs" not in c + assert "Formatter.show_time" not in c - @skipIf( - not os.path.isfile(CLIENT_TEMPL), CLIENT_TEMPL + " is not available" - ) + @skipIf(not CLIENT_TEMPL, "templates/chef_client.rb.tmpl is not available") def test_validation_cert_and_validation_key(self): # test validation_cert content is written to validation_key path - tpl_file = util.load_text_file(CLIENT_TEMPL) - self.patchUtils(self.tmp) - self.patchOS(self.tmp) - - util.write_file("/etc/cloud/templates/chef_client.rb.tmpl", tpl_file) + util.write_file( + "/etc/cloud/templates/chef_client.rb.tmpl", CLIENT_TEMPL + ) v_path = "/etc/chef/vkey.pem" v_cert = "this is my cert" cfg = { @@ -262,15 +302,13 @@ def test_validation_cert_and_validation_key(self): } cc_chef.handle("chef", cfg, get_cloud(), []) content = util.load_text_file(cc_chef.CHEF_RB_PATH) - self.assertIn(v_path, content) + assert v_path in content util.load_text_file(v_path) - self.assertEqual(v_cert, util.load_text_file(v_path)) + assert v_cert == util.load_text_file(v_path) + @skipIf(not CLIENT_TEMPL, "templates/chef_client.rb.tmpl is not available") def test_validation_cert_with_system(self): # test validation_cert content is not written over system file - tpl_file = util.load_text_file(CLIENT_TEMPL) - self.patchUtils(self.tmp) - self.patchOS(self.tmp) v_path = "/etc/chef/vkey.pem" v_cert = "system" @@ -283,13 +321,15 @@ def test_validation_cert_with_system(self): "validation_cert": v_cert, }, } - util.write_file("/etc/cloud/templates/chef_client.rb.tmpl", tpl_file) + util.write_file( + "/etc/cloud/templates/chef_client.rb.tmpl", CLIENT_TEMPL + ) util.write_file(v_path, expected_cert) cc_chef.handle("chef", cfg, get_cloud(), []) content = util.load_text_file(cc_chef.CHEF_RB_PATH) - self.assertIn(v_path, content) + assert v_path in content util.load_text_file(v_path) - self.assertEqual(expected_cert, util.load_text_file(v_path)) + assert expected_cert == util.load_text_file(v_path) @skipUnlessJsonSchema() diff --git a/tests/unittests/config/test_cc_power_state_change.py b/tests/unittests/config/test_cc_power_state_change.py index 8a1886ca..ce8d74b2 100644 --- a/tests/unittests/config/test_cc_power_state_change.py +++ b/tests/unittests/config/test_cc_power_state_change.py @@ -47,7 +47,7 @@ def test_empty_mode(self): self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_valid_modes(self): - cfg = {"power_state": {}} + cfg: dict = {"power_state": {}} for mode in ("halt", "poweroff", "reboot"): cfg["power_state"]["mode"] = mode check_lps_ret(psc.load_power_state(cfg, self.dist), mode=mode) diff --git a/tests/unittests/config/test_cc_rsyslog.py b/tests/unittests/config/test_cc_rsyslog.py index ac662f4c..0fb08130 100644 --- a/tests/unittests/config/test_cc_rsyslog.py +++ b/tests/unittests/config/test_cc_rsyslog.py @@ -340,7 +340,7 @@ def test_install_rsyslog_on_freebsd(self, m_which): with mock.patch.object( cloud.distro, "install_packages" ) as m_install: - handle("rsyslog", {"rsyslog": config}, cloud, None) + handle("rsyslog", {"rsyslog": config}, cloud, []) m_which.assert_called_with(config["check_exe"]) m_install.assert_called_with(config["packages"]) @@ -356,6 +356,6 @@ def test_no_install_rsyslog_with_check_exe(self, m_which, m_isbsd): m_isbsd.return_value = False m_which.return_value = "/usr/sbin/rsyslogd" with mock.patch.object(cloud.distro, "install_packages") as m_install: - handle("rsyslog", {"rsyslog": config}, cloud, None) + handle("rsyslog", {"rsyslog": config}, cloud, []) m_which.assert_called_with(config["check_exe"]) m_install.assert_not_called() diff --git a/tests/unittests/config/test_cc_set_passwords.py b/tests/unittests/config/test_cc_set_passwords.py index c068f62d..a706917d 100644 --- a/tests/unittests/config/test_cc_set_passwords.py +++ b/tests/unittests/config/test_cc_set_passwords.py @@ -27,7 +27,9 @@ ["systemctl", "show", "--property", "ActiveState", "--value", "ssh"] ) SYSTEMD_RESTART_CALL = mock.call( - ["systemctl", "restart", "ssh"], capture=True, rcs=None + ["systemctl", "restart", "ssh", "--job-mode=ignore-dependencies"], + capture=True, + rcs=None, ) SERVICE_RESTART_CALL = mock.call( ["service", "ssh", "restart"], capture=True, rcs=None diff --git a/tests/unittests/config/test_cc_snap.py b/tests/unittests/config/test_cc_snap.py index 9666ab06..c5026783 100644 --- a/tests/unittests/config/test_cc_snap.py +++ b/tests/unittests/config/test_cc_snap.py @@ -314,7 +314,7 @@ def test_handle_adds_assertions( cfg = { "snap": {"assertions": [SYSTEM_USER_ASSERTION, ACCOUNT_ASSERTION]} } - handle("snap", cfg=cfg, cloud=fake_cloud, args=None) + handle("snap", cfg=cfg, cloud=fake_cloud, args=[]) content = "\n".join(cfg["snap"]["assertions"]) util.write_file(compare_file, content.encode("utf-8")) assert util.load_text_file(compare_file) == util.load_text_file( diff --git a/tests/unittests/config/test_cc_timezone.py b/tests/unittests/config/test_cc_timezone.py index b8ba443f..3f7fc22b 100644 --- a/tests/unittests/config/test_cc_timezone.py +++ b/tests/unittests/config/test_cc_timezone.py @@ -35,5 +35,5 @@ def test_set_timezone_sles(self, fake_filesystem): assert {"TIMEZONE": cfg["timezone"]} == dict(n_cfg) - contents = util.load_text_file("/etc/localtime") - assert dummy_contents == contents.strip() + localtime_contents = util.load_text_file("/etc/localtime") + assert dummy_contents == localtime_contents.strip() diff --git a/tests/unittests/config/test_cc_ubuntu_autoinstall.py b/tests/unittests/config/test_cc_ubuntu_autoinstall.py index 1a492ad0..5504164c 100644 --- a/tests/unittests/config/test_cc_ubuntu_autoinstall.py +++ b/tests/unittests/config/test_cc_ubuntu_autoinstall.py @@ -38,33 +38,6 @@ ) -class TestvalidateConfigSchema: - @pytest.mark.parametrize( - "src_cfg,error_msg", - [ - pytest.param( - {"autoinstall": 1}, - "autoinstall: Expected dict type but found: int", - id="err_non_dict", - ), - pytest.param( - {"autoinstall": {}}, - "autoinstall: Missing required 'version' key", - id="err_require_version_key", - ), - pytest.param( - {"autoinstall": {"version": "v1"}}, - "autoinstall.version: Expected int type but found: str", - id="err_version_non_int", - ), - ], - ) - def test_runtime_validation_errors(self, src_cfg, error_msg): - """cloud-init raises errors at runtime on invalid autoinstall config""" - with pytest.raises(SchemaValidationError, match=error_msg): - cc_ubuntu_autoinstall.validate_config_schema(src_cfg) - - @mock.patch(MODPATH + "util.wait_for_snap_seeded") @mock.patch(MODPATH + "subp.subp") class TestHandleAutoinstall: @@ -73,14 +46,6 @@ class TestHandleAutoinstall: @pytest.mark.parametrize( "cfg,snap_list,subp_calls,logs,snap_wait_called", [ - pytest.param( - {}, - SAMPLE_SNAP_LIST_OUTPUT, - [], - ["Skipping module named name, no 'autoinstall' key"], - False, - id="skip_no_cfg", - ), pytest.param( {"autoinstall": {"version": 1}}, SAMPLE_SNAP_LIST_OUTPUT, @@ -149,6 +114,8 @@ class TestAutoInstallSchema: {"autoinstall": {}}, "autoinstall: 'version' is a required property", ), + ({"autoinstall": {"version": 1}}, None), + ({"autoinstall": {"version": "v1"}}, "is not of type 'integer'"), ), ) @skipUnlessJsonSchema() diff --git a/tests/unittests/config/test_cc_ubuntu_pro.py b/tests/unittests/config/test_cc_ubuntu_pro.py index 056a2542..09d1d8ed 100644 --- a/tests/unittests/config/test_cc_ubuntu_pro.py +++ b/tests/unittests/config/test_cc_ubuntu_pro.py @@ -57,19 +57,24 @@ def fake_uaclient(mocker): m_uaclient = mock.Mock() sys.modules["uaclient"] = m_uaclient + mock_exceptions_module = mock.Mock() # Exceptions - _exceptions = namedtuple( - "exceptions", + Exceptions = namedtuple( + "Exceptions", [ "UserFacingError", "AlreadyAttachedError", ], - )( + ) + mock_exceptions_module.UserFacingError = FakeUserFacingError + mock_exceptions_module.AlreadyAttachedError = FakeAlreadyAttachedError + sys.modules["uaclient.api.exceptions"] = mock_exceptions_module + _exceptions = Exceptions( FakeUserFacingError, FakeAlreadyAttachedError, ) - sys.modules["uaclient.api.exceptions"] = _exceptions + return _exceptions @pytest.mark.usefixtures("fake_uaclient") @@ -834,7 +839,7 @@ def test_handle_attach( caplog, ): """Non-Pro schemas and instance.""" - handle("nomatter", cfg=cfg, cloud=cloud, args=None) + handle("nomatter", cfg=cfg, cloud=cloud, args=[]) for record_tuple in log_record_tuples: assert record_tuple in caplog.record_tuples if maybe_install_call_args_list is not None: @@ -961,7 +966,7 @@ def test_handle_auto_attach_vs_attach( m_auto_attach.side_effect = auto_attach_side_effect with expectation: - handle("nomatter", cfg=cfg, cloud=cloud, args=None) + handle("nomatter", cfg=cfg, cloud=cloud, args=[]) for record_tuple in log_record_tuples: assert record_tuple in caplog.record_tuples @@ -1006,7 +1011,7 @@ def test_no_fallback_attach( enable or disable pro auto-attach. """ m_should_auto_attach.return_value = is_pro - handle("nomatter", cfg=cfg, cloud=self.cloud, args=None) + handle("nomatter", cfg=cfg, cloud=self.cloud, args=[]) assert not m_attach.call_args_list @pytest.mark.parametrize( @@ -1061,7 +1066,7 @@ def test_handle_errors(self, cfg, match): "nomatter", cfg=cfg, cloud=self.cloud, - args=None, + args=[], ) @mock.patch(f"{MPATH}.subp.subp") @@ -1087,7 +1092,7 @@ def test_pro_config_error_invalid_url(self, m_subp, caplog): "nomatter", cfg=cfg, cloud=self.cloud, - args=None, + args=[], ) assert not caplog.text @@ -1107,7 +1112,7 @@ def test_fallback_to_attach_no_token( "nomatter", cfg=cfg, cloud=self.cloud, - args=None, + args=[], ) assert [] == m_subp.call_args_list assert ( diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 8782cfdf..44cef64d 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -1164,6 +1164,72 @@ def test_main_absent_config_file(self, _read_cfg_paths, capsys): ), "Valid schema", ), + ( + "network-config", + ( + b"network:\n version: 1\n config:\n - type: physical\n" + b" name: eth0\n subnets:\n - type: manual\n" + ), + "Valid schema", + ), + ( + "network-config", + ( + b"network:\n version: 1\n config:\n - type: physical\n" + b" name: eth0\n subnets:\n - type: static\n" + ), + "Valid schema", + ), + ( + "network-config", + ( + b"network:\n version: 1\n config:\n - type: physical\n" + b" name: eth0\n subnets:\n - type: static6\n" + ), + "Valid schema", + ), + ( + "network-config", + ( + b"network:\n version: 1\n config:\n - type: physical\n" + b" name: eth0\n subnets:\n - type: dhcp6\n" + ), + "Valid schema", + ), + ( + "network-config", + ( + b"network:\n version: 1\n config:\n - type: physical\n" + b" name: eth0\n subnets:\n - type: dhcp4\n" + ), + "Valid schema", + ), + ( + "network-config", + ( + b"network:\n version: 1\n config:\n - type: physical\n" + b" name: eth0\n subnets:\n - type: ipv6_slaac\n" + ), + "Valid schema", + ), + ( + "network-config", + ( + b"network:\n version: 1\n config:\n - type: physical\n" + b" name: eth0\n subnets:\n" + b" - type: ipv6_dhcpv6-stateful\n" + ), + "Valid schema", + ), + ( + "network-config", + ( + b"network:\n version: 1\n config:\n - type: physical\n" + b" name: eth0\n subnets:\n" + b" - type: ipv6_dhcpv6-stateless\n" + ), + "Valid schema", + ), ), ) @mock.patch("cloudinit.net.netplan.available", return_value=False) diff --git a/tests/unittests/distros/test_gentoo.py b/tests/unittests/distros/test_gentoo.py index a307b9a2..979e6d82 100644 --- a/tests/unittests/distros/test_gentoo.py +++ b/tests/unittests/distros/test_gentoo.py @@ -2,27 +2,41 @@ from cloudinit import atomic_helper, util from tests.unittests.distros import _get_distro -from tests.unittests.helpers import CiTestCase +from tests.unittests.helpers import CiTestCase, mock class TestGentoo(CiTestCase): - def test_write_hostname(self): + def test_write_hostname(self, whatever=False): distro = _get_distro("gentoo") hostname = "myhostname" hostfile = self.tmp_path("hostfile") distro._write_hostname(hostname, hostfile) - self.assertEqual( - 'hostname="myhostname"\n', util.load_text_file(hostfile) - ) + if distro.uses_systemd(): + self.assertEqual("myhostname\n", util.load_text_file(hostfile)) + else: + self.assertEqual( + 'hostname="myhostname"\n', util.load_text_file(hostfile) + ) - def test_write_existing_hostname_with_comments(self): + def test_write_existing_hostname_with_comments(self, whatever=False): distro = _get_distro("gentoo") hostname = "myhostname" contents = '#This is the hostname\nhostname="localhost"' hostfile = self.tmp_path("hostfile") atomic_helper.write_file(hostfile, contents, omode="w") distro._write_hostname(hostname, hostfile) - self.assertEqual( - '#This is the hostname\nhostname="myhostname"\n', - util.load_text_file(hostfile), - ) + if distro.uses_systemd(): + self.assertEqual( + "#This is the hostname\nmyhostname\n", + util.load_text_file(hostfile), + ) + else: + self.assertEqual( + '#This is the hostname\nhostname="myhostname"\n', + util.load_text_file(hostfile), + ) + + +@mock.patch("cloudinit.distros.uses_systemd", return_value=False) +class TestGentooOpenRC(TestGentoo): + pass diff --git a/tests/unittests/distros/test_netconfig.py b/tests/unittests/distros/test_netconfig.py index b447757b..54b387ea 100644 --- a/tests/unittests/distros/test_netconfig.py +++ b/tests/unittests/distros/test_netconfig.py @@ -691,12 +691,16 @@ def control_path(self): return "/etc/sysconfig/network" def _apply_and_verify( - self, apply_fn, config, expected_cfgs=None, bringup=False + self, + apply_fn, + config, + expected_cfgs=None, + bringup=False, + tmpd=None, ): if not expected_cfgs: raise ValueError("expected_cfg must not be None") - tmpd = None with mock.patch("cloudinit.net.sysconfig.available") as m_avail: m_avail.return_value = True with self.reRooted(tmpd) as tmpd: @@ -785,6 +789,58 @@ def test_apply_network_config_ipv6_rh(self): expected_cfgs=expected_cfgs.copy(), ) + def test_sysconfig_network_no_overwite_ipv6_rh(self): + expected_cfgs = { + self.ifcfg_path("eth0"): dedent( + """\ + BOOTPROTO=none + DEFROUTE=yes + DEVICE=eth0 + IPV6ADDR=2607:f0d0:1002:0011::2/64 + IPV6INIT=yes + IPV6_AUTOCONF=no + IPV6_DEFAULTGW=2607:f0d0:1002:0011::1 + IPV6_FORCE_ACCEPT_RA=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """ + ), + self.ifcfg_path("eth1"): dedent( + """\ + BOOTPROTO=dhcp + DEVICE=eth1 + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """ + ), + self.control_path(): dedent( + """\ + NETWORKING=yes + NETWORKING_IPV6=yes + IPV6_AUTOCONF=no + NOZEROCONF=yes + """ + ), + } + tmpdir = self.tmp_dir() + file_mode = 0o644 + # pre-existing config in /etc/sysconfig/network should not be removed + with self.reRooted(tmpdir) as tmpdir: + util.write_file( + self.control_path(), + "".join("NOZEROCONF=yes") + "\n", + file_mode, + ) + + self._apply_and_verify( + self.distro.apply_network_config, + V1_NET_CFG_IPV6, + expected_cfgs=expected_cfgs.copy(), + tmpd=tmpdir, + ) + def test_vlan_render_unsupported(self): """Render officially unsupported vlan names.""" cfg = { diff --git a/tests/unittests/net/test_network_state.py b/tests/unittests/net/test_network_state.py index a03f60f8..5161b9cc 100644 --- a/tests/unittests/net/test_network_state.py +++ b/tests/unittests/net/test_network_state.py @@ -1,6 +1,8 @@ # This file is part of cloud-init. See LICENSE file for license information. import ipaddress +from typing import Dict from unittest import mock +from unittest.mock import MagicMock import pytest import yaml @@ -67,9 +69,10 @@ def setUp(self): super(TestNetworkStateParseConfig, self).setUp() nsi_path = netstate_path + ".NetworkStateInterpreter" self.add_patch(nsi_path, "m_nsi") + self.m_nsi: MagicMock def test_missing_version_returns_none(self): - ncfg = {} + ncfg: Dict[str, int] = {} with self.assertRaises(RuntimeError): network_state.parse_net_config_data(ncfg) diff --git a/tests/unittests/runs/test_merge_run.py b/tests/unittests/runs/test_merge_run.py index e7f32d03..7cd43c63 100644 --- a/tests/unittests/runs/test_merge_run.py +++ b/tests/unittests/runs/test_merge_run.py @@ -66,7 +66,7 @@ def test_none_ds(self): self.assertEqual(mirror["arches"], ["i386", "amd64", "blah"]) mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertTrue(len(failures) == 0) + self.assertFalse(failures) self.assertTrue(os.path.exists("/etc/blah.ini")) self.assertIn("write_files", which_ran) contents = util.load_text_file("/etc/blah.ini") diff --git a/tests/unittests/runs/test_simple_run.py b/tests/unittests/runs/test_simple_run.py index eec2db00..e8737ea2 100644 --- a/tests/unittests/runs/test_simple_run.py +++ b/tests/unittests/runs/test_simple_run.py @@ -97,7 +97,7 @@ def test_none_ds_runs_modules_which_do_not_define_distros(self): mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertTrue(len(failures) == 0) + self.assertFalse(failures) self.assertTrue(os.path.exists("/etc/blah.ini")) self.assertIn("write_files", which_ran) contents = util.load_text_file("/etc/blah.ini") @@ -125,7 +125,7 @@ def test_none_ds_skips_modules_which_define_unmatched_distros(self): mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertTrue(len(failures) == 0) + self.assertFalse(failures) self.assertIn( "Skipping modules 'spacewalk' because they are not verified on" " distro 'ubuntu'", @@ -154,7 +154,7 @@ def test_none_ds_runs_modules_which_distros_all(self): mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertTrue(len(failures) == 0) + self.assertFalse(failures) self.assertIn("runcmd", which_ran) self.assertNotIn( "Skipping modules 'runcmd' because they are not verified on" @@ -189,7 +189,7 @@ def test_none_ds_forces_run_via_unverified_modules(self): mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertTrue(len(failures) == 0) + self.assertFalse(failures) self.assertIn("spacewalk", which_ran) self.assertIn( "running unverified_modules: 'spacewalk'", self.logs.getvalue() @@ -223,5 +223,5 @@ def test_none_ds_run_with_no_config_modules(self): mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertTrue(len(failures) == 0) + self.assertFalse(failures) self.assertEqual([], which_ran) diff --git a/tests/unittests/sources/helpers/test_openstack.py b/tests/unittests/sources/helpers/test_openstack.py index 6ec0bd75..519392b0 100644 --- a/tests/unittests/sources/helpers/test_openstack.py +++ b/tests/unittests/sources/helpers/test_openstack.py @@ -219,6 +219,7 @@ def test_bond_mac(self): "type": "vlan", "vlan_id": 123, "vlan_link": "bond0", + "mac_address": "xx:xx:xx:xx:xx:00", }, {"address": "1.1.1.1", "type": "nameserver"}, ], @@ -346,6 +347,7 @@ def test_dns_servers(self): ], "vlan_id": 123, "vlan_link": "bond0", + "mac_address": "xx:xx:xx:xx:xx:00", }, ], } diff --git a/tests/unittests/sources/test_aliyun.py b/tests/unittests/sources/test_aliyun.py index 2639302b..2d61ff8a 100644 --- a/tests/unittests/sources/test_aliyun.py +++ b/tests/unittests/sources/test_aliyun.py @@ -9,46 +9,93 @@ from cloudinit import helpers from cloudinit.sources import DataSourceAliYun as ay -from cloudinit.sources.DataSourceEc2 import convert_ec2_metadata_network_config +from cloudinit.sources.helpers.aliyun import ( + convert_ecs_metadata_network_config, +) +from cloudinit.util import load_json from tests.unittests import helpers as test_helpers -DEFAULT_METADATA = { - "instance-id": "aliyun-test-vm-00", - "eipv4": "10.0.0.1", - "hostname": "test-hostname", - "image-id": "m-test", - "launch-index": "0", - "mac": "00:16:3e:00:00:00", - "network-type": "vpc", - "private-ipv4": "192.168.0.1", - "serial-number": "test-string", - "vpc-cidr-block": "192.168.0.0/16", - "vpc-id": "test-vpc", - "vswitch-id": "test-vpc", - "vswitch-cidr-block": "192.168.0.0/16", - "zone-id": "test-zone-1", - "ntp-conf": { - "ntp_servers": [ - "ntp1.aliyun.com", - "ntp2.aliyun.com", - "ntp3.aliyun.com", - ] - }, - "source-address": [ - "http://mirrors.aliyun.com", - "http://mirrors.aliyuncs.com", - ], - "public-keys": { - "key-pair-1": {"openssh-key": "ssh-rsa AAAAB3..."}, - "key-pair-2": {"openssh-key": "ssh-rsa AAAAB3..."}, +DEFAULT_METADATA_RAW = r"""{ + "disks": { + "bp15spwwhlf8bbbn7xxx": { + "id": "d-bp15spwwhlf8bbbn7xxx", + "name": "" + } + }, + "dns-conf": { + "nameservers": [ + "100.100.2.136", + "100.100.2.138" + ] + }, + "hibernation": { + "configured": "false" + }, + "instance": { + "instance-name": "aliyun-test-vm-00", + "instance-type": "ecs.g8i.large", + "last-host-landing-time": "2024-11-17 10:02:41", + "max-netbw-egress": "2560000", + "max-netbw-ingress": "2560000", + "virtualization-solution": "ECS Virt", + "virtualization-solution-version": "2.0" + }, + "network": { + "interfaces": { + "macs": { + "00:16:3e:14:59:58": { + "gateway": "172.16.101.253", + "netmask": "255.255.255.0", + "network-interface-id": "eni-bp13i3ed90icgdgaxxxx" + } + } + } + }, + "ntp-conf": { + "ntp-servers": [ + "ntp1.aliyun.com", + "ntp1.cloud.aliyuncs.com" + ] + }, + "public-keys": { + "0": { + "openssh-key": "ssh-rsa AAAAB3Nza" }, -} + "skp-bp1test": { + "openssh-key": "ssh-rsa AAAAB3Nza" + } + }, + "eipv4": "121.66.77.88", + "hostname": "aliyun-test-vm-00", + "image-id": "ubuntu_24_04_x64_20G_alibase_20241016.vhd", + "instance-id": "i-bp15ojxppkmsnyjxxxxx", + "mac": "00:16:3e:14:59:58", + "network-type": "vpc", + "owner-account-id": "123456", + "private-ipv4": "172.16.111.222", + "region-id": "cn-hangzhou", + "serial-number": "3ca05955-a892-46b3-a6fc-xxxxxx", + "source-address": "http://mirrors.cloud.aliyuncs.com", + "sub-private-ipv4-list": "172.16.101.215", + "vpc-cidr-block": "172.16.0.0/12", + "vpc-id": "vpc-bp1uwvjta7txxxxxxx", + "vswitch-cidr-block": "172.16.101.0/24", + "vswitch-id": "vsw-bp12cibmw6078qv123456", + "zone-id": "cn-hangzhou-j" +}""" + +DEFAULT_METADATA = load_json(DEFAULT_METADATA_RAW) DEFAULT_USERDATA = """\ #cloud-config hostname: localhost""" +DEFAULT_VENDORDATA = """\ +#cloud-config +bootcmd: +- echo hello world > /tmp/vendor""" + class TestAliYunDatasource(test_helpers.ResponsesTestCase): def setUp(self): @@ -67,6 +114,10 @@ def default_metadata(self): def default_userdata(self): return DEFAULT_USERDATA + @property + def default_vendordata(self): + return DEFAULT_VENDORDATA + @property def metadata_url(self): return ( @@ -78,12 +129,29 @@ def metadata_url(self): + "/" ) + @property + def metadata_all_url(self): + return ( + os.path.join( + self.metadata_address, + self.ds.min_metadata_version, + "meta-data", + ) + + "/all" + ) + @property def userdata_url(self): return os.path.join( self.metadata_address, self.ds.min_metadata_version, "user-data" ) + @property + def vendordata_url(self): + return os.path.join( + self.metadata_address, self.ds.min_metadata_version, "vendor-data" + ) + # EC2 provides an instance-identity document which must return 404 here # for this test to pass. @property @@ -133,9 +201,17 @@ def register_helper(register, base_url, body): register = functools.partial(self.responses.add, responses.GET) register_helper(register, base_url, data) - def regist_default_server(self): + def regist_default_server(self, register_json_meta_path=True): self.register_mock_metaserver(self.metadata_url, self.default_metadata) + if register_json_meta_path: + self.register_mock_metaserver( + self.metadata_all_url, DEFAULT_METADATA_RAW + ) self.register_mock_metaserver(self.userdata_url, self.default_userdata) + self.register_mock_metaserver( + self.vendordata_url, self.default_userdata + ) + self.register_mock_metaserver(self.identity_url, self.default_identity) self.responses.add(responses.PUT, self.token_url, "API-TOKEN") @@ -175,7 +251,25 @@ def test_with_mock_server(self, m_is_aliyun, m_resolv): self._test_get_iid() self._test_host_name() self.assertEqual("aliyun", self.ds.cloud_name) - self.assertEqual("ec2", self.ds.platform) + self.assertEqual("aliyun", self.ds.platform) + self.assertEqual( + "metadata (http://100.100.100.200)", self.ds.subplatform + ) + + @mock.patch("cloudinit.sources.DataSourceEc2.util.is_resolvable") + @mock.patch("cloudinit.sources.DataSourceAliYun._is_aliyun") + def test_with_mock_server_without_json_path(self, m_is_aliyun, m_resolv): + m_is_aliyun.return_value = True + self.regist_default_server(register_json_meta_path=False) + ret = self.ds.get_data() + self.assertEqual(True, ret) + self.assertEqual(1, m_is_aliyun.call_count) + self._test_get_data() + self._test_get_sshkey() + self._test_get_iid() + self._test_host_name() + self.assertEqual("aliyun", self.ds.cloud_name) + self.assertEqual("aliyun", self.ds.platform) self.assertEqual( "metadata (http://100.100.100.200)", self.ds.subplatform ) @@ -221,7 +315,7 @@ def test_aliyun_local_with_mock_server( self._test_get_iid() self._test_host_name() self.assertEqual("aliyun", self.ds.cloud_name) - self.assertEqual("ec2", self.ds.platform) + self.assertEqual("aliyun", self.ds.platform) self.assertEqual( "metadata (http://100.100.100.200)", self.ds.subplatform ) @@ -272,31 +366,28 @@ def test_parse_public_keys(self): public_keys["key-pair-0"]["openssh-key"], ) - def test_route_metric_calculated_without_device_number(self): - """Test that route-metric code works without `device-number` - - `device-number` is part of EC2 metadata, but not supported on aliyun. - Attempting to access it will raise a KeyError. - - LP: #1917875 - """ - netcfg = convert_ec2_metadata_network_config( + def test_route_metric_calculated_with_multiple_network_cards(self): + """Test that route-metric code works with multiple network cards""" + netcfg = convert_ecs_metadata_network_config( { "interfaces": { "macs": { - "06:17:04:d7:26:09": { - "interface-id": "eni-e44ef49e", + "00:16:3e:14:59:58": { + "ipv6-gateway": "2408:xxxxx", + "ipv6s": "[2408:xxxxxx]", + "network-interface-id": "eni-bp13i1xxxxx", }, - "06:17:04:d7:26:08": { - "interface-id": "eni-e44ef49f", + "00:16:3e:39:43:27": { + "gateway": "172.16.101.253", + "netmask": "255.255.255.0", + "network-interface-id": "eni-bp13i2xxxx", }, } } }, - mock.Mock(), macs_to_nics={ - "06:17:04:d7:26:09": "eth0", - "06:17:04:d7:26:08": "eth1", + "00:16:3e:14:59:58": "eth0", + "00:16:3e:39:43:27": "eth1", }, ) @@ -314,6 +405,28 @@ def test_route_metric_calculated_without_device_number(self): netcfg["ethernets"]["eth1"].keys() ) + # eth0 network meta-data have ipv6s info, ipv6 should True + met0_dhcp6 = netcfg["ethernets"]["eth0"]["dhcp6"] + assert met0_dhcp6 is True + + netcfg = convert_ecs_metadata_network_config( + { + "interfaces": { + "macs": { + "00:16:3e:14:59:58": { + "gateway": "172.16.101.253", + "netmask": "255.255.255.0", + "network-interface-id": "eni-bp13ixxxx", + } + } + } + }, + macs_to_nics={"00:16:3e:14:59:58": "eth0"}, + ) + met0 = netcfg["ethernets"]["eth0"] + # single network card would have no dhcp4-overrides + assert "dhcp4-overrides" not in met0 + class TestIsAliYun(test_helpers.CiTestCase): ALIYUN_PRODUCT = "Alibaba Cloud ECS" diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py index 6d6f801e..5e16e75f 100644 --- a/tests/unittests/sources/test_azure.py +++ b/tests/unittests/sources/test_azure.py @@ -2266,7 +2266,7 @@ def test_username_from_imds(self): dsrc.cfg["system_info"]["default_user"]["name"], "username1" ) - def test_disable_password_from_imds(self): + def test_disable_password_from_imds_true(self): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { "ovfcontent": construct_ovf_env(), @@ -2281,7 +2281,25 @@ def test_disable_password_from_imds(self): self.m_fetch.return_value = imds_data_with_os_profile dsrc = self._get_ds(data) dsrc.get_data() - self.assertTrue(dsrc.metadata["disable_password"]) + self.assertFalse(dsrc.cfg["ssh_pwauth"]) + + def test_disable_password_from_imds_false(self): + sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} + data = { + "ovfcontent": construct_ovf_env(), + "sys_cfg": sys_cfg, + "write_ovf_to_seed_dir": False, + } + imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA) + imds_data_with_os_profile["compute"]["osProfile"] = dict( + adminUsername="username1", + computerName="hostname1", + disablePasswordAuthentication="false", + ) + self.m_fetch.return_value = imds_data_with_os_profile + dsrc = self._get_ds(data) + dsrc.get_data() + self.assertTrue(dsrc.cfg["ssh_pwauth"]) def test_userdata_from_imds(self): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} @@ -3398,6 +3416,27 @@ def test_basic_setup_without_wireserver_opt( assert azure_ds._wireserver_endpoint == "168.63.129.16" assert azure_ds._ephemeral_dhcp_ctx.iface == lease["interface"] + def test_retry_missing_driver( + self, azure_ds, caplog, mock_ephemeral_dhcp_v4, mock_sleep + ): + lease = { + "interface": "fakeEth0", + } + mock_ephemeral_dhcp_v4.return_value.obtain_lease.side_effect = [ + FileNotFoundError, + FileNotFoundError, + lease, + ] + + azure_ds._setup_ephemeral_networking() + + assert mock_ephemeral_dhcp_v4.return_value.mock_calls == [ + mock.call.obtain_lease(), + mock.call.obtain_lease(), + mock.call.obtain_lease(), + ] + assert "File not found during DHCP" in caplog.text + def test_no_retry_missing_dhclient_error( self, azure_ds, @@ -3835,8 +3874,8 @@ def test_no_pps(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 - assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 + assert not self.mock_kvp_report_failure_to_host.mock_calls + assert not self.mock_azure_report_failure_to_fabric.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 # Verify dmesg reported via KVP. @@ -3920,8 +3959,8 @@ def test_no_pps_gpa(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 - assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 + assert not self.mock_kvp_report_failure_to_host.mock_calls + assert not self.mock_azure_report_failure_to_fabric.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 # Verify dmesg reported via KVP. @@ -4006,7 +4045,7 @@ def test_no_pps_gpa_fail(self): # Verify reports via KVP. assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 1 assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 1 - assert len(self.mock_kvp_report_success_to_host.mock_calls) == 0 + assert not self.mock_kvp_report_success_to_host.mock_calls # Verify dmesg reported via KVP. assert len(self.mock_report_dmesg_to_kvp.mock_calls) == 1 @@ -4189,7 +4228,7 @@ def test_running_pps(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 + assert not self.mock_kvp_report_failure_to_host.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 2 # Verify dmesg reported via KVP. @@ -4316,7 +4355,7 @@ def test_running_pps_gpa(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 + assert not self.mock_kvp_report_failure_to_host.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 2 # Verify dmesg reported via KVP. @@ -4445,7 +4484,7 @@ def test_savable_pps(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 + assert not self.mock_kvp_report_failure_to_host.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 2 # Verify dmesg reported via KVP. @@ -4581,7 +4620,7 @@ def test_savable_pps_gpa(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 + assert not self.mock_kvp_report_failure_to_host.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 2 # Verify dmesg reported via KVP. @@ -4822,7 +4861,7 @@ def test_recovery_pps(self, pps_type): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 + assert not self.mock_kvp_report_failure_to_host.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 @pytest.mark.parametrize("pps_type", ["Savable", "Running", "Unknown"]) @@ -4855,7 +4894,7 @@ def test_source_pps_fails_initial_dhcp(self, pps_type): # Verify reports via KVP. assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 2 - assert len(self.mock_kvp_report_success_to_host.mock_calls) == 0 + assert not self.mock_kvp_report_success_to_host.mock_calls @pytest.mark.parametrize( "subp_side_effect", @@ -4971,7 +5010,7 @@ def test_imds_failure_results_in_provisioning_failure(self): # Verify reports via KVP. assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 1 - assert len(self.mock_kvp_report_success_to_host.mock_calls) == 0 + assert not self.mock_kvp_report_success_to_host.mock_calls class TestCheckAzureProxyAgent: diff --git a/tests/unittests/sources/test_configdrive.py b/tests/unittests/sources/test_configdrive.py index 70da4812..6e97b992 100644 --- a/tests/unittests/sources/test_configdrive.py +++ b/tests/unittests/sources/test_configdrive.py @@ -597,7 +597,7 @@ def test_find_candidates(self): devs_with_answers = {} def my_devs_with(*args, **kwargs): - criteria = args[0] if len(args) else kwargs.pop("criteria", None) + criteria = args[0] if args else kwargs.pop("criteria", None) return devs_with_answers.get(criteria, []) def my_is_partition(dev): @@ -896,12 +896,15 @@ def test_convert_reads_system_prefers_name(self, get_interfaces_by_mac): def test_convert_raises_value_error_on_missing_name(self): macs = {"aa:aa:aa:aa:aa:00": "ens1"} - self.assertRaises( - ValueError, - openstack.convert_net_json, - NETWORK_DATA, - known_macs=macs, - ) + with mock.patch( + "cloudinit.sources.helpers.openstack.util.udevadm_settle" + ): + self.assertRaises( + ValueError, + openstack.convert_net_json, + NETWORK_DATA, + known_macs=macs, + ) def test_conversion_with_route(self): ncfg = openstack.convert_net_json( diff --git a/tests/unittests/sources/test_ec2.py b/tests/unittests/sources/test_ec2.py index 7d2eab15..2d7ffb52 100644 --- a/tests/unittests/sources/test_ec2.py +++ b/tests/unittests/sources/test_ec2.py @@ -1736,16 +1736,6 @@ def test_identify_aws_endian(self, m_collect): ) assert ec2.CloudNames.AWS == ec2.identify_platform() - @mock.patch("cloudinit.sources.DataSourceEc2._collect_platform_data") - def test_identify_aliyun(self, m_collect): - """aliyun should be identified if product name equals to - Alibaba Cloud ECS - """ - m_collect.return_value = self.collmock( - product_name="Alibaba Cloud ECS" - ) - assert ec2.CloudNames.ALIYUN == ec2.identify_platform() - @mock.patch("cloudinit.sources.DataSourceEc2._collect_platform_data") def test_identify_zstack(self, m_collect): """zstack should be identified if chassis-asset-tag diff --git a/tests/unittests/sources/test_gce.py b/tests/unittests/sources/test_gce.py index dec79b53..350ebd12 100644 --- a/tests/unittests/sources/test_gce.py +++ b/tests/unittests/sources/test_gce.py @@ -488,3 +488,42 @@ def test_datasource_doesnt_use_ephemeral_dhcp(self, m_dhcp): ds = DataSourceGCE.DataSourceGCE(sys_cfg={}, distro=None, paths=None) ds._get_data() assert m_dhcp.call_count == 0 + + @mock.patch( + M_PATH + "EphemeralDHCPv4", + autospec=True, + ) + @mock.patch(M_PATH + "net.find_candidate_nics") + def test_datasource_on_dhcp_lease_failure( + self, m_find_candidate_nics, m_dhcp + ): + self._set_mock_metadata() + distro = mock.MagicMock() + distro.get_tmp_exec_path = self.tmp_dir + ds = DataSourceGCE.DataSourceGCELocal( + sys_cfg={}, distro=distro, paths=None + ) + m_find_candidate_nics.return_value = [ + "ens0p4", + "ens0p5", + ] + m_dhcp.return_value.__enter__.side_effect = ( + NoDHCPLeaseError, + NoDHCPLeaseError, + ) + assert ds._get_data() is False + assert m_dhcp.call_args_list == [ + mock.call(distro, iface="ens0p4"), + mock.call(distro, iface="ens0p5"), + ] + + expected_logs = ( + "Looking for the primary NIC in: ['ens0p4', 'ens0p5']", + "Unable to obtain a DHCP lease for ens0p4", + "Unable to obtain a DHCP lease for ens0p5", + ) + for msg in expected_logs: + self.assertIn( + msg, + self.logs.getvalue(), + ) diff --git a/tests/unittests/sources/test_lxd.py b/tests/unittests/sources/test_lxd.py index b7123456..9d3ae417 100644 --- a/tests/unittests/sources/test_lxd.py +++ b/tests/unittests/sources/test_lxd.py @@ -101,12 +101,12 @@ def lxd_ds(request, paths): Return an instantiated DataSourceLXD. This also performs the mocking required for the default test case: - * ``is_platform_viable`` returns True, + * ``ds_detect`` returns True, * ``read_metadata`` returns ``LXD_V1_METADATA`` (This uses the paths fixture for the required helpers.Paths object) """ - with mock.patch(DS_PATH + "is_platform_viable", return_value=True): + with mock.patch(DS_PATH + "DataSourceLXD.ds_detect", return_value=True): with mock.patch( DS_PATH + "read_metadata", return_value=lxd_metadata() ): @@ -121,12 +121,12 @@ def lxd_ds_no_network_config(request, paths): Return an instantiated DataSourceLXD. This also performs the mocking required for the default test case: - * ``is_platform_viable`` returns True, + * ``ds_detect`` returns True, * ``read_metadata`` returns ``LXD_V1_METADATA_NO_NETWORK_CONFIG`` (This uses the paths fixture for the required helpers.Paths object) """ - with mock.patch(DS_PATH + "is_platform_viable", return_value=True): + with mock.patch(DS_PATH + "DataSourceLXD.ds_detect", return_value=True): with mock.patch( DS_PATH + "read_metadata", return_value=lxd_metadata_no_network_config(), @@ -369,7 +369,7 @@ def test_expected_viable( """Return True only when LXD_SOCKET_PATH exists and is a socket.""" m_exists.return_value = exists m_lstat.return_value = LStatResponse(lstat_mode) - assert expected is lxd.is_platform_viable() + assert expected is lxd.DataSourceLXD.ds_detect() m_exists.assert_has_calls([mock.call(lxd.LXD_SOCKET_PATH)]) if exists: m_lstat.assert_has_calls([mock.call(lxd.LXD_SOCKET_PATH)]) diff --git a/tests/unittests/sources/test_maas.py b/tests/unittests/sources/test_maas.py index d6e1658e..5d31c916 100644 --- a/tests/unittests/sources/test_maas.py +++ b/tests/unittests/sources/test_maas.py @@ -4,11 +4,13 @@ from unittest import mock import pytest +import responses import yaml from cloudinit import helpers, settings, url_helper from cloudinit.sources import DataSourceMAAS -from tests.unittests.helpers import populate_dir +from tests.unittests.helpers import get_mock_paths, populate_dir +from tests.unittests.util import MockDistro class TestMAASDataSource: @@ -105,7 +107,7 @@ def mock_read_maas_seed_url(self, data, seed, version="19991231"): return what read_maas_seed_url returns.""" def my_readurl(*args, **kwargs): - if len(args): + if args: url = args[0] else: url = kwargs["url"] @@ -125,7 +127,7 @@ def my_readurl(*args, **kwargs): def test_seed_url_valid(self, tmpdir): """Verify that valid seed_url is read as such.""" - valid = { + valid: dict = { "meta-data/instance-id": "i-instanceid", "meta-data/local-hostname": "test-hostname", "meta-data/public-keys": "test-hostname", @@ -221,6 +223,56 @@ def tests_wb_local_stage_detects_datasource_on_initramfs_network( klibc_net_cfg.write(initramfs_file) assert expected == ds.get_data() + @responses.activate + def test_get_data_with_retry(self, mocker, tmp_path, caplog): + """Ensure we can get data from IMDS even if some attempts fail.""" + mocker.patch("time.sleep") + metadata_url = "http://169.254.169.254/MAAS/metadata" + response_data = { + "instance-id": "i-123", + "local-hostname": "myhostname", + "public-keys": "ssh-rsa AAAAB...yc2E= keyname", + "vendor-data": "my-vendordata", + } + + responses.add( + responses.GET, + url=f"{metadata_url}/2012-03-01/meta-data/instance-id", + status=404, + ) + + for key, value in response_data.items(): + responses.add( + responses.GET, + f"{metadata_url}/2012-03-01/meta-data/{key}", + value.encode(), + ) + + responses.add( + responses.GET, + url=f"{metadata_url}/2012-03-01/user-data", + status=404, + ) + responses.add( + responses.GET, + f"{metadata_url}/2012-03-01/user-data", + b"my-userdata", + ) + + cfg = {"datasource": {"MAAS": {"metadata_url": metadata_url}}} + ds = DataSourceMAAS.DataSourceMAAS( + cfg, MockDistro(), get_mock_paths(tmp_path)({}) + ) + assert ds.get_data() + assert ds.metadata["instance-id"] == "i-123" + assert ds.metadata["local-hostname"] == "myhostname" + assert ds.metadata["public-keys"] == "ssh-rsa AAAAB...yc2E= keyname" + assert ds.vendordata_raw == "my-vendordata" + assert ds.userdata_raw == b"my-userdata" + assert ( + "Please wait 1 seconds while we wait to try again" in caplog.text + ) + @mock.patch("cloudinit.sources.DataSourceMAAS.url_helper.OauthUrlHelper") class TestGetOauthHelper: diff --git a/tests/unittests/sources/test_openstack.py b/tests/unittests/sources/test_openstack.py index 154a7620..669148d8 100644 --- a/tests/unittests/sources/test_openstack.py +++ b/tests/unittests/sources/test_openstack.py @@ -92,7 +92,7 @@ def _register_uris(version, ec2_files, ec2_meta, os_files, *, responses_mock): def match_ec2_url(uri, headers): path = uri.path.strip("/") - if len(path) == 0: + if not path: return (200, headers, "\n".join(EC2_VERSIONS)) path = uri.path.lstrip("/") if path in ec2_files: @@ -687,6 +687,27 @@ def fake_asset_tag_dmi_read(dmi_key): "Expected ds_detect == True on Huawei Cloud VM", ) + @test_helpers.mock.patch(MOCK_PATH + "dmi.read_dmi_data") + def test_ds_detect_samsung_cloud_platform_chassis_asset_tag( + self, m_dmi, m_is_x86 + ): + """Return True on OpenStack reporting + Samsung Cloud Platform VM asset-tag.""" + m_is_x86.return_value = True + + def fake_asset_tag_dmi_read(dmi_key): + if dmi_key == "system-product-name": + return "c7.large.2" # No match + if dmi_key == "chassis-asset-tag": + return "Samsung Cloud Platform" + assert False, "Unexpected dmi read of %s" % dmi_key + + m_dmi.side_effect = fake_asset_tag_dmi_read + self.assertTrue( + self._fake_ds().ds_detect(), + "Expected ds_detect == True on Samsung Cloud Platform VM", + ) + @test_helpers.mock.patch(MOCK_PATH + "dmi.read_dmi_data") def test_ds_detect_oraclecloud_chassis_asset_tag(self, m_dmi, m_is_x86): """Return True on OpenStack reporting Oracle cloud asset-tag.""" diff --git a/tests/unittests/sources/test_ovf.py b/tests/unittests/sources/test_ovf.py index f543407f..b3bc84a8 100644 --- a/tests/unittests/sources/test_ovf.py +++ b/tests/unittests/sources/test_ovf.py @@ -537,6 +537,11 @@ def test_vmware_rpctool_fails_and_vmtoolsd_fails(self, m_subp, m_which): ] self.assertEqual(NOT_FOUND, dsovf.transport_vmware_guestinfo()) self.assertEqual(2, m_subp.call_count) + self.assertNotIn( + "WARNING", + self.logs.getvalue(), + "exit code of 1 by rpctool and vmtoolsd should not cause warning.", + ) def test_vmware_rpctool_fails_and_vmtoolsd_success(self, m_subp, m_which): """When vmware-rpctool fails but vmtoolsd succeeds""" diff --git a/tests/unittests/sources/test_wsl.py b/tests/unittests/sources/test_wsl.py index 2012cd90..ebe01827 100644 --- a/tests/unittests/sources/test_wsl.py +++ b/tests/unittests/sources/test_wsl.py @@ -402,7 +402,7 @@ def test_get_data_cc(self, m_lsb_release, paths, tmpdir): cast(MIMEMultipart, ud), "text/cloud-config" ) assert userdata is not None - assert "wsl.conf" in cast(str, userdata) + assert "wsl.conf" in userdata @mock.patch("cloudinit.util.lsb_release") def test_get_data_sh(self, m_lsb_release, tmpdir, paths): @@ -421,11 +421,8 @@ def test_get_data_sh(self, m_lsb_release, tmpdir, paths): ud = ds.get_userdata() assert ud is not None - userdata = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/x-shellscript" - ), + userdata = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/x-shellscript" ) assert COMMAND in userdata @@ -539,19 +536,13 @@ def test_data_precedence(self, m_get_linux_dist, tmpdir, paths): ud = ds.get_userdata() assert ud is not None - userdata = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/cloud-config" - ), + userdata = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/cloud-config" ) assert "wsl.conf" in userdata assert "packages" not in userdata - shell_script = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/x-shellscript" - ), + shell_script = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/x-shellscript" ) assert "" == shell_script @@ -597,11 +588,8 @@ def test_interaction_with_pro(self, m_get_linux_dist, tmpdir, paths): ud = ds.get_userdata() assert ud is not None - userdata = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/cloud-config" - ), + userdata = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/cloud-config" ) assert "wsl.conf" in userdata assert "packages" not in userdata @@ -639,11 +627,8 @@ def test_landscape_vs_local_user(self, m_get_linux_dist, tmpdir, paths): assert ds.get_data() is True ud = ds.get_userdata() assert ud is not None - userdata = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/cloud-config" - ), + userdata = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/cloud-config" ) assert ( @@ -700,11 +685,8 @@ def test_landscape_provided_data(self, m_get_linux_dist, tmpdir, paths): ud = ds.get_userdata() assert ud is not None - userdata = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/cloud-config" - ), + userdata = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/cloud-config" ) assert "ubuntu_pro" in userdata, "Agent data should be present" @@ -761,11 +743,8 @@ def test_landscape_empty_data(self, m_get_linux_dist, tmpdir, paths): ud = ds.get_userdata() assert ud is not None - userdata = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/cloud-config" - ), + userdata = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/cloud-config" ) assert ( @@ -813,22 +792,16 @@ def test_landscape_shell_script(self, m_get_linux_dist, tmpdir, paths): ud = ds.get_userdata() assert ud is not None - userdata = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/cloud-config" - ), + userdata = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/cloud-config" ) assert ( "agent_test" in userdata and "agent_token" in userdata ), "Agent data should be present" - shell_script = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/x-shellscript" - ), + shell_script = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/x-shellscript" ) assert COMMAND in shell_script @@ -877,11 +850,8 @@ def test_with_landscape_no_tags(self, m_get_linux_dist, tmpdir, paths): ud = ds.get_userdata() assert ud is not None - userdata = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/cloud-config" - ), + userdata = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/cloud-config" ) assert ( @@ -929,11 +899,8 @@ def test_with_no_tags_at_all(self, m_get_linux_dist, tmpdir, paths): ud = ds.get_userdata() assert ud is not None - userdata = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/cloud-config" - ), + userdata = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/cloud-config" ) assert "landscapetest" in userdata assert "up4w_token" in userdata @@ -980,11 +947,8 @@ def test_with_no_client_subkey(self, m_get_linux_dist, tmpdir, paths): ud = ds.get_userdata() assert ud is not None - userdata = cast( - str, - join_payloads_from_content_type( - cast(MIMEMultipart, ud), "text/cloud-config" - ), + userdata = join_payloads_from_content_type( + cast(MIMEMultipart, ud), "text/cloud-config" ) assert "landscapetest" not in userdata assert ( diff --git a/tests/unittests/sources/vmware/test_vmware_config_file.py b/tests/unittests/sources/vmware/test_vmware_config_file.py index fd4bb481..837efa21 100644 --- a/tests/unittests/sources/vmware/test_vmware_config_file.py +++ b/tests/unittests/sources/vmware/test_vmware_config_file.py @@ -1,8 +1,10 @@ # Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2016-2022 VMware INC. +# Copyright (C) 2006-2024 Broadcom. All Rights Reserved. +# Broadcom Confidential. The term "Broadcom" refers to Broadcom Inc. +# and/or its subsidiaries. # # Author: Sankar Tanguturi -# Pengpeng Sun +# Pengpeng Sun # # This file is part of cloud-init. See LICENSE file for license information. @@ -17,10 +19,7 @@ from cloudinit.sources.helpers.vmware.imc.config_file import ( ConfigFile as WrappedConfigFile, ) -from cloudinit.sources.helpers.vmware.imc.config_nic import ( - NicConfigurator, - gen_subnet, -) +from cloudinit.sources.helpers.vmware.imc.config_nic import NicConfigurator from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( get_network_data_from_vmware_cust_cfg, get_non_network_data_from_vmware_cust_cfg, @@ -164,36 +163,21 @@ def test_get_config_nameservers(self): network_config = get_network_data_from_vmware_cust_cfg(config, False) - self.assertEqual(1, network_config.get("version")) - - config_types = network_config.get("config") - name_servers = None - dns_suffixes = None - - for type in config_types: - if type.get("type") == "nameserver": - name_servers = type.get("address") - dns_suffixes = type.get("search") - break + self.assertEqual(2, network_config.get("version")) - self.assertEqual(["10.20.145.1", "10.20.145.2"], name_servers, "dns") - self.assertEqual( - ["eng.vmware.com", "proxy.vmware.com"], dns_suffixes, "suffixes" - ) + ethernets = network_config.get("ethernets") - def test_gen_subnet(self): - """Tests if gen_subnet properly calculates network subnet from - IPv4 address and netmask""" - ip_subnet_list = [ - ["10.20.87.253", "255.255.252.0", "10.20.84.0"], - ["10.20.92.105", "255.255.252.0", "10.20.92.0"], - ["192.168.0.10", "255.255.0.0", "192.168.0.0"], - ] - for entry in ip_subnet_list: + for _, config in ethernets.items(): + self.assertTrue(isinstance(config, dict)) + name_servers = config.get("nameservers").get("addresses") + dns_suffixes = config.get("nameservers").get("search") self.assertEqual( - entry[2], - gen_subnet(entry[0], entry[1]), - "Subnet for a specified ip and netmask", + ["10.20.145.1", "10.20.145.2"], name_servers, "dns" + ) + self.assertEqual( + ["eng.vmware.com", "proxy.vmware.com"], + dns_suffixes, + "suffixes", ) def test_get_config_dns_suffixes(self): @@ -206,162 +190,141 @@ def test_get_config_dns_suffixes(self): network_config = get_network_data_from_vmware_cust_cfg(config, False) - self.assertEqual(1, network_config.get("version")) - - config_types = network_config.get("config") - name_servers = None - dns_suffixes = None + self.assertEqual(2, network_config.get("version")) - for type in config_types: - if type.get("type") == "nameserver": - name_servers = type.get("address") - dns_suffixes = type.get("search") - break + ethernets = network_config.get("ethernets") - self.assertEqual([], name_servers, "dns") - self.assertEqual(["eng.vmware.com"], dns_suffixes, "suffixes") + for _, config in ethernets.items(): + self.assertTrue(isinstance(config, dict)) + name_servers = config.get("nameservers").get("addresses") + dns_suffixes = config.get("nameservers").get("search") + self.assertEqual(None, name_servers, "dns") + self.assertEqual(["eng.vmware.com"], dns_suffixes, "suffixes") def test_get_nics_list_dhcp(self): - """Tests if NicConfigurator properly calculates network subnets + """Tests if NicConfigurator properly calculates ethernets for a configuration with a list of DHCP NICs""" cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") config = Config(cf) - nicConfigurator = NicConfigurator(config.nics, False) - nics_cfg_list = nicConfigurator.generate() - - self.assertEqual(2, len(nics_cfg_list), "number of config elements") - - nic1 = {"name": "NIC1"} - nic2 = {"name": "NIC2"} - for cfg in nics_cfg_list: - if cfg.get("name") == nic1.get("name"): - nic1.update(cfg) - elif cfg.get("name") == nic2.get("name"): - nic2.update(cfg) - - self.assertEqual("physical", nic1.get("type"), "type of NIC1") - self.assertEqual("NIC1", nic1.get("name"), "name of NIC1") - self.assertEqual( - "00:50:56:a6:8c:08", nic1.get("mac_address"), "mac address of NIC1" + nicConfigurator = NicConfigurator( + config.nics, config.name_servers, config.dns_suffixes, False ) - subnets = nic1.get("subnets") - self.assertEqual(1, len(subnets), "number of subnets for NIC1") - subnet = subnets[0] - self.assertEqual("dhcp", subnet.get("type"), "DHCP type for NIC1") - self.assertEqual("auto", subnet.get("control"), "NIC1 Control type") - - self.assertEqual("physical", nic2.get("type"), "type of NIC2") - self.assertEqual("NIC2", nic2.get("name"), "name of NIC2") - self.assertEqual( - "00:50:56:a6:5a:de", nic2.get("mac_address"), "mac address of NIC2" - ) - subnets = nic2.get("subnets") - self.assertEqual(1, len(subnets), "number of subnets for NIC2") - subnet = subnets[0] - self.assertEqual("dhcp", subnet.get("type"), "DHCP type for NIC2") - self.assertEqual("auto", subnet.get("control"), "NIC2 Control type") + ethernets_dict = nicConfigurator.generate() + + self.assertTrue(isinstance(ethernets_dict, dict)) + self.assertEqual(2, len(ethernets_dict), "number of ethernets") + + for name, config in ethernets_dict.items(): + if name == "NIC1": + self.assertEqual( + "00:50:56:a6:8c:08", + config.get("match").get("macaddress"), + "mac address of NIC1", + ) + self.assertEqual( + True, config.get("wakeonlan"), "wakeonlan of NIC1" + ) + self.assertEqual( + True, config.get("dhcp4"), "DHCPv4 enablement of NIC1" + ) + self.assertEqual( + False, + config.get("dhcp4-overrides").get("use-dns"), + "use-dns enablement for dhcp4-overrides of NIC1", + ) + if name == "NIC2": + self.assertEqual( + "00:50:56:a6:5a:de", + config.get("match").get("macaddress"), + "mac address of NIC2", + ) + self.assertEqual( + True, config.get("wakeonlan"), "wakeonlan of NIC2" + ) + self.assertEqual( + True, config.get("dhcp4"), "DHCPv4 enablement of NIC2" + ) + self.assertEqual( + False, + config.get("dhcp4-overrides").get("use-dns"), + "use-dns enablement for dhcp4-overrides of NIC2", + ) def test_get_nics_list_static(self): - """Tests if NicConfigurator properly calculates network subnets + """Tests if NicConfigurator properly calculates ethernets for a configuration with 2 static NICs""" cf = ConfigFile("tests/data/vmware/cust-static-2nic.cfg") config = Config(cf) - nicConfigurator = NicConfigurator(config.nics, False) - nics_cfg_list = nicConfigurator.generate() - - self.assertEqual(2, len(nics_cfg_list), "number of elements") - - nic1 = {"name": "NIC1"} - nic2 = {"name": "NIC2"} - route_list = [] - for cfg in nics_cfg_list: - cfg_type = cfg.get("type") - if cfg_type == "physical": - if cfg.get("name") == nic1.get("name"): - nic1.update(cfg) - elif cfg.get("name") == nic2.get("name"): - nic2.update(cfg) - - self.assertEqual("physical", nic1.get("type"), "type of NIC1") - self.assertEqual("NIC1", nic1.get("name"), "name of NIC1") - self.assertEqual( - "00:50:56:a6:8c:08", nic1.get("mac_address"), "mac address of NIC1" - ) - - subnets = nic1.get("subnets") - self.assertEqual(2, len(subnets), "Number of subnets") - - static_subnet = [] - static6_subnet = [] - - for subnet in subnets: - subnet_type = subnet.get("type") - if subnet_type == "static": - static_subnet.append(subnet) - elif subnet_type == "static6": - static6_subnet.append(subnet) - else: - self.assertEqual(True, False, "Unknown type") - if "route" in subnet: - for route in subnet.get("routes"): - route_list.append(route) - - self.assertEqual(1, len(static_subnet), "Number of static subnet") - self.assertEqual(1, len(static6_subnet), "Number of static6 subnet") - - subnet = static_subnet[0] - self.assertEqual( - "10.20.87.154", - subnet.get("address"), - "IPv4 address of static subnet", - ) - self.assertEqual( - "255.255.252.0", subnet.get("netmask"), "NetMask of static subnet" - ) - self.assertEqual( - "auto", subnet.get("control"), "control for static subnet" - ) - - subnet = static6_subnet[0] - self.assertEqual( - "fc00:10:20:87::154", - subnet.get("address"), - "IPv6 address of static subnet", - ) - self.assertEqual( - "64", subnet.get("netmask"), "NetMask of static6 subnet" - ) - - route_set = set(["10.20.87.253", "10.20.87.105", "192.168.0.10"]) - for route in route_list: - self.assertEqual(10000, route.get("metric"), "metric of route") - gateway = route.get("gateway") - if gateway in route_set: - route_set.discard(gateway) - else: - self.assertEqual(True, False, "invalid gateway %s" % (gateway)) - - self.assertEqual("physical", nic2.get("type"), "type of NIC2") - self.assertEqual("NIC2", nic2.get("name"), "name of NIC2") - self.assertEqual( - "00:50:56:a6:ef:7d", nic2.get("mac_address"), "mac address of NIC2" - ) - - subnets = nic2.get("subnets") - self.assertEqual(1, len(subnets), "Number of subnets for NIC2") - - subnet = subnets[0] - self.assertEqual("static", subnet.get("type"), "Subnet type") - self.assertEqual( - "192.168.6.102", subnet.get("address"), "Subnet address" - ) - self.assertEqual( - "255.255.0.0", subnet.get("netmask"), "Subnet netmask" + nicConfigurator = NicConfigurator( + config.nics, config.name_servers, config.dns_suffixes, False ) + ethernets_dict = nicConfigurator.generate() + + self.assertTrue(isinstance(ethernets_dict, dict)) + self.assertEqual(2, len(ethernets_dict), "number of ethernets") + + for name, config in ethernets_dict.items(): + print(config) + if name == "NIC1": + self.assertEqual( + "00:50:56:a6:8c:08", + config.get("match").get("macaddress"), + "mac address of NIC1", + ) + self.assertEqual( + True, config.get("wakeonlan"), "wakeonlan of NIC1" + ) + self.assertEqual( + False, config.get("dhcp4"), "DHCPv4 enablement of NIC1" + ) + self.assertEqual( + False, config.get("dhcp6"), "DHCPv6 enablement of NIC1" + ) + self.assertEqual( + ["10.20.87.154/22", "fc00:10:20:87::154/64"], + config.get("addresses"), + "IP addresses of NIC1", + ) + self.assertEqual( + [ + {"to": "10.20.84.0/22", "via": "10.20.87.253"}, + {"to": "10.20.84.0/22", "via": "10.20.87.105"}, + { + "to": "fc00:10:20:87::/64", + "via": "fc00:10:20:87::253", + }, + ], + config.get("routes"), + "routes of NIC1", + ) + if name == "NIC2": + self.assertEqual( + "00:50:56:a6:ef:7d", + config.get("match").get("macaddress"), + "mac address of NIC2", + ) + self.assertEqual( + True, config.get("wakeonlan"), "wakeonlan of NIC2" + ) + self.assertEqual( + False, config.get("dhcp4"), "DHCPv4 enablement of NIC2" + ) + self.assertEqual( + ["192.168.6.102/16"], + config.get("addresses"), + "IP addresses of NIC2", + ) + self.assertEqual( + [ + {"to": "192.168.0.0/16", "via": "192.168.0.10"}, + ], + config.get("routes"), + "routes of NIC2", + ) def test_custom_script(self): cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") @@ -408,13 +371,18 @@ def _get_NicConfigurator(self, text): fp.write(text) fp.close() cfg = Config(ConfigFile(fp.name)) - return NicConfigurator(cfg.nics, use_system_devices=False) + return NicConfigurator( + cfg.nics, + cfg.name_servers, + cfg.dns_suffixes, + use_system_devices=False, + ) finally: if fp: os.unlink(fp.name) - def test_non_primary_nic_without_gateway(self): - """A non primary nic set is not required to have a gateway.""" + def test_static_nic_without_ipv4_netmask(self): + """netmask is optional for static ipv4 configuration.""" config = textwrap.dedent( """\ [NETWORK] @@ -432,29 +400,48 @@ def test_non_primary_nic_without_gateway(self): IPv4_MODE = BACKWARDS_COMPATIBLE BOOTPROTO = static IPADDR = 10.20.87.154 - NETMASK = 255.255.252.0 """ ) nc = self._get_NicConfigurator(config) self.assertEqual( - [ - { - "type": "physical", - "name": "NIC1", - "mac_address": "00:50:56:a6:8c:08", - "subnets": [ - { - "control": "auto", - "type": "static", - "address": "10.20.87.154", - "netmask": "255.255.252.0", - } - ], + { + "NIC1": { + "match": {"macaddress": "00:50:56:a6:8c:08"}, + "wakeonlan": True, + "dhcp4": False, + "addresses": ["10.20.87.154/32"], + "set-name": "NIC1", } - ], + }, nc.generate(), ) + def test_static_nic_without_ipv6_netmask(self): + """netmask is mandatory for static ipv6 configuration.""" + config = textwrap.dedent( + """\ + [NETWORK] + NETWORKING = yes + BOOTPROTO = dhcp + HOSTNAME = myhost1 + DOMAINNAME = eng.vmware.com + + [NIC-CONFIG] + NICS = NIC1 + + [NIC1] + MACADDR = 00:50:56:a6:8c:08 + ONBOOT = yes + IPv4_MODE = BACKWARDS_COMPATIBLE + BOOTPROTO = static + IPADDR = 10.20.87.154 + IPv6ADDR|1 = fc00:10:20:87::154 + """ + ) + nc = self._get_NicConfigurator(config) + with self.assertRaises(ValueError): + nc.generate() + def test_non_primary_nic_with_gateway(self): """A non primary nic set can have a gateway.""" config = textwrap.dedent( @@ -480,29 +467,16 @@ def test_non_primary_nic_with_gateway(self): ) nc = self._get_NicConfigurator(config) self.assertEqual( - [ - { - "type": "physical", - "name": "NIC1", - "mac_address": "00:50:56:a6:8c:08", - "subnets": [ - { - "control": "auto", - "type": "static", - "address": "10.20.87.154", - "netmask": "255.255.252.0", - "routes": [ - { - "type": "route", - "destination": "10.20.84.0/22", - "gateway": "10.20.87.253", - "metric": 10000, - } - ], - } - ], + { + "NIC1": { + "match": {"macaddress": "00:50:56:a6:8c:08"}, + "wakeonlan": True, + "dhcp4": False, + "addresses": ["10.20.87.154/22"], + "routes": [{"to": "10.20.84.0/22", "via": "10.20.87.253"}], + "set-name": "NIC1", } - ], + }, nc.generate(), ) @@ -540,29 +514,19 @@ def test_cust_non_primary_nic_with_gateway_(self): ) nc = self._get_NicConfigurator(config) self.assertEqual( - [ - { - "type": "physical", - "name": "NIC1", - "mac_address": "00:50:56:ac:d1:8a", - "subnets": [ - { - "control": "auto", - "type": "static", - "address": "100.115.223.75", - "netmask": "255.255.255.0", - "routes": [ - { - "type": "route", - "destination": "100.115.223.0/24", - "gateway": "100.115.223.254", - "metric": 10000, - } - ], - } + { + "NIC1": { + "match": {"macaddress": "00:50:56:ac:d1:8a"}, + "wakeonlan": True, + "dhcp4": False, + "addresses": ["100.115.223.75/24"], + "routes": [ + {"to": "100.115.223.0/24", "via": "100.115.223.254"} ], + "set-name": "NIC1", + "nameservers": {"addresses": ["8.8.8.8"]}, } - ], + }, nc.generate(), ) @@ -592,22 +556,16 @@ def test_a_primary_nic_with_gateway(self): ) nc = self._get_NicConfigurator(config) self.assertEqual( - [ - { - "type": "physical", - "name": "NIC1", - "mac_address": "00:50:56:a6:8c:08", - "subnets": [ - { - "control": "auto", - "type": "static", - "address": "10.20.87.154", - "netmask": "255.255.252.0", - "gateway": "10.20.87.253", - } - ], + { + "NIC1": { + "match": {"macaddress": "00:50:56:a6:8c:08"}, + "wakeonlan": True, + "dhcp4": False, + "addresses": ["10.20.87.154/22"], + "routes": [{"to": "0.0.0.0/0", "via": "10.20.87.253"}], + "set-name": "NIC1", } - ], + }, nc.generate(), ) diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 5d47e552..309acd34 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -254,6 +254,11 @@ MOCK_VIRT_IS_KVM_QEMU = {"name": "detect_virt", "RET": "qemu", "ret": 0} IS_KVM_QEMU_ENV = {"SYSTEMD_VIRTUALIZATION": "vm:qemu"} MOCK_VIRT_IS_VMWARE = {"name": "detect_virt", "RET": "vmware", "ret": 0} +MOCK_VIRT_IS_NOT_VMWARE = { + "name": "detect_virt", + "RET": "not-vmware", + "ret": 0, +} IS_VMWARE_ENV = {"SYSTEMD_VIRTUALIZATION": "vm:vmware"} # currenty' SmartOS hypervisor "bhyve" is unknown by systemd-detect-virt. MOCK_VIRT_IS_VM_OTHER = {"name": "detect_virt", "RET": "vm-other", "ret": 0} @@ -468,10 +473,11 @@ def _call_via_dict(self, data, rootd=None, **kwargs): def _test_ds_found(self, name): data = copy.deepcopy(VALID_CFG[name]) - - return self._check_via_dict( - data, RC_FOUND, dslist=[data.pop("ds"), DS_NONE] - ) + dslist = [] + for ds in data.pop("ds").split(","): + dslist.append(ds.strip()) + dslist.append(DS_NONE) + return self._check_via_dict(data, RC_FOUND, dslist=dslist) def _test_ds_not_found(self, name): data = copy.deepcopy(VALID_CFG[name]) @@ -928,6 +934,10 @@ def test_openstack_huawei_cloud(self): """Open Huawei Cloud identification.""" self._test_ds_found("OpenStack-HuaweiCloud") + def test_openstack_samsung_cloud_platform(self): + """Open Samsung Cloud Platform identification.""" + self._test_ds_found("OpenStack-SamsungCloudPlatform") + def test_openstack_asset_tag_nova(self): """OpenStack identification via asset tag OpenStack Nova.""" self._test_ds_found("OpenStack-AssetTag-Nova") @@ -1172,6 +1182,26 @@ def test_vmware_on_vmware_when_vmware_customization_is_enabled(self): """VMware is identified when vmware customization is enabled.""" self._test_ds_found("VMware-vmware-customization") + def test_vmware_ovf_on_vmware_with_vmware_customization_and_ovf_schema( + self, + ): + """VMware and OVF are identified when: + 1. On VMware platform. + 2. VMware customization is enabled. + 3. iso9660 cdrom path contains ovf schema.""" + self._test_ds_found( + "VMware-OVF-on-vmware-with-vmware-customization-and-ovf-schema" + ) + + def test_ovf_not_on_vmware_with_vmware_customization_and_ovf_schema(self): + """OVF is identified when: + 1. Not on VMware platform. + 2. VMware customization is enabled. + 3. iso9660 cdrom path contains ovf schema.""" + self._test_ds_found( + "OVF-not-on-vmware-with-vmware-customization-and-ovf-schema" + ) + def test_vmware_on_vmware_open_vm_tools_64(self): """VMware is identified when open-vm-tools installed in /usr/lib64.""" cust64 = copy.deepcopy(VALID_CFG["VMware-vmware-customization"]) @@ -1269,6 +1299,24 @@ def test_vmware_guestinfo_activated_by_vendordata(self): """VMware: guestinfo transport activated by vendordata""" self._test_ds_found("VMware-GuestInfo-Vendordata") + def test_vmware_ovf_on_vmware_with_guestinfo_metadata_and_ovf_schema(self): + """VMware and OVF are identified when: + 1. On VMware platform. + 2. guestinfo transport activated by metadata + 3. iso9660 cdrom path contains ovf schema.""" + self._test_ds_found( + "VMware-OVF-on-vmware-with-guestinfo-metadata-and-ovf-schema" + ) + + def test_ovf_not_on_vmware_with_guestinfo_metadata_and_ovf_schema(self): + """OVF is identified when: + 1. Not on VMware platform. + 2. guestinfo transport activated by metadata + 3. iso9660 cdrom path contains ovf schema.""" + self._test_ds_found( + "OVF-not-on-vmware-with-guestinfo-metadata-and-ovf-schema" + ) + class TestAkamai(DsIdentifyBase): def test_found_by_sys_vendor(self): @@ -2076,6 +2124,12 @@ def _print_run_output(rc, out, err, cfg, files): "files": {P_CHASSIS_ASSET_TAG: "HUAWEICLOUD\n"}, "mocks": [MOCK_VIRT_IS_KVM], }, + "OpenStack-SamsungCloudPlatform": { + # Samsung Cloud Platform hosts use OpenStack + "ds": "OpenStack", + "files": {P_CHASSIS_ASSET_TAG: "Samsung Cloud Platform\n"}, + "mocks": [MOCK_VIRT_IS_KVM], + }, "OpenStack-AssetTag-Nova": { # VMware vSphere can't modify product-name, LP: #1669875 "ds": "OpenStack", @@ -2481,6 +2535,92 @@ def _print_run_output(rc, out, err, cfg, files): "etc/cloud/cloud.cfg": "disable_vmware_customization: false\n", }, }, + "VMware-OVF-on-vmware-with-vmware-customization-and-ovf-schema": { + "ds": "VMware, OVF", + "mocks": [ + MOCK_VIRT_IS_VMWARE, + { + "name": "vmware_has_rpctool", + "ret": 0, + "out": "/usr/bin/vmware-rpctool", + }, + { + "name": "vmware_has_vmtoolsd", + "ret": 1, + "out": "/usr/bin/vmtoolsd", + }, + { + "name": "blkid", + "ret": 0, + "out": blkid_out( + [ + {"DEVNAME": "sr0", "TYPE": "iso9660", "LABEL": ""}, + { + "DEVNAME": "sr1", + "TYPE": "iso9660", + "LABEL": "ignoreme", + }, + { + "DEVNAME": "vda1", + "TYPE": "vfat", + "PARTUUID": uuid4(), + }, + ] + ), + }, + ], + "files": { + # Setup vmware customization enabled + "usr/lib/vmware-tools/plugins/vmsvc/libdeployPkgPlugin.so": "here", + "etc/cloud/cloud.cfg": "disable_vmware_customization: false\n", + # Setup ovf schema + "dev/sr0": "pretend ovf iso has " + OVF_MATCH_STRING + "\n", + "sys/class/block/sr0/size": "2048\n", + }, + }, + "OVF-not-on-vmware-with-vmware-customization-and-ovf-schema": { + "ds": "OVF", + "mocks": [ + MOCK_VIRT_IS_NOT_VMWARE, + { + "name": "vmware_has_rpctool", + "ret": 0, + "out": "/usr/bin/vmware-rpctool", + }, + { + "name": "vmware_has_vmtoolsd", + "ret": 1, + "out": "/usr/bin/vmtoolsd", + }, + { + "name": "blkid", + "ret": 0, + "out": blkid_out( + [ + {"DEVNAME": "sr0", "TYPE": "iso9660", "LABEL": ""}, + { + "DEVNAME": "sr1", + "TYPE": "iso9660", + "LABEL": "ignoreme", + }, + { + "DEVNAME": "vda1", + "TYPE": "vfat", + "PARTUUID": uuid4(), + }, + ] + ), + }, + ], + "files": { + # Setup vmware customization enabled + "usr/lib/vmware-tools/plugins/vmsvc/libdeployPkgPlugin.so": "here", + "etc/cloud/cloud.cfg": "disable_vmware_customization: false\n", + # Setup ovf schema + "dev/sr0": "pretend ovf iso has " + OVF_MATCH_STRING + "\n", + "sys/class/block/sr0/size": "2048\n", + }, + }, "VMware-EnvVar-NoData": { "ds": "VMware", "mocks": [ @@ -2757,6 +2897,112 @@ def _print_run_output(rc, out, err, cfg, files): MOCK_VIRT_IS_VMWARE, ], }, + "VMware-OVF-on-vmware-with-guestinfo-metadata-and-ovf-schema": { + "ds": "VMware, OVF", + "mocks": [ + MOCK_VIRT_IS_VMWARE, + { + "name": "vmware_has_rpctool", + "ret": 1, + "out": "/usr/bin/vmware-rpctool", + }, + { + "name": "vmware_has_vmtoolsd", + "ret": 0, + "out": "/usr/bin/vmtoolsd", + }, + { + "name": "vmware_guestinfo_metadata", + "ret": 0, + "out": "---", + }, + { + "name": "vmware_guestinfo_userdata", + "ret": 1, + }, + { + "name": "vmware_guestinfo_vendordata", + "ret": 1, + }, + { + "name": "blkid", + "ret": 0, + "out": blkid_out( + [ + {"DEVNAME": "sr0", "TYPE": "iso9660", "LABEL": ""}, + { + "DEVNAME": "sr1", + "TYPE": "iso9660", + "LABEL": "ignoreme", + }, + { + "DEVNAME": "vda1", + "TYPE": "vfat", + "PARTUUID": uuid4(), + }, + ] + ), + }, + ], + "files": { + # Setup ovf schema + "dev/sr0": "pretend ovf iso has " + OVF_MATCH_STRING + "\n", + "sys/class/block/sr0/size": "2048\n", + }, + }, + "OVF-not-on-vmware-with-guestinfo-metadata-and-ovf-schema": { + "ds": "OVF", + "mocks": [ + MOCK_VIRT_IS_NOT_VMWARE, + { + "name": "vmware_has_rpctool", + "ret": 1, + "out": "/usr/bin/vmware-rpctool", + }, + { + "name": "vmware_has_vmtoolsd", + "ret": 0, + "out": "/usr/bin/vmtoolsd", + }, + { + "name": "vmware_guestinfo_metadata", + "ret": 0, + "out": "---", + }, + { + "name": "vmware_guestinfo_userdata", + "ret": 1, + }, + { + "name": "vmware_guestinfo_vendordata", + "ret": 1, + }, + { + "name": "blkid", + "ret": 0, + "out": blkid_out( + [ + {"DEVNAME": "sr0", "TYPE": "iso9660", "LABEL": ""}, + { + "DEVNAME": "sr1", + "TYPE": "iso9660", + "LABEL": "ignoreme", + }, + { + "DEVNAME": "vda1", + "TYPE": "vfat", + "PARTUUID": uuid4(), + }, + ] + ), + }, + ], + "files": { + # Setup ovf schema + "dev/sr0": "pretend ovf iso has " + OVF_MATCH_STRING + "\n", + "sys/class/block/sr0/size": "2048\n", + }, + }, "Ec2-Outscale": { "ds": "Ec2", "files": { diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index e63e0eb6..e31e3a2e 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -5082,7 +5082,7 @@ def test_gi_excludes_any_without_mac_address(self, mocks): assert "tun0" in self._se_get_devicelist() found = [ent for ent in ret if "tun0" in ent] - assert len(found) == 0 + assert not found def test_gi_excludes_stolen_macs(self, mocks): ret = net.get_interfaces() diff --git a/tests/unittests/test_net_activators.py b/tests/unittests/test_net_activators.py index a720ada8..84876b73 100644 --- a/tests/unittests/test_net_activators.py +++ b/tests/unittests/test_net_activators.py @@ -247,8 +247,8 @@ def test_available(self, activator, available_calls, available_mocks): ), {}, ), - ((["systemctl", "reload-or-try-restart", "NetworkManager.service"],), {}), -] + ((["systemctl", "try-reload-or-restart", "NetworkManager.service"],), {}), +] + NETWORK_MANAGER_BRING_UP_CALL_LIST NETWORKD_BRING_UP_CALL_LIST: list = [ ((["ip", "link", "set", "dev", "eth0", "up"],), {}), diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index f9bc8897..ceb98b29 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -799,6 +799,22 @@ def test_get_hostname_fqdn_from_without_fqdn_or_hostname(self): mock.call(metadata_only=False), ] == cloud.get_hostname.call_args_list + def test_get_hostname_fqdn_from_numeric_fqdn(self): + """When cfg fqdn is numeric, ensure it is treated as a string.""" + hostname, fqdn, _ = util.get_hostname_fqdn( + cfg={"fqdn": 12345}, cloud=None + ) + self.assertEqual("12345", hostname) + self.assertEqual("12345", fqdn) + + def test_get_hostname_fqdn_from_numeric_fqdn_with_domain(self): + """When cfg fqdn is numeric with a domain, ensure correct parsing.""" + hostname, fqdn, _ = util.get_hostname_fqdn( + cfg={"fqdn": "12345.example.com"}, cloud=None + ) + self.assertEqual("12345", hostname) + self.assertEqual("12345.example.com", fqdn) + def test_get_hostname_fqdn_from_passes_metadata_only_to_cloud(self): """Calls to cloud.get_hostname pass the metadata_only parameter.""" cloud = mock.MagicMock() diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 98034440..6e2740ff 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -5,6 +5,7 @@ ader1990 adobley afbjorklund ajmyyra +akhuettel akutz AlexBaranowski alexsander-souza @@ -32,6 +33,7 @@ blackhelicoptersdotnet bmhughes brianphaley BrinKe-dev +bryanfraschetti CalvoM candlerb CarlosNihelton @@ -74,11 +76,13 @@ frikilax frittentheke GabrielNagy garzdin +gglzf4 giggsoff gilbsgilbs glyg halfdime-code hamalq +hamistao hcartiaux holmanb impl @@ -106,6 +110,7 @@ jordimassaguerpla jqueuniet jsf9k jshen28 +jumpojoy kadiron kaiwalyakoparkar kallioli @@ -131,6 +136,8 @@ ManassehZhou manuelisimo MarkMielke marlluslustosa +masihkhatibzadeh99 +mathmarchand matthewruffell maxnet Mazorius @@ -139,6 +146,7 @@ metajiji michaelrommel mitechie MjMoshiri +MostafaTarek124eru mxwebdev nazunalika nelsonad-ops @@ -167,15 +175,19 @@ rhansen riedel rishitashaw rmhsawyer +RomainDusi rongz609 s-makin SadeghHayeri sarahwzadara sbraz scorpion44 +SeanSith shaardie +shaerpour shell-skrimp shi2wei3 +ShPakvel simondeziel slingamn slyon @@ -199,6 +211,7 @@ tsanghan tSU-RooT tyb-truth tylerschultz +us0310306 vorlonofportland vteratipally Vultaire diff --git a/tools/ds-identify b/tools/ds-identify index e00b05e8..ac0f82f5 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -130,7 +130,7 @@ DI_DSNAME="" # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ CloudSigma CloudStack DigitalOcean Vultr AliYun Ec2 GCE OpenNebula OpenStack \ -OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud VMware \ +VMware OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud \ LXD NWCS Akamai WSL CloudCIX" DI_DSLIST="" DI_MODE="" @@ -1462,6 +1462,10 @@ dscheck_OpenStack() { return ${DS_FOUND} fi + if dmi_chassis_asset_tag_matches "Samsung Cloud Platform"; then + return ${DS_FOUND} + fi + # LP: #1669875 : allow identification of OpenStack by asset tag if dmi_chassis_asset_tag_matches "$nova"; then return ${DS_FOUND} diff --git a/tox.ini b/tox.ini index a6bccc7d..e7f3f6f1 100644 --- a/tox.ini +++ b/tox.ini @@ -247,6 +247,17 @@ passenv = SSH_AUTH_SOCK OS_* +[testenv:integration-tests-fast] +deps = + -r{toxinidir}/integration-requirements.txt + -r{toxinidir}/test-requirements.txt +commands = {envpython} -m pytest --log-cli-level=INFO -n auto -m "not hypothesis_slow" -m "not serial" {posargs:tests/integration_tests} +passenv = + CLOUD_INIT_* + PYCLOUDLIB_* + SSH_AUTH_SOCK + OS_* + [testenv:integration-tests-ci] deps = -r{toxinidir}/integration-requirements.txt commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests}