Merge pull request #507 from citrus-it/py313

pkg: ship python 3.13 modules

Author: oetiker

src/Makefile

@@ -101,3 +101,5 @@ $(TESTRESULTS): $(PYTESTS)
 
 FRC:
 
+.NO_PARALLEL: $(PYTESTS)
+

src/Makefile.com

@@ -34,7 +34,7 @@ CFLAGS = -m64 -Wall -Werror -Wextra -gdwarf-2 -gstrict-dwarf \
 CPPFLAGS = -D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS
 
 # Whitespace separated list of versions to build and test.
-PYVERSIONS = 3.12
+PYVERSIONS = 3.12 3.13
 # The single version used for shebang lines and packaging.
 PYVER = 3.12
 

src/brand/bhyve/requirements.txt

@@ -1,8 +1,8 @@
 #
 # This file was automatically produced by tools/updatereqs
-# Generated on Tue Nov 12 15:34:50 UTC 2024
+# Generated on Sun Dec 29 15:47:15 UTC 2024
 # Do not edit directly
 #
 construct==2.10.70
 PyYAML==6.0.2
-wheel==0.45.0
+wheel==0.45.1

src/modules/misc.py

@@ -3212,27 +3212,6 @@ def cmp(a, b):
         return NotImplemented
 
 
-def set_memory_limit(bytes, allow_override=True):
-    """Limit memory consumption of current process to 'bytes'.
-    This only limits the brk() and mmap() calls made by python."""
-
-    if allow_override:
-        try:
-            bytes = int(os.environ["PKG_CLIENT_MAX_PROCESS_SIZE"])
-        except (KeyError, ValueError):
-            pass
-
-    try:
-        resource.setrlimit(resource.RLIMIT_VMEM, (bytes, bytes))
-    except AttributeError:
-        # If platform doesn't support RLIMIT_VMEM, just ignore it.
-        pass
-    except ValueError:
-        # An unprivileged user can not raise a previously set limit,
-        # if that ever happens, just ignore it.
-        pass
-
-
 def force_bytes(s, encoding="utf-8", errors="strict"):
     """Force the string into bytes."""
 

src/modules/p5p.py

@@ -598,9 +598,6 @@ def add(self, pathname, arcname=None):
         self.__queue_offset += entry_sz
         self.__queue.append((pathname, ti.name))
 
-        # Discard tarinfo; it would be more efficient to keep these in
-        # memory, but at a significant memory footprint cost.
-        ti.tarfile = None
         del ti
 
     def __add_publisher_files(
@@ -1311,7 +1308,6 @@ def close(self, progtrack=None):
             ti = tfile.members.pop()
             if progtrack:
                 progtrack.archive_add_progress(1, tfile.offset - start_offset)
-            ti.tarfile = None
             del ti
 
         # Cleanup temporary files.

src/pkg/Makefile

@@ -108,7 +108,7 @@ INCORP            = consolidation\:ips\:ips-incorporation
 
 PM_TRANSFORMS     = defaults
 # For python version migrations
-#PM_TRANSFORMS     += pynext
+PM_TRANSFORMS     += pynext
 
 i386_DEFINES      = \
 	i386_ONLY=''    \

src/pkg/external_deps.txt

@@ -11,7 +11,20 @@
     pkg:/library/python-3/rapidjson-312
     pkg:/library/python-3/setuptools-312
     #
+    pkg:/runtime/python-313
+    pkg:/library/python-3/cffi-313
+    pkg:/library/python-3/coverage-313
+    pkg:/library/python-3/cryptography-313
+    pkg:/library/python-3/jsonrpclib-313
+    pkg:/library/python-3/jsonschema-313
+    pkg:/library/python-3/pybonjour-313
+    pkg:/library/python-3/pycurl-313
+    pkg:/library/python-3/pyopenssl-313
+    pkg:/library/python-3/rapidjson-313
+    pkg:/library/python-3/setuptools-313
+    #
     pkg:/system/library/python/libbe-312
+    pkg:/system/library/python/libbe-313
     #
     pkg:/SUNWcs
     pkg:/archiver/gnu-tar

src/pkg/manifests/package:pkg-35.p5m

@@ -28,6 +28,7 @@
 #
 <include bundledlicences.3.12.core.p5m>
 <include bundledmodules.3.12.core.p5m>
+<include bundledmodules.3.13.core.p5m>
 set name=pkg.fmri value=pkg:/package/pkg@$(PKGVERS)
 set name=pkg.summary value="Image Packaging System - pkg(7)"
 set name=pkg.description \

src/pkg/manifests/package:pkg.p5m

@@ -15,6 +15,7 @@
 
 <include bundledlicences.3.12.bhyve.p5m>
 <include bundledmodules.3.12.bhyve.p5m>
+<include bundledmodules.3.13.bhyve.p5m>
 set name=pkg.fmri value=pkg:/system/zones/brand/bhyve@$(PKGVERS)
 set name=pkg.summary value="Image Packaging System branded zone - bhyve zones"
 set name=pkg.description value="Support for bhyve branded zones"

src/pkg/manifests/system:zones:brand:bhyve.p5m

@@ -15,19 +15,21 @@
 # version of python alongside the existing ones to aid transition or
 # testing
 
+<transform dir path=$(PYDIR)$ -> emit %(action.name) path=TBD%(path) >
+
 <transform file dir path=$(PYDIRVP) -> \
   emit %(action.name) path=TBD%(path) \
   pkg.depend.bypass-generate=%(pkg.depend.bypass-generate;notfound='notfound')>
 
-<transform path=TBD -> delete pkg.depend.bypass-generate notfound>
+<transform file path=usr/lib/brand/bhyve/(uefi/)?__pycache__/ -> \
+  emit %(action.name) path=TBD%(path) \
+  pkg.depend.bypass-generate=%(pkg.depend.bypass-generate;notfound='notfound')>
 
-# Deliver 3.12 modules alongside the 3.11 ones (temporary, during migration)
-<transform path=TBD -> edit path 312 311>
-<transform path=TBD -> edit path 3.12 3.11>
+<transform path=TBD -> delete pkg.depend.bypass-generate notfound>
 
-# During the 3.11 to 3.12 upgrade, we need to also translate the loadable
-# module names as they were unqualified with 3.11
-<transform path=TBD -> edit path cpython-311-$(TRIPLE) cpython-311>
+# Deliver new modules alongside the old ones (temporary, during migration)
+<transform path=TBD -> edit path 312 313>
+<transform path=TBD -> edit path 3.12 3.13>
 
 <transform path=TBD -> edit path TBD ''>
 

src/pkg/transforms/pynext

@@ -464,9 +464,6 @@ def main_func():
 
     temp_root = misc.config_temp_root()
 
-    # set process limits for memory consumption to 8GB
-    misc.set_memory_limit(8 * 1024 * 1024 * 1024)
-
     global_settings.client_name = "pkgrecv"
     target = os.environ.get("PKG_DEST", None)
     src_uri = os.environ.get("PKG_SRC", None)

src/pull.py

@@ -18,7 +18,6 @@ ply==3.11
 portend==3.1.0
 prettytable==3.7.0
 pybonjour @ https://mirrors.omnios.org/pymodules/pybonjour/pybonjour-1.1.1-python3.tar.gz#sha256
-pycurl==7.44.1 # stuck on 7.44.1 - https://github.com/pycurl/pycurl/issues/748
 pydantic==1.10.18
 pytz==2023.3
 tempora==5.2.2

src/requirements-aarch64.txt

@@ -1,6 +1,6 @@
 #
 # This file was automatically produced by tools/updatereqs
-# Generated on Tue Nov 12 15:34:30 UTC 2024
+# Generated on Sun Dec 29 15:46:58 UTC 2024
 # Do not edit directly
 #
 autocommand==2.2.2
@@ -10,18 +10,17 @@ jaraco.collections==5.1.0
 jaraco.context==6.0.1
 jaraco.functools==4.1.0
 jaraco.text==4.0.0
-Mako==1.3.6
+Mako==1.3.8
 MarkupSafe==3.0.2
 more-itertools==10.5.0
 ply==3.11
 portend==3.2.0
 prettytable==3.12.0
 pybonjour @ https://mirrors.omnios.org/pymodules/pybonjour/pybonjour-1.1.1-python3.tar.gz#sha256
-pycurl==7.44.1 # stuck on 7.44.1 - https://github.com/pycurl/pycurl/issues/748
 python-dateutil==2.9.0.post0
-setuptools==75.4.0
-six==1.16.0
+setuptools==75.6.0
+six==1.17.0
 tempora==5.7.0
 wcwidth==0.2.13
-wheel==0.45.0
+wheel==0.45.1
 zc.lockfile==3.0.post1

src/requirements.txt

@@ -115,94 +115,6 @@ def test_psinfo(self):
         libc = ctypes.CDLL("libc.so")
         self.assertEqual(psinfo.pr_zoneid, libc.getzoneid())
 
-    def test_memory_limit(self):
-        """Verify that set_memory_limit works."""
-
-        # memory limit to test, keep small to avoid test slowdown
-        mem_cap = 100 * 1024 * 1024
-        # measured process resources. Note, that we have a static
-        # overhead in waste.py for the forking of ps, so while 20M seems
-        # large compared to a 100M limit, in a real world example with
-        # 8G limit it's fairly small.
-        mem_tol = 20 * 1024 * 1024
-
-        waste_mem_py = """
-import os
-import resource
-import subprocess
-
-import pkg.misc as misc
-
-misc.set_memory_limit({0})
-i = 0
-x = {{}}
-try:
-        while True:
-                i += 1
-                x[i] = range(i)
-except MemoryError:
-        # give us some breathing room (enough so the test with env var works)
-        misc.set_memory_limit({0} * 3, allow_override=False)
-        print(subprocess.check_output(['ps', '-o', 'rss=', '-p',
-            str(os.getpid())], universal_newlines=True).strip())
-""".format(
-            str(mem_cap)
-        )
-
-        # Re-setting limits which are higher than original limit can
-        # only be done by root.
-        self.assertTrue(os.geteuid() == 0, "must be root to run this test")
-
-        tmpdir = tempfile.mkdtemp(dir=self.test_root)
-        tmpfile = os.path.join(tmpdir, "waste.py")
-        with open(tmpfile, "w") as f:
-            f.write(waste_mem_py)
-
-        pyv = ".".join(platform.python_version_tuple()[:2])
-
-        res = int(subprocess.check_output([f"python{pyv}", tmpfile]))
-        # convert from kB to bytes
-        res *= 1024
-
-        self.debug("mem_cap:   " + str(mem_cap))
-        self.debug("proc size: " + str(res))
-
-        self.assertTrue(
-            res < mem_cap + mem_tol, "process mem consumption too high"
-        )
-        self.assertTrue(
-            res > mem_cap - mem_tol, "process mem consumption too low"
-        )
-
-        # test if env var works
-        os.environ["PKG_CLIENT_MAX_PROCESS_SIZE"] = str(mem_cap * 2)
-        res = int(subprocess.check_output([f"python{pyv}", tmpfile]))
-        res *= 1024
-
-        self.debug("mem_cap:   " + str(mem_cap))
-        self.debug("proc size: " + str(res))
-
-        self.assertTrue(
-            res < mem_cap * 2 + mem_tol, "process mem consumption too high"
-        )
-        # self.assertTrue(res > mem_cap * 2 - mem_tol,
-        #    "process mem consumption too low")
-
-        # test if invalid env var is handled correctly
-        os.environ["PKG_CLIENT_MAX_PROCESS_SIZE"] = "octopus"
-        res = int(subprocess.check_output([f"python{pyv}", tmpfile]))
-        res *= 1024
-
-        self.debug("mem_cap:   " + str(mem_cap))
-        self.debug("proc size: " + str(res))
-
-        self.assertTrue(
-            res < mem_cap + mem_tol, "process mem consumption too high"
-        )
-        self.assertTrue(
-            res > mem_cap - mem_tol, "process mem consumption too low"
-        )
-
 
 if __name__ == "__main__":
     unittest.main()