diff --git a/.github/workflows/litevm.yml b/.github/workflows/litevm.yml index eedbc3ac..8865c687 100644 --- a/.github/workflows/litevm.yml +++ b/.github/workflows/litevm.yml @@ -46,10 +46,7 @@ jobs: sudo apt-get install qemu-kvm zstd gzip bzip2 cpio busybox-static fio \ autoconf automake check gcc git liblzma-dev \ libelf-dev libdw-dev libtool make pkgconf zlib1g-dev \ - binutils-dev - sudo wget -O /usr/bin/rpm2cpio https://github.com/rpm-software-management/rpm/raw/rpm-4.19.0-release/scripts/rpm2cpio.sh - echo '0403da24a797ccfa0cfd37bd4a6d6049370b9773e558da6173ae6ad25f97a428 /usr/bin/rpm2cpio' | sha256sum -c - - sudo chmod 755 /usr/bin/rpm2cpio + binutils-dev rpm2cpio - name: Setup test environment # Pinned virtualenv and tox are for the last versions which support # detecting Python 3.6 and running tests on it. @@ -60,7 +57,7 @@ jobs: - name: Build and install drgn with CTF support run: | cd .. - git clone https://github.com/brenns10/drgn -b ctf_0.0.30 + git clone https://github.com/brenns10/drgn -b ctf_0.0.31 cd drgn ../drgn-tools/venv/bin/python setup.py install - name: Run tests diff --git a/CHANGELOG.md b/CHANGELOG.md index f44a2915..f8753a38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,52 +1,37 @@ Changelog ========= -Release Cycle -------------- - -Prior to v1, the version numbers loosely followed the rule that new features -would bump the minor version level, and bug fix releases would bump the patch -version level. - -Beginning with v1, a new scheme is adopted which allows for a "development" -version, and a stable version. Using the version numbers `x.y.z`, we have: - -1. The **development** version is of the form `x.0.z`, where `x` represents the - major version under development. Each release is performed by incrementing - `z`, the patch level, regardless of the type of changes. The development - version ends with the release of the "stable" version of `x.1.0`. The - development version is maintained on the `main` branch. -2. The **stable** version is of the form `x.y.z`, where `y >= 1`, and `x` is of - course the major version. The "stable" versions are the only ones which are - released to Oracle Linux as RPMs. Releases will generally increment `z`, the - patch version, for bug fix releases. It's possible that in rare cases, we - will increment `y` for backports, in cases where we want to backport a module - to the stable release. The stable version is maintained in a branch named - `stable/vX`, where `X` is replaced with the major version number (e.g. - `stable/v1`). - -The stable version is maintained in parallel as the development version is -developed. Fixes in the stable release must first be present in the development -release (and all newer stable releases, if applicable). - -For the most part, regular maintenance of the stable version will end with the -release of the next stable version, but maintenance may continue at our -discretion. - -Examples: - -- `1.1.0` - the initial public release of the `1.x` stable series. -- `1.1.1` - the first bugfix release of the `1.x` stable series. -- `2.0.0` - the initial development version of the `2.x`. -- `2.0.1` - an incremental development release in `2.x` development. It may - contain bug fixes or new features. -- `2.1.0` - the initial public release of the `2.x` stable series. - - -Unreleased ----------- - -Changes which are committed to git, but not yet released, may appear here. +Development Model +----------------- + +Development of drgn-tools happens on the `main` branch. Bug fixes and new +features are committed there. Periodically, once we believe the set of features +is ready to be released, we tag a new version for release. These new versions +may increase the major or minor version of drgn-tools. Examples of these +versions are: + +- 2.0.0 +- 2.1.0 +- 3.0.0 + +When bugs are found during the development of the next version of drgn-tools, we +may backport them to the latest release in the form of a patch release. In that +case, a stable branch would be created. For example, a bug in the 2.1.0 release +would be backported into a branch named `stable/v2.1.x`, and a new patch release +2.1.1 would be created. + +Stable branches are maintained only until the release of the next version. + +Periodically, we may create development builds for internal distribution. It is +desirable for these builds to have a version number greater than any released +version, but not yet incremented to the next major/minor version. In this case, +we may set the patch version to "900" (and may increment it if multiple +development releases are warranted). + +All changelogs since the 1.1.0 release can be found in the RPM packaging +`%changelog` section, at `buildrpm/python-drgn-tools.spec`. Below are historical +changelogs for reference. + 1.1.0 - Tue, Aug 27, 2023 ------------------------- diff --git a/buildrpm/python-drgn-tools.spec b/buildrpm/python-drgn-tools.spec index 5e213389..5e389f5b 100644 --- a/buildrpm/python-drgn-tools.spec +++ b/buildrpm/python-drgn-tools.spec @@ -20,7 +20,7 @@ Name: python-drgn-tools -Version: 2.0.0 +Version: 2.1.0 Release: 1%{?dist} Summary: Helper scripts for drgn, containing the corelens utility @@ -59,7 +59,7 @@ a running kernel image (via /proc/kcore).} # The drgn dependency can be fulfilled by drgn with, or without, CTF support. # However, drgn-tools is tied to specific drgn releases. %global drgn_min 0.0.25 -%global drgn_max 0.0.31 +%global drgn_max 0.0.32 %package -n drgn-tools Summary: %{summary} @@ -132,6 +132,14 @@ rm %{buildroot}/usr/bin/DRGN %endif %changelog +* Thu Apr 17 2025 Stephen Brennan - 2.1.0-1 +- Add helper and module for unsubmitted pending work (Imran Khan) +- Add -V option to display version, and include the version in corelens reports (Stephen Brennan) [Orabug: 37503503] +- targetcli: add portal info (Richard Li) [Orabug: 37444641] +- Add --mmslot option to kvm module (Siddhi Katage) [Orabug: 37357370] +- Fix crash in show_unexpired_delayed_works with CTF (Stephen Brennan) [Orabug: 37695749] +- Add support for drgn 0.0.31 (Stephen Brennan) + * Fri Jan 10 2025 Stephen Brennan - 2.0.0-1 - Installing drgn-tools does not pull in drgn as a dependency (Stephen Brennan) [Orabug: 37126732] - Circular freelist causes infinite loop in corelens "slabinfo" module (Stephen Brennan) [Orabug: 37170860] diff --git a/drgn_tools/lock.py b/drgn_tools/lock.py index 7993b6c5..7c7630a3 100755 --- a/drgn_tools/lock.py +++ b/drgn_tools/lock.py @@ -510,4 +510,3 @@ def run(self, prog: Program, args: argparse.Namespace) -> None: print("Dont filter with both time and PID") return scan_lock(prog, args.stack, args.time, args.pid) - get_deadlock_info(prog, args.stack, args.time, args.pid) diff --git a/drgn_tools/locking.py b/drgn_tools/locking.py index 06b9eb4a..8d8e75c5 100644 --- a/drgn_tools/locking.py +++ b/drgn_tools/locking.py @@ -338,6 +338,17 @@ def for_each_rwsem_waiter_entity(rwsem: Object) -> Iterable[Object]: yield waiter +def rwsem_count(rwsem: Object) -> int: + try: + return rwsem.count.counter.value_() + except AttributeError: + # For all modern kernels, rwsem.count is an atomic_long_t. Prior to + # 8ee62b1870be8 ("locking/rwsem: Convert sem->count to + # 'atomic_long_t'"), which was merged in 4.8, it was a long. It's not + # too hard to fall back. + return rwsem.count.value_() + + def get_rwsem_owner(rwsem: Object) -> Tuple[Object, "RwsemStateCode"]: """ Find owner of given rwsem @@ -346,7 +357,7 @@ def get_rwsem_owner(rwsem: Object) -> Tuple[Object, "RwsemStateCode"]: :returns: type of owner and ``struct task_struct *`` if owner can be found, NULL otherwise """ prog = rwsem.prog_ - if not rwsem.count.counter.value_(): + if not rwsem_count(rwsem): return NULL(prog, "struct task_struct *"), RwsemStateCode.UNLOCKED if is_rwsem_writer_owned(rwsem): @@ -461,12 +472,13 @@ def is_rwsem_reader_owned(rwsem: Object) -> bool: case when type of owner could not be determined or when rwsem is free.) """ - if not rwsem.count.counter.value_(): # rwsem is free + count = rwsem_count(rwsem) + if not count: # rwsem is free return False if rwsem.owner.type_.type_name() == "atomic_long_t": - owner_is_writer = rwsem.count.counter.value_() & _RWSEM_WRITER_LOCKED + owner_is_writer = count & _RWSEM_WRITER_LOCKED owner_is_reader = ( - (rwsem.count.counter.value_() & _RWSEM_READER_MASK) + (count & _RWSEM_READER_MASK) and (rwsem.owner.counter.value_() & _RWSEM_READER_OWNED) and (owner_is_writer == 0) ) @@ -491,11 +503,12 @@ def is_rwsem_writer_owned(rwsem: Object) -> bool: case when type of owner could not be determined or when rwsem was free.) """ - if not rwsem.count.counter.value_(): # rwsem is free + count = rwsem_count(rwsem) + if not count: # rwsem is free return False if rwsem.owner.type_.type_name() == "atomic_long_t": - owner_is_writer = rwsem.count.counter.value_() & _RWSEM_WRITER_LOCKED + owner_is_writer = count & _RWSEM_WRITER_LOCKED return bool(owner_is_writer) else: if not rwsem.owner.value_(): @@ -536,7 +549,8 @@ def get_rwsem_info(rwsem: Object, callstack: int = 0) -> None: # of rwsem ->count and ->owner bits. print(f"({rwsem.type_.type_name()})0x{rwsem.value_():x}") - if not rwsem.count.counter.value_(): + count = rwsem_count(rwsem) + if not count: print("rwsem is free.") return @@ -557,7 +571,7 @@ def get_rwsem_info(rwsem: Object, callstack: int = 0) -> None: # So try to retrieve that info. if rwsem.owner.type_.type_name() == "atomic_long_t": num_readers = ( - rwsem.count.counter.value_() & _RWSEM_READER_MASK + count & _RWSEM_READER_MASK ) >> _RWSEM_READER_SHIFT print(f"Owned by {num_readers} reader(s)") else: diff --git a/drgn_tools/sys.py b/drgn_tools/sys.py index 4c60913b..de447dc3 100644 --- a/drgn_tools/sys.py +++ b/drgn_tools/sys.py @@ -81,7 +81,12 @@ def get_sysinfo(prog: Program) -> Dict[str, Any]: uts = prog["init_uts_ns"] else: raise Exception("error: could not find utsname information") - timekeeper = prog["shadow_timekeeper"] + try: + timekeeper = prog["shadow_timekeeper"] + except KeyError: + # 20c7b582e88b8 ("timekeeping: Move shadow_timekeeper into tk_core") + # Starting in v6.13 + timekeeper = prog["tk_core"].shadow_timekeeper date = time.ctime(timekeeper.xtime_sec) uptime = str(datetime.timedelta(seconds=int(timekeeper.ktime_sec))) jiffies = int(prog["jiffies"]) diff --git a/drgn_tools/workqueue.py b/drgn_tools/workqueue.py index 3537af18..b5a8795a 100644 --- a/drgn_tools/workqueue.py +++ b/drgn_tools/workqueue.py @@ -651,8 +651,8 @@ def show_unexpired_delayed_works( "entry", ): if ( - prog["delayed_work_timer_fn"].address_of_() - == timer.function + prog.symbol("delayed_work_timer_fn").address + == timer.function.value_() ): dwork = container_of( timer, diff --git a/mkdist.py b/mkdist.py new file mode 100644 index 00000000..a8785cb7 --- /dev/null +++ b/mkdist.py @@ -0,0 +1,114 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +""" +Create a zipapp distribution of drgn-tools which can provided to customers. +""" +import argparse +import shutil +import subprocess +import sys +import tempfile +import zipapp +from pathlib import Path + + +def main(): + entry_points = { + "corelens": "drgn_tools.corelens:main", + "cli": "drgn_tools.cli:main", + } + parser = argparse.ArgumentParser( + description="create drgn-tools distributions" + ) + parser.add_argument( + "--interpreter", + default="/usr/bin/python3", + help="Set the interpreter (if different from target system python)", + ) + parser.add_argument( + "--output", + "-o", + default=None, + help="Set the output file", + ) + parser.add_argument( + "--entry-point", + default="corelens", + help=f"Select an entry point ({','.join(entry_points.keys())} " + "or a function name)", + ) + parser.add_argument( + "--quiet", + "-q", + action="store_true", + help="just do it without any prompts or info", + ) + args = parser.parse_args() + + print( + """\ +Please note: the contents of the drgn_tools/ directory will be used to create +this distribution AS-IS! If you have any contents in that directory which should +not be distributed to a customer, please Ctrl-C now and clean them up. You may +want to use: + + git clean -ndx drgn_tools/ + +To see if you have any untracked files. You can use: + + git clean -fdx drgn_tools/ + +To delete everything listed by the prior command. Finally, you can use: + + git status drgn_tools/ + +To verify which files have uncommitted changes. It's totally fine to include +extra files & uncommitted changes, but it's important to be sure you only +include what you intended. + +Please hit enter to acknowledge and continue, or Ctrl-C to abort.\ +""" + ) + input() + base_dir = Path(__file__).parent + if args.entry_point in entry_points: + output_file = args.output or f"{args.entry_point}.pyz" + entry_point = entry_points[args.entry_point] + else: + output_file = args.output or "drgn_tools.pyz" + entry_point = args.entry_point + + # Be sure that we re-generate the "_version.py" file for accuracy + subprocess.run( + [sys.executable, base_dir / "setup.py", "--version"], + check=True, + stdout=subprocess.DEVNULL, + cwd=base_dir, + ) + + # Only the contents of "drgn_tools" should be included. All other files that + # are part of the project should be excluded. + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + shutil.copytree(base_dir / "drgn_tools", tmp / "drgn_tools") + zipapp.create_archive( + td, output_file, interpreter=args.interpreter, main=entry_point + ) + + print( + f"""\ +Created a distribution: {output_file} + +It can be directly copied to a target system, and it can be directly executed. +The target system MUST have a /usr/bin/python3 and have drgn installed. The +target system DOES NOT need to have drgn-tools installed -- but if it does, that +is fine. Nothing on the target system will be modified. + +You can use "unzip -l {output_file}" to check the contents of the zip file to +ensure only what you intended to include is present.\ +""" + ) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index fc5c7cba..1379b9d8 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ long_description = "drgn helper script repository" -RELEASE_VERSION = "2.0.0" +RELEASE_VERSION = "2.1.0" PACKAGES = ["drgn_tools"] @@ -88,7 +88,7 @@ def get_version(): description="drgn helper script repository", long_description=long_description, install_requires=[ - "drgn>=0.0.25,<0.0.31", + "drgn>=0.0.25,<0.0.32", ], url="https://github.com/oracle-samples/drgn-tools", author="Oracle Linux Sustaining Engineering Team", diff --git a/testing/litevm/rpm.py b/testing/litevm/rpm.py index 64d47927..54d3cef2 100644 --- a/testing/litevm/rpm.py +++ b/testing/litevm/rpm.py @@ -251,9 +251,6 @@ def get_oot_modules(self) -> List[Path]: # configurations and any customizations. It's not officially supported, but # it's an excellent test bed to ensure we are ready to support the latest # upstream features. - # We also apparently need to add "-modules-core" RPMs, because there weren't - # enough kernel RPMs yet. - # Tests currently fail on UEK-next. Uncomment this to enable the tests: TestKernel( 9, "next", @@ -269,6 +266,8 @@ def get_oot_modules(self) -> List[Path]: ), pkgbase="kernel-ueknext", ), + # UEK8 further distributes modules, so we need to add -modules-core. + TestKernel(9, 8, "x86_64", ["kernel-uek-core", "kernel-uek-modules", "kernel-uek-modules-core"]), # UEK7 switches from a single "kernel-uek" to "-core" and "-modules". # The "kernel-uek" package still exists as a placeholder. TestKernel(9, 7, "x86_64", ["kernel-uek-core", "kernel-uek-modules"]), diff --git a/tests/test_workqueue.py b/tests/test_workqueue.py index 57e53503..3cbaad4c 100644 --- a/tests/test_workqueue.py +++ b/tests/test_workqueue.py @@ -1,6 +1,7 @@ # Copyright (c) 2023, Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ import drgn +import pytest from drgn.helpers.linux.cpumask import for_each_online_cpu from drgn.helpers.linux.percpu import per_cpu from drgn.helpers.linux.percpu import per_cpu_ptr @@ -86,27 +87,32 @@ def test_for_each_pwq(prog: drgn.Program) -> None: assert pwqs.sort() == cpu_pwqs_list.sort() +@pytest.mark.skip_live def test_for_each_pending_work_on_cpu(prog: drgn.Program) -> None: for work in wq.for_each_pending_work_on_cpu(prog, 0): pass +@pytest.mark.skip_live def test_for_each_pending_work_in_pool(prog: drgn.Program) -> None: pool = per_cpu(prog["cpu_worker_pools"], 0)[0].address_of_() for work in wq.for_each_pending_work_in_pool(pool): pass +@pytest.mark.skip_live def test_for_each_pending_work_of_pwq(prog: drgn.Program) -> None: cpu_pwqs_0 = wq.workqueue_get_pwq(prog["system_wq"], 0) for work in wq.for_each_pending_work_of_pwq(cpu_pwqs_0): pass +@pytest.mark.skip_live def test_show_all_workqueues(prog: drgn.Program) -> None: wq.show_all_workqueues(prog) +@pytest.mark.skip_live def test_show_unexpired_delayed_works( prog: drgn.Program, ) -> None: