complete dependency tree readiness resolution

Author: varun-r-mallya

.gitignore

@@ -9,3 +9,4 @@ __pycache__/
 .ipynb_checkpoints/
 vmlinux.py
 ~*
+vmlinux.h

pythonbpf/vmlinux_parser/class_handler.py

@@ -33,20 +33,22 @@ def process_vmlinux_post_ast(
     symbols_in_module, imported_module = get_module_symbols("vmlinux")
 
     current_symbol_name = elem_type_class.__name__
+    logger.info(f"Begin {current_symbol_name} Processing")
     field_table = {}
     is_complex_type = False
     containing_type: Optional[Any] = None
     ctype_complex_type: Optional[Any] = None
     type_length: Optional[int] = None
     module_name = getattr(elem_type_class, "__module__", None)
 
-    if current_symbol_name in processing_stack:
-        logger.info(f"Circular dependency detected for {current_symbol_name}, skipping")
-        return True
-
     # Check if already processed
     if handler.has_node(current_symbol_name):
-        logger.info(f"Node {current_symbol_name} already processed and ready")
+        logger.debug(f"Node {current_symbol_name} already processed and ready")
+        return True
+
+    # XXX:Check it's use. It's probably not being used.
+    if current_symbol_name in processing_stack:
+        logger.debug(f"Dependency already in processing stack for {current_symbol_name}, skipping")
         return True
 
     processing_stack.add(current_symbol_name)
@@ -87,13 +89,12 @@ def process_vmlinux_post_ast(
                 elem_name, elem_temp_list = elem
                 [elem_type, elem_bitfield_size] = elem_temp_list
                 local_module_name = getattr(elem_type, "__module__", None)
+                new_dep_node.add_field(elem_name, elem_type, ready=False)
                 if local_module_name == ctypes.__name__:
-                    new_dep_node.add_field(elem_name, elem_type, ready=False)
                     new_dep_node.set_field_bitfield_size(elem_name, elem_bitfield_size)
                     new_dep_node.set_field_ready(elem_name, is_ready=True)
-                    logger.info(f"Field {elem_name} is direct ctypes type: {elem_type}")
+                    logger.debug(f"Field {elem_name} is direct ctypes type: {elem_type}")
                 elif local_module_name == "vmlinux":
-                    new_dep_node.add_field(elem_name, elem_type, ready=False)
                     new_dep_node.set_field_bitfield_size(elem_name, elem_bitfield_size)
                     logger.debug(
                         f"Processing vmlinux field: {elem_name}, type: {elem_type}"
@@ -103,6 +104,7 @@ def process_vmlinux_post_ast(
                         containing_type = elem_type._type_
                         if hasattr(elem_type, "_length_") and is_complex_type:
                             type_length = elem_type._length_
+
                         if containing_type.__module__ == "vmlinux":
                             pass
                         elif containing_type.__module__ == ctypes.__name__:
@@ -117,7 +119,7 @@ def process_vmlinux_post_ast(
                             raise ImportError(
                                 f"Unsupported module of {containing_type}"
                             )
-                        logger.info(
+                        logger.debug(
                             f"{containing_type} containing type of parent {elem_name} with {elem_type} and ctype {ctype_complex_type} and length {type_length}"
                         )
                         new_dep_node.set_field_containing_type(
@@ -129,20 +131,17 @@ def process_vmlinux_post_ast(
                         )
                         new_dep_node.set_field_type(elem_name, elem_type)
                         if containing_type.__module__ == "vmlinux":
-                            if process_vmlinux_post_ast(
-                                containing_type, llvm_handler, handler, processing_stack
-                            ):
-                                new_dep_node.set_field_ready(elem_name, True)
+                            process_vmlinux_post_ast(containing_type, llvm_handler, handler, processing_stack)
+                            new_dep_node.set_field_ready(elem_name, True)
                         elif containing_type.__module__ == ctypes.__name__:
-                            logger.info(f"Processing ctype internal{containing_type}")
+                            logger.debug(f"Processing ctype internal{containing_type}")
+                            new_dep_node.set_field_ready(elem_name, True)
                         else:
                             raise TypeError(
                                 "Module not supported in recursive resolution"
                             )
-                        continue
-                    if process_vmlinux_post_ast(
-                        elem_type, llvm_handler, handler, processing_stack
-                    ):
+                    else:
+                        process_vmlinux_post_ast(elem_type, llvm_handler, handler, processing_stack)
                         new_dep_node.set_field_ready(elem_name, True)
                 else:
                     raise ValueError(
@@ -152,5 +151,5 @@ def process_vmlinux_post_ast(
     else:
         raise ImportError("UNSUPPORTED Module")
 
-    print(current_symbol_name, "DONE")
-    print(f"handler readiness {handler.is_ready}")
+    logging.info(f"{current_symbol_name} processed and handler readiness {handler.is_ready}")
+    return True

pythonbpf/vmlinux_parser/dependency_node.py

@@ -20,41 +20,41 @@ def set_ready(self, is_ready: bool = True) -> None:
         """Set the readiness state of this field."""
         self.ready = is_ready
 
-    def set_value(self, value: Any, mark_ready: bool = True) -> None:
+    def set_value(self, value: Any, mark_ready: bool = False) -> None:
         """Set the value of this field and optionally mark it as ready."""
         self.value = value
         if mark_ready:
             self.ready = True
 
-    def set_type(self, given_type, mark_ready: bool = True) -> None:
+    def set_type(self, given_type, mark_ready: bool = False) -> None:
         """Set value of the type field and mark as ready"""
         self.type = given_type
         if mark_ready:
             self.ready = True
 
     def set_containing_type(
-        self, containing_type: Optional[Any], mark_ready: bool = True
+        self, containing_type: Optional[Any], mark_ready: bool = False
     ) -> None:
         """Set the containing_type of this field and optionally mark it as ready."""
         self.containing_type = containing_type
         if mark_ready:
             self.ready = True
 
-    def set_type_size(self, type_size: Any, mark_ready: bool = True) -> None:
+    def set_type_size(self, type_size: Any, mark_ready: bool = False) -> None:
         """Set the type_size of this field and optionally mark it as ready."""
         self.type_size = type_size
         if mark_ready:
             self.ready = True
 
     def set_ctype_complex_type(
-        self, ctype_complex_type: Any, mark_ready: bool = True
+        self, ctype_complex_type: Any, mark_ready: bool = False
     ) -> None:
         """Set the ctype_complex_type of this field and optionally mark it as ready."""
         self.ctype_complex_type = ctype_complex_type
         if mark_ready:
             self.ready = True
 
-    def set_bitfield_size(self, bitfield_size: Any, mark_ready: bool = True) -> None:
+    def set_bitfield_size(self, bitfield_size: Any, mark_ready: bool = False) -> None:
         """Set the bitfield_size of this field and optionally mark it as ready."""
         self.bitfield_size = bitfield_size
         if mark_ready:
@@ -138,7 +138,7 @@ def get_field(self, name: str) -> Field:
         """Get a field by name."""
         return self.fields[name]
 
-    def set_field_value(self, name: str, value: Any, mark_ready: bool = True) -> None:
+    def set_field_value(self, name: str, value: Any, mark_ready: bool = False) -> None:
         """Set a field's value and optionally mark it as ready."""
         if name not in self.fields:
             raise KeyError(f"Field '{name}' does not exist in node '{self.name}'")
@@ -147,7 +147,7 @@ def set_field_value(self, name: str, value: Any, mark_ready: bool = True) -> Non
         # Invalidate readiness cache
         self._ready_cache = None
 
-    def set_field_type(self, name: str, type: Any, mark_ready: bool = True) -> None:
+    def set_field_type(self, name: str, type: Any, mark_ready: bool = False) -> None:
         """Set a field's type and optionally mark it as ready."""
         if name not in self.fields:
             raise KeyError(f"Field '{name}' does not exist in node '{self.name}'")
@@ -157,7 +157,7 @@ def set_field_type(self, name: str, type: Any, mark_ready: bool = True) -> None:
         self._ready_cache = None
 
     def set_field_containing_type(
-        self, name: str, containing_type: Any, mark_ready: bool = True
+        self, name: str, containing_type: Any, mark_ready: bool = False
     ) -> None:
         """Set a field's containing_type and optionally mark it as ready."""
         if name not in self.fields:
@@ -168,7 +168,7 @@ def set_field_containing_type(
         self._ready_cache = None
 
     def set_field_type_size(
-        self, name: str, type_size: Any, mark_ready: bool = True
+        self, name: str, type_size: Any, mark_ready: bool = False
     ) -> None:
         """Set a field's type_size and optionally mark it as ready."""
         if name not in self.fields:
@@ -179,7 +179,7 @@ def set_field_type_size(
         self._ready_cache = None
 
     def set_field_ctype_complex_type(
-        self, name: str, ctype_complex_type: Any, mark_ready: bool = True
+        self, name: str, ctype_complex_type: Any, mark_ready: bool = False
     ) -> None:
         """Set a field's ctype_complex_type and optionally mark it as ready."""
         if name not in self.fields:
@@ -190,7 +190,7 @@ def set_field_ctype_complex_type(
         self._ready_cache = None
 
     def set_field_bitfield_size(
-        self, name: str, bitfield_size: Any, mark_ready: bool = True
+        self, name: str, bitfield_size: Any, mark_ready: bool = False
     ) -> None:
         """Set a field's bitfield_size and optionally mark it as ready."""
         if name not in self.fields:
@@ -200,7 +200,7 @@ def set_field_bitfield_size(
         # Invalidate readiness cache
         self._ready_cache = None
 
-    def set_field_ready(self, name: str, is_ready: bool = True) -> None:
+    def set_field_ready(self, name: str, is_ready: bool = False) -> None:
         """Mark a field as ready or not ready."""
         if name not in self.fields:
             raise KeyError(f"Field '{name}' does not exist in node '{self.name}'")
@@ -218,8 +218,8 @@ def is_ready(self) -> bool:
 
         # Calculate readiness only when needed
         if not self.fields:
-            self._ready_cache = False
-            return False
+            self._ready_cache = True
+            return True
 
         self._ready_cache = all(elem.ready for elem in self.fields.values())
         return self._ready_cache
@@ -231,3 +231,7 @@ def get_field_values(self) -> Dict[str, Any]:
     def get_ready_fields(self) -> Dict[str, Field]:
         """Get all fields that are marked as ready."""
         return {name: elem for name, elem in self.fields.items() if elem.ready}
+
+    def get_not_ready_fields(self) -> Dict[str, Field]:
+        """Get all fields that are marked as not ready."""
+        return {name: elem for name, elem in self.fields.items() if not elem.ready}

pythonbpf/vmlinux_parser/ir_generation.py

@@ -1,8 +1,11 @@
-# here, we will iterate through the dependencies and generate IR once dependencies are resolved fully
+import logging
 from .dependency_handler import DependencyHandler
 
+logger = logging.getLogger(__name__)
 
 class IRGenerator:
-    def __init__(self, module, handler):
+    def __init__(self, module, handler: DependencyHandler):
         self.module = module
         self.handler: DependencyHandler = handler
+        if not handler.is_ready:
+            raise ImportError("Semantic analysis of vmlinux imports failed. Cannot generate IR")

tests/failing_tests/xdp_pass.py

@@ -2,11 +2,9 @@
 from pythonbpf.maps import HashMap
 from pythonbpf.helper import XDP_PASS
 from vmlinux import struct_xdp_md
-
-from vmlinux import struct_ring_buffer_per_cpu  # noqa: F401
 from vmlinux import struct_xdp_buff  # noqa: F401
+from vmlinux import struct_ring_buffer_per_cpu  # noqa: F401
 
-# from vmlinux import struct_xdp_md
 from ctypes import c_int64
 
 # Instructions to how to run this program