Skip to content

Fix unfalsifiable tests that swallow assertion failures #1199

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Asher- wants to merge 1 commit into
base: main
Choose a base branch
from
+25 −61
Open

Fix unfalsifiable tests that swallow assertion failures #1199

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 17 additions & 49 deletions test/solidlsp/python/test_symbol_retrieval.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ def test_request_defining_symbol_imported_class(self, language_server: SolidLang
assert defining_symbol.get("name") == "User"

@pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True)
@pytest.mark.xfail(reason="LS resolves to @dataclass decorator instead of create_user method — pre-existing limitation")
def test_request_defining_symbol_method_call(self, language_server: SolidLanguageServer) -> None:
"""Test request_defining_symbol for a method call."""
# Create an example file path for a file that calls UserService.create_user
Expand All @@ -227,19 +228,11 @@ def test_request_defining_symbol_method_call(self, language_server: SolidLanguag
defining_symbol = language_server.request_defining_symbol(examples_file_path, 10, 30)

# Verify that we found the defining symbol - should be the create_user method
# Because this might fail if the structure isn't exactly as expected, we'll use try-except
try:
assert defining_symbol is not None
assert defining_symbol.get("name") == "create_user"
# The defining symbol should be in the services.py file
if "location" in defining_symbol and "uri" in defining_symbol["location"]:
assert "services.py" in defining_symbol["location"]["uri"]
except AssertionError:
# If the file structure doesn't match what we expect, we can't guarantee this test
# will pass, so we'll consider it a warning rather than a failure
import warnings

warnings.warn("Could not verify method call definition - file structure may differ from expected")
assert defining_symbol is not None
assert defining_symbol.get("name") == "create_user"
# The defining symbol should be in the services.py file
if "location" in defining_symbol and "uri" in defining_symbol["location"]:
assert "services.py" in defining_symbol["location"]["uri"]

@pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True)
def test_request_defining_symbol_none(self, language_server: SolidLanguageServer) -> None:
Expand Down Expand Up @@ -290,27 +283,6 @@ def test_request_defining_symbol_nested_function(self, language_server: SolidLan
# Test 3: Find definition of a method-local function
defining_symbol = language_server.request_defining_symbol(file_path, 9, 15) # Position inside func_within_func

# This is challenging for many language servers and may fail
try:
assert defining_symbol is not None
assert defining_symbol.get("name") == "func_within_func"
except (AssertionError, TypeError, KeyError):
# This is expected to potentially fail in many implementations
import warnings

warnings.warn("Could not resolve nested class method definition - implementation limitation")

# Test 2: Find definition of the nested class
defining_symbol = language_server.request_defining_symbol(file_path, 15, 18) # Position of NestedClass

# This should resolve to the NestedClass
assert defining_symbol is not None
assert defining_symbol.get("name") == "NestedClass"
assert defining_symbol.get("kind") == SymbolKind.Class.value

# Test 3: Find definition of a method-local function
defining_symbol = language_server.request_defining_symbol(file_path, 9, 15) # Position inside func_within_func

# This is challenging for many language servers and may fail
assert defining_symbol is not None
assert defining_symbol.get("name") == "func_within_func"
Expand Down Expand Up @@ -346,24 +318,20 @@ def test_symbol_methods_integration(self, language_server: SolidLanguageServer)

# Try to get the container information for our method, but be flexible
# since implementations may vary
# verify container hierarchy: either containerName includes the class,
# or we can find the class by scanning containing symbols
container_name = defining_symbol.get("containerName", None)
if container_name and "UserService" in container_name:
# If containerName contains UserService, that's a valid implementation
pass
pass # containerName directly confirms the hierarchy
else:
# Try an alternative approach - looking for the containing class
try:
# Look for the class symbol in the file
for line in range(5, 12): # Approximate range where UserService class should be defined
symbol = language_server.request_containing_symbol(file_path, line, 5) # column 5 should be within class definition
if symbol and symbol.get("name") == "UserService" and symbol.get("kind") == SymbolKind.Class.value:
# Found the class - this is also a valid implementation
break
except Exception:
# Just log a warning - this is an alternative verification and not essential
import warnings

warnings.warn("Could not verify container hierarchy - implementation detail")
# alternative: look for the containing class via request_containing_symbol
found_class = False
for line in range(5, 12):
symbol = language_server.request_containing_symbol(file_path, line, 5)
if symbol and symbol.get("name") == "UserService" and symbol.get("kind") == SymbolKind.Class.value:
found_class = True
break
assert found_class, "Could not verify container hierarchy: UserService class not found"

@pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True)
def test_symbol_tree_structure(self, language_server: SolidLanguageServer) -> None:
Expand Down
20 changes: 8 additions & 12 deletions test/solidlsp/vue/test_vue_error_cases.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,8 @@ def test_document_symbols_nonexistent_file(self, language_server: SolidLanguageS
# If no exception, verify result is empty or indicates file not found
symbols = result.get_all_symbols_and_roots()
assert len(symbols[0]) == 0, f"Non-existent file should return empty symbols, got {len(symbols[0])} symbols"
except (FileNotFoundError, Exception) as e:
# Expected - file doesn't exist
assert True, f"Appropriately raised exception for non-existent file: {e}"
except FileNotFoundError:
pass # expected — file doesn't exist

@pytest.mark.parametrize("language_server", [Language.VUE], indirect=True)
def test_containing_symbol_nonexistent_file(self, language_server: SolidLanguageServer) -> None:
Expand All @@ -146,9 +145,8 @@ def test_containing_symbol_nonexistent_file(self, language_server: SolidLanguage
result = language_server.request_containing_symbol(nonexistent_file, 10, 10)
# If no exception, verify result indicates file not found
assert result is None or result == {}, f"Non-existent file should return None or empty dict, got: {result}"
except (FileNotFoundError, Exception) as e:
# Expected - file doesn't exist
assert True, f"Appropriately raised exception for non-existent file: {e}"
except FileNotFoundError:
pass # expected — file doesn't exist

@pytest.mark.parametrize("language_server", [Language.VUE], indirect=True)
def test_references_nonexistent_file(self, language_server: SolidLanguageServer) -> None:
Expand All @@ -165,9 +163,8 @@ def test_references_nonexistent_file(self, language_server: SolidLanguageServer)
assert result is None or isinstance(result, list), f"Non-existent file should return None or list, got: {result}"
if isinstance(result, list):
assert len(result) == 0, f"Non-existent file should return empty list, got {len(result)} references"
except (FileNotFoundError, Exception) as e:
# Expected - file doesn't exist
assert True, f"Appropriately raised exception for non-existent file: {e}"
except FileNotFoundError:
pass # expected — file doesn't exist

@pytest.mark.parametrize("language_server", [Language.VUE], indirect=True)
def test_definition_nonexistent_file(self, language_server: SolidLanguageServer) -> None:
Expand All @@ -183,9 +180,8 @@ def test_definition_nonexistent_file(self, language_server: SolidLanguageServer)
# If no exception, verify result is empty
assert isinstance(result, list), f"request_definition should return a list, got: {type(result)}"
assert len(result) == 0, f"Non-existent file should return empty list, got {len(result)} definitions"
except (FileNotFoundError, Exception) as e:
# Expected - file doesn't exist
assert True, f"Appropriately raised exception for non-existent file: {e}"
except FileNotFoundError:
pass # expected — file doesn't exist


class TestVueUndefinedSymbols:
Expand Down
Loading