Skip to content

Commit 04cb8f5

Browse files
einarfeinarf
authored
Clean up the path resolver for doc members (#2612)
* Split the import resolver into its own module * basic testing
1 parent 27d2866 commit 04cb8f5

File tree

2 files changed

+207
-142
lines changed

2 files changed

+207
-142
lines changed

‎util/doc_helpers/__init__.py

Lines changed: 67 additions & 142 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,12 @@
1-
from __future__ import annotations
2-
31
import re
4-
import ast
5-
import dataclasses
6-
from pathlib import Path
72
from typing import Iterable
3+
from pathlib import Path
84

95
from .vfs import VirtualFile, Vfs, F
10-
11-
12-
__all__ = (
13-
'get_module_path',
14-
'EMPTY_TUPLE',
15-
'F',
16-
'SharedPaths',
17-
'NotExcludedBy',
18-
'VirtualFile',
19-
'Vfs'
20-
)
21-
6+
from .import_resolver import build_import_tree
227

238
EMPTY_TUPLE = tuple()
24-
25-
26-
class SharedPaths:
27-
"""These are often used to set up a Vfs and open files."""
28-
REPO_UTILS_DIR = Path(__file__).parent.parent.resolve()
29-
REPO_ROOT = REPO_UTILS_DIR.parent
30-
ARCADE_ROOT = REPO_ROOT / "arcade"
31-
DOC_ROOT = REPO_ROOT / "doc"
32-
API_DOC_ROOT = DOC_ROOT / "api_docs"
9+
_VALID_MODULE_SEGMENT = re.compile(r"[_a-zA-Z][_a-z0-9]*")
3310

3411

3512
class NotExcludedBy:
@@ -45,7 +22,14 @@ def __call__(self, item) -> bool:
4522
return item not in self.items
4623

4724

48-
_VALID_MODULE_SEGMENT = re.compile(r"[_a-zA-Z][_a-z0-9]*")
25+
class SharedPaths:
26+
"""These are often used to set up a Vfs and open files."""
27+
REPO_UTILS_DIR = Path(__file__).parent.parent.resolve()
28+
REPO_ROOT = REPO_UTILS_DIR.parent
29+
ARCADE_ROOT = REPO_ROOT / "arcade"
30+
DOC_ROOT = REPO_ROOT / "doc"
31+
API_DOC_ROOT = DOC_ROOT / "api_docs"
32+
4933

5034

5135
def get_module_path(module: str, root = SharedPaths.REPO_ROOT) -> Path:
@@ -90,127 +74,68 @@ def get_module_path(module: str, root = SharedPaths.REPO_ROOT) -> Path:
9074
f"{module}")
9175

9276
return current
77+
class SharedPaths:
78+
"""These are often used to set up a Vfs and open files."""
79+
REPO_UTILS_DIR = Path(__file__).parent.parent.resolve()
80+
REPO_ROOT = REPO_UTILS_DIR.parent
81+
ARCADE_ROOT = REPO_ROOT / "arcade"
82+
DOC_ROOT = REPO_ROOT / "doc"
83+
API_DOC_ROOT = DOC_ROOT / "api_docs"
9384

9485

95-
# Tools for resolving the lowest import of a member in Arcade.
96-
# Members are imported in various `__init__` files and we want
97-
# present. arcade.Sprite instead of arcade.sprite.Sprite as an example.
98-
# Build a tree using the ast module looking at the __init__ files
99-
# and recurse the tree to find the lowest import of a member.
100-
101-
@dataclasses.dataclass
102-
class ImportNode:
103-
"""A node in the import tree."""
104-
name: str
105-
parent: ImportNode | None = None
106-
children: list[ImportNode] = dataclasses.field(default_factory=list)
107-
imports: list[Import] = dataclasses.field(default_factory=list)
108-
level: int = 0
109-
110-
def get_full_module_path(self) -> str:
111-
"""Get the module path from the root to this node."""
112-
if self.parent is None:
113-
return self.name
114-
115-
name = self.parent.get_full_module_path()
116-
if name:
117-
return f"{name}.{self.name}"
118-
return self.name
119-
120-
def resolve(self, full_path: str) -> str:
121-
"""Return the lowest import of a member in the tree."""
122-
name = full_path.split(".")[-1]
123-
124-
# Find an import in this module likely to be the one we want.
125-
for imp in self.imports:
126-
if imp.name == name and imp.from_module in full_path:
127-
return f"{imp.module}.{imp.name}"
128-
129-
# Move on to children
130-
for child in self.children:
131-
result = child.resolve(full_path)
132-
if result:
133-
return result
134-
135-
# Return the full path if we can't find any relevant imports.
136-
# It means the member is in a sub-module and are not importer anywhere.
137-
return full_path
138-
139-
def print_tree(self, depth=0):
140-
"""Print the tree."""
141-
print(" " * depth * 4, "---", self.name)
142-
for imp in self.imports:
143-
print(" " * (depth + 1) * 4, f"-> {imp}")
144-
for child in self.children:
145-
child.print_tree(depth + 1)
146-
147-
148-
@dataclasses.dataclass
149-
class Import:
150-
"""Unified representation of an import statement."""
151-
name: str # name of the member
152-
module: str # The module this import is from
153-
from_module: str # The module the member was imported from
154-
155-
156-
def build_import_tree(root: Path) -> ImportNode:
157-
"""
158-
Build a tree of all the modules in a package.
86+
87+
def get_module_path(module: str, root = SharedPaths.REPO_ROOT) -> Path:
88+
"""Quick-n-dirty module path estimation relative to the repo root.
15989
16090
Args:
161-
root: The root of the package to build the tree from.
91+
module: A module path in the project.
92+
Raises:
93+
ValueError: When a can't be computed.
16294
Returns:
163-
The root node of the tree.
95+
An absolute file path to the module
16496
"""
165-
node = _parse_import_node_recursive(root, parent=None)
166-
if node is None:
167-
raise RuntimeError("No __init__.py found in root")
168-
return node
97+
# Convert module.name.here to module/name/here
98+
current = root
99+
for index, part in enumerate(module.split('.')):
100+
if not _VALID_MODULE_SEGMENT.fullmatch(part):
101+
raise ValueError(
102+
f'Invalid module segment at index {index}: {part!r}')
103+
# else:
104+
# print(current, part)
105+
current /= part
106+
107+
# Account for the two kinds of modules:
108+
# 1. arcade/module.py
109+
# 2. arcade/module/__init__.py
110+
as_package = current / "__init__.py"
111+
have_package = as_package.is_file()
112+
as_file = current.with_suffix('.py')
113+
have_file = as_file.is_file()
114+
115+
# TODO: When 3.10 becomes our min Python, make this a match-case?
116+
if have_package and have_file:
117+
raise ValueError(
118+
f"Module conflict between {as_package} and {as_file}")
119+
elif have_package:
120+
current = as_package
121+
elif have_file:
122+
current = as_file
123+
else:
124+
raise ValueError(
125+
f"No folder package or file module detected for "
126+
f"{module}")
169127

128+
return current
170129

171-
def _parse_import_node_recursive(
172-
path: Path,
173-
parent: ImportNode | None = None,
174-
depth=0,
175-
) -> ImportNode | None:
176-
"""Quickly gather import data using ast in a simplified/unified format.
177130

178-
This is a recursive function that works itself down the directory tree
179-
looking for __init__.py files and parsing them for imports.
180-
"""
181-
_file = path / "__init__.py"
182-
if not _file.exists():
183-
return None
184-
185-
# Build the node
186-
name = _file.parts[-2]
187-
node = ImportNode(name, parent=parent)
188-
module = ast.parse(_file.read_text())
189-
190-
full_module_path = node.get_full_module_path()
191-
192-
for ast_node in ast.walk(module):
193-
if isinstance(ast_node, ast.Import):
194-
for alias in ast_node.names:
195-
if not alias.name.startswith("arcade."):
196-
continue
197-
imp = Import(
198-
name=alias.name.split(".")[-1],
199-
module=full_module_path,
200-
from_module=".".join(alias.name.split(".")[:-1])
201-
)
202-
node.imports.append(imp)
203-
elif isinstance(ast_node, ast.ImportFrom):
204-
if ast_node.level == 0 and not ast_node.module.startswith("arcade"):
205-
continue
206-
for alias in ast_node.names:
207-
imp = Import(alias.name, full_module_path, ast_node.module)
208-
node.imports.append(imp)
209-
210-
# Recurse subdirectories
211-
for child_dir in path.iterdir():
212-
child = _parse_import_node_recursive(child_dir, parent=node, depth=depth + 1)
213-
if child:
214-
node.children.append(child)
215-
216-
return node
131+
132+
__all__ = (
133+
'get_module_path',
134+
'SharedPaths',
135+
'EMPTY_TUPLE',
136+
'F',
137+
'NotExcludedBy',
138+
'VirtualFile',
139+
'Vfs',
140+
'build_import_tree',
141+
)

‎util/doc_helpers/import_resolver.py

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
from __future__ import annotations
2+
3+
import ast
4+
import dataclasses
5+
from pathlib import Path
6+
7+
8+
# Tools for resolving the lowest import of a member in Arcade.
9+
# Members are imported in various `__init__` files and we want
10+
# present. arcade.Sprite instead of arcade.sprite.Sprite as an example.
11+
# Build a tree using the ast module looking at the __init__ files
12+
# and recurse the tree to find the lowest import of a member.
13+
14+
@dataclasses.dataclass
15+
class ImportNode:
16+
"""A node in the import tree."""
17+
name: str
18+
parent: ImportNode | None = None
19+
children: list[ImportNode] = dataclasses.field(default_factory=list)
20+
imports: list[Import] = dataclasses.field(default_factory=list)
21+
level: int = 0
22+
23+
def get_full_module_path(self) -> str:
24+
"""Get the module path from the root to this node."""
25+
if self.parent is None:
26+
return self.name
27+
28+
name = self.parent.get_full_module_path()
29+
if name:
30+
return f"{name}.{self.name}"
31+
return self.name
32+
33+
def resolve(self, full_path: str) -> str:
34+
"""Return the lowest import of a member in the tree."""
35+
name = full_path.split(".")[-1]
36+
37+
# Find an import in this module likely to be the one we want.
38+
for imp in self.imports:
39+
if imp.name == name and imp.from_module in full_path:
40+
return f"{imp.module}.{imp.name}"
41+
42+
# Move on to children
43+
for child in self.children:
44+
result = child.resolve(full_path)
45+
if result:
46+
return result
47+
48+
# Return the full path if we can't find any relevant imports.
49+
# It means the member is in a sub-module and are not importer anywhere.
50+
return full_path
51+
52+
def print_tree(self, depth=0):
53+
"""Print the tree."""
54+
print(" " * depth * 4, "---", self.name)
55+
for imp in self.imports:
56+
print(" " * (depth + 1) * 4, f"-> {imp}")
57+
for child in self.children:
58+
child.print_tree(depth + 1)
59+
60+
61+
@dataclasses.dataclass
62+
class Import:
63+
"""Unified representation of an import statement."""
64+
name: str # name of the member
65+
module: str # The module this import is from
66+
from_module: str # The module the member was imported from
67+
68+
69+
def build_import_tree(root: Path) -> ImportNode:
70+
"""
71+
Build a tree of all the modules in a package.
72+
73+
Args:
74+
root: The root of the package to build the tree from.
75+
Returns:
76+
The root node of the tree.
77+
"""
78+
node = _parse_import_node_recursive(root, parent=None)
79+
if node is None:
80+
raise RuntimeError("No __init__.py found in root")
81+
return node
82+
83+
84+
def _parse_import_node_recursive(
85+
path: Path,
86+
parent: ImportNode | None = None,
87+
depth=0,
88+
) -> ImportNode | None:
89+
"""Quickly gather import data using ast in a simplified/unified format.
90+
91+
This is a recursive function that works itself down the directory tree
92+
looking for __init__.py files and parsing them for imports.
93+
"""
94+
_file = path / "__init__.py"
95+
if not _file.exists():
96+
return None
97+
98+
# Build the node
99+
name = _file.parts[-2]
100+
node = ImportNode(name, parent=parent)
101+
module = ast.parse(_file.read_text())
102+
103+
full_module_path = node.get_full_module_path()
104+
105+
for ast_node in ast.walk(module):
106+
if isinstance(ast_node, ast.Import):
107+
for alias in ast_node.names:
108+
if not alias.name.startswith("arcade."):
109+
continue
110+
imp = Import(
111+
name=alias.name.split(".")[-1],
112+
module=full_module_path,
113+
from_module=".".join(alias.name.split(".")[:-1])
114+
)
115+
node.imports.append(imp)
116+
elif isinstance(ast_node, ast.ImportFrom):
117+
if ast_node.level == 0 and not ast_node.module.startswith("arcade"):
118+
continue
119+
for alias in ast_node.names:
120+
imp = Import(alias.name, full_module_path, ast_node.module)
121+
node.imports.append(imp)
122+
123+
# Recurse subdirectories
124+
for child_dir in path.iterdir():
125+
child = _parse_import_node_recursive(child_dir, parent=node, depth=depth + 1)
126+
if child:
127+
node.children.append(child)
128+
129+
return node
130+
131+
132+
if __name__ == "__main__":
133+
# Basic testing. cwd: util/
134+
root = build_import_tree(Path(__file__).parent.parent.parent.resolve() / "arcade")
135+
136+
# Check paths
137+
path = root.resolve("arcade.sprite.Sprite")
138+
print(path)
139+
path = root.resolve("arcade.camera.Camera2D")
140+
print(path)

0 commit comments

Comments
 (0)