Merge pull request #3182 from ciscorn/extract_pyi

Improve .pyi generation

.github/workflows/build.yml

@@ -37,7 +37,7 @@ jobs:
       run: |
         sudo apt-get install -y eatmydata
         sudo eatmydata apt-get install -y gettext librsvg2-bin mingw-w64
-        pip install requests sh click setuptools cpp-coveralls "Sphinx<4" sphinx-rtd-theme recommonmark sphinx-autoapi sphinxcontrib-svg2pdfconverter polib pyyaml astroid
+        pip install requests sh click setuptools cpp-coveralls "Sphinx<4" sphinx-rtd-theme recommonmark sphinx-autoapi sphinxcontrib-svg2pdfconverter polib pyyaml astroid isort
     - name: Versions
       run: |
         gcc --version

tools/extract_pyi.py

@@ -2,24 +2,91 @@
 #
 # SPDX-License-Identifier: MIT
 
+import ast
 import os
+import re
 import sys
-import astroid
 import traceback
 
-top_level = sys.argv[1].strip("/")
+import isort
-stub_directory = sys.argv[2]
+
+
+IMPORTS_IGNORE = frozenset({'int', 'float', 'bool', 'str', 'bytes', 'tuple', 'list', 'set', 'dict', 'bytearray', 'file', 'buffer'})
+IMPORTS_TYPING = frozenset({'Any', 'Optional', 'Union', 'Tuple', 'List', 'Sequence'})
+IMPORTS_TYPESHED = frozenset({'ReadableBuffer', 'WritableBuffer'})
+
+
+def is_any(node):
+    node_type = type(node)
+    if node is None:
+        return True
+    if node_type == ast.Name and node.id == "Any":
+        return True
+    if (node_type == ast.Attribute and type(node.value) == ast.Name
+            and node.value.id == "typing" and node.attr == "Any"):
+        return True
+    return False
+
+
+def report_missing_annotations(tree):
+    for node in ast.walk(tree):
+        node_type = type(node)
+        if node_type == ast.AnnAssign:
+            if is_any(node.annotation):
+                print(f"Missing attribute type on line {node.lineno}")
+        elif node_type == ast.arg:
+            if is_any(node.annotation) and node.arg != "self":
+                print(f"Missing argument type: {node.arg} on line {node.lineno}")
+        elif node_type == ast.FunctionDef:
+            if is_any(node.returns) and node.name != "__init__":
+                print(f"Missing return type: {node.name} on line {node.lineno}")
+
+
+def extract_imports(tree):
+    modules = set()
+    typing = set()
+    typeshed = set()
+
+    def collect_annotations(anno_tree):
+        if anno_tree is None:
+            return
+        for node in ast.walk(anno_tree):
+            node_type = type(node)
+            if node_type == ast.Name:
+                if node.id in IMPORTS_IGNORE:
+                    continue
+                elif node.id in IMPORTS_TYPING:
+                    typing.add(node.id)
+                elif node.id in IMPORTS_TYPESHED:
+                    typeshed.add(node.id)
+                elif not node.id[0].isupper():
+                    modules.add(node.id)
+
+    for node in ast.walk(tree):
+        node_type = type(node)
+        if (node_type == ast.AnnAssign) or (node_type == ast.arg):
+            collect_annotations(node.annotation)
+        elif node_type == ast.FunctionDef:
+            collect_annotations(node.returns)
+
+    return {
+        "modules": sorted(modules),
+        "typing": sorted(typing),
+        "typeshed": sorted(typeshed),
+    }
+
 
 def convert_folder(top_level, stub_directory):
     ok = 0
     total = 0
     filenames = sorted(os.listdir(top_level))
     pyi_lines = []
+
     for filename in filenames:
         full_path = os.path.join(top_level, filename)
         file_lines = []
         if os.path.isdir(full_path):
-            mok, mtotal = convert_folder(full_path, os.path.join(stub_directory, filename))
+            (mok, mtotal) = convert_folder(full_path, os.path.join(stub_directory, filename))
             ok += mok
             total += mtotal
         elif filename.endswith(".c"):
@@ -44,44 +111,57 @@ def convert_folder(top_level, stub_directory):
             pyi_lines.extend(file_lines)
 
     if not pyi_lines:
-        return ok, total
+        return (ok, total)
 
     stub_filename = os.path.join(stub_directory, "__init__.pyi")
     print(stub_filename)
     stub_contents = "".join(pyi_lines)
-    os.makedirs(stub_directory, exist_ok=True)
-    with open(stub_filename, "w") as f:
-        f.write(stub_contents)
 
     # Validate that the module is a parseable stub.
     total += 1
     try:
-        tree = astroid.parse(stub_contents)
+        tree = ast.parse(stub_contents)
-        for i in tree.body:
+        imports = extract_imports(tree)
-            if 'name' in i.__dict__:
+        report_missing_annotations(tree)
-                print(i.__dict__['name'])
-                for j in i.body:
-                    if isinstance(j, astroid.scoped_nodes.FunctionDef):
-                        if None in j.args.__dict__['annotations']:
-                            print(f"Missing parameter type: {j.__dict__['name']} on line {j.__dict__['lineno']}\n")
-                        if j.returns:
-                            if 'Any' in j.returns.__dict__.values():
-                                print(f"Missing return type: {j.__dict__['name']} on line {j.__dict__['lineno']}")
-                    elif isinstance(j, astroid.node_classes.AnnAssign):
-                        if 'name' in j.__dict__['annotation'].__dict__:
-                            if j.__dict__['annotation'].__dict__['name'] == 'Any':
-                                print(f"missing attribute type on line {j.__dict__['lineno']}")
-
         ok += 1
-    except astroid.exceptions.AstroidSyntaxError as e:
+    except SyntaxError as e:
-        e = e.__cause__
         traceback.print_exception(type(e), e, e.__traceback__)
+        return (ok, total)
+
+    # Add import statements
+    import_lines = ["from __future__ import annotations"]
+    import_lines.extend(f"import {m}" for m in imports["modules"])
+    import_lines.append("from typing import " + ", ".join(imports["typing"]))
+    import_lines.append("from _typeshed import " + ", ".join(imports["typeshed"]))
+    import_body = "\n".join(import_lines)
+    m = re.match(r'(\s*""".*?""")', stub_contents, flags=re.DOTALL)
+    if m:
+        stub_contents = m.group(1) + "\n\n" + import_body + "\n\n" + stub_contents[m.end():]
+    else:
+        stub_contents = import_body + "\n\n" + stub_contents
+    stub_contents = isort.code(stub_contents)
+
+    # Adjust blank lines
+    stub_contents = re.sub(r"\n+class", "\n\n\nclass", stub_contents)
+    stub_contents = re.sub(r"\n+def", "\n\n\ndef", stub_contents)
+    stub_contents = re.sub(r"\n+^(\s+)def", lambda m: f"\n\n{m.group(1)}def", stub_contents, flags=re.M)
+    stub_contents = stub_contents.strip() + "\n"
+
+    os.makedirs(stub_directory, exist_ok=True)
+    with open(stub_filename, "w") as f:
+        f.write(stub_contents)
+
     print()
-    return ok, total
+    return (ok, total)
 
-ok, total = convert_folder(top_level, stub_directory)
 
-print(f"{ok} ok out of {total}")
+if __name__ == "__main__":
+    top_level = sys.argv[1].strip("/")
+    stub_directory = sys.argv[2]
 
-if ok != total:
+    (ok, total) = convert_folder(top_level, stub_directory)
-    sys.exit(total - ok)
+
+    print(f"Parsing .pyi files: {total - ok} failed, {ok} passed")
+
+    if ok != total:
+        sys.exit(total - ok)