docs/find-missing.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Copyright (c) 2019-2023. The SimGrid Team. All rights reserved.
   5
   6 # This program is free software; you can redistribute it and/or modify it
   7 # under the terms of the license (GNU LGPL) which comes with this package.
   8
   9 """
  10 Search for symbols documented in both the XML files produced by Doxygen and the python modules,
  11 but not documented with breathe in the RST files.
  12
  13 This script is tailored to SimGrid own needs.
  14
  15 If you are missing some dependencies, try:  pip3 install --requirement docs/requirements.txt
  16 """
  17
  18 import os
  19 import re
  20 import sys
  21 import xml.etree.ElementTree as ET
  22 import inspect
  23
  24 xml_files = [
  25     'build/xml/classsimgrid_1_1s4u_1_1Activity.xml',
  26     'build/xml/classsimgrid_1_1s4u_1_1Actor.xml',
  27     'build/xml/classsimgrid_1_1s4u_1_1Barrier.xml',
  28     'build/xml/classsimgrid_1_1s4u_1_1Comm.xml',
  29     'build/xml/classsimgrid_1_1s4u_1_1ConditionVariable.xml',
  30     'build/xml/classsimgrid_1_1s4u_1_1Disk.xml',
  31     'build/xml/classsimgrid_1_1s4u_1_1Engine.xml',
  32     'build/xml/classsimgrid_1_1s4u_1_1Exec.xml',
  33     'build/xml/classsimgrid_1_1s4u_1_1Host.xml',
  34     'build/xml/classsimgrid_1_1s4u_1_1Io.xml',
  35     'build/xml/classsimgrid_1_1s4u_1_1Link.xml',
  36     'build/xml/classsimgrid_1_1s4u_1_1Mailbox.xml',
  37     'build/xml/classsimgrid_1_1s4u_1_1Mutex.xml',
  38     'build/xml/classsimgrid_1_1s4u_1_1NetZone.xml',
  39     'build/xml/classsimgrid_1_1s4u_1_1Semaphore.xml',
  40     'build/xml/classsimgrid_1_1s4u_1_1VirtualMachine.xml',
  41     'build/xml/classsimgrid_1_1xbt_1_1signal_3_01R_07P_8_8_8_08_4.xml',
  42     'build/xml/namespacesimgrid_1_1s4u_1_1this__actor.xml',
  43     'build/xml/actor_8h.xml',
  44     'build/xml/barrier_8h.xml',
  45     'build/xml/cond_8h.xml',
  46     'build/xml/engine_8h.xml',
  47     'build/xml/forward_8h.xml',
  48     'build/xml/host_8h.xml',
  49     'build/xml/link_8h.xml',
  50     'build/xml/mailbox_8h.xml',
  51     'build/xml/mutex_8h.xml',
  52     'build/xml/semaphore_8h.xml',
  53     'build/xml/vm_8h.xml',
  54     'build/xml/zone_8h.xml'
  55 ]
  56
  57 python_modules = [
  58     'simgrid'
  59 ]
  60 python_ignore = [
  61     'simgrid.ActorKilled'
  62 ]
  63
  64
  65 ############ Search the python elements in the source, and report undocumented ones
  66 ############
  67
  68 # data structure in which we store the declaration we find
  69 python_decl = {}
  70
  71 def handle_python_module(fullname, englobing, elm):
  72     """Recursive function exploring the python modules."""
  73
  74     def found_decl(kind, obj):
  75         """Helper function that add an object in the python_decl data structure"""
  76         if kind not in python_decl:
  77             python_decl[kind] = []
  78         python_decl[kind].append(obj)
  79
  80
  81     if fullname in python_ignore:
  82         print("Ignore Python symbol '{}' as requested.".format(fullname))
  83         return
  84
  85     if inspect.isroutine(elm) and inspect.isclass(englobing):
  86         found_decl("method", fullname)
  87 #        print('.. automethod:: {}'.format(fullname))
  88     elif inspect.isroutine(elm) and (not inspect.isclass(englobing)):
  89         found_decl("function", fullname)
  90 #        print('.. autofunction:: {}'.format(fullname))
  91     elif inspect.isdatadescriptor(elm):
  92         found_decl("attribute", fullname)
  93 #        print('.. autoattribute:: {}'.format(fullname))
  94     elif isinstance(elm, (int, str)): # We do have such a data, directly in the SimGrid top module
  95         found_decl("data", fullname)
  96 #        print('.. autodata:: {}'.format(fullname))
  97     elif inspect.ismodule(elm) or inspect.isclass(elm):
  98         for name, data in inspect.getmembers(elm):
  99             if name.startswith('__'):
 100                 continue
 101 #            print("Recurse on {}.{}".format(fullname, name))
 102             handle_python_module("{}.{}".format(fullname, name), elm, data)
 103     else:
 104         print('UNHANDLED TYPE {} : {!r} Type: {}'.format(fullname, elm, type(elm)))
 105
 106 # Start the recursion on the provided Python modules
 107 for name in python_modules:
 108     try:
 109         module = __import__(name)
 110     except Exception:
 111         if os.path.exists("../lib") and "../lib" not in sys.path:
 112             print("Adding ../lib to PYTHONPATH as {} cannot be imported".format(name))
 113             sys.path.append("../lib")
 114             try:
 115                 module = __import__(name)
 116             except Exception:
 117                 print("Cannot import {}, even with PYTHONPATH=../lib".format(name))
 118                 sys.exit(1)
 119         else:
 120             print("Cannot import {}".format(name))
 121             sys.exit(1)
 122     for sub in dir(module):
 123         if sub[0] == '_':
 124             continue
 125         handle_python_module("{}.{}".format(name, sub), module, getattr(module, sub))
 126
 127 # Forget about the python declarations that were actually done in the RST
 128 for kind in python_decl:
 129     with os.popen('grep \'[[:blank:]]*auto{}::\' source/*rst|sed \'s/^.*auto{}:: //\''.format(kind, kind)) as pse:
 130         for fullname in (l.strip() for l in pse):
 131             if fullname not in python_decl[kind]:
 132                 print("Warning: {} documented but declaration not found in python.".format(fullname))
 133             else:
 134                 python_decl[kind].remove(fullname)
 135 # Dump the missing ones
 136 for kind in python_decl:
 137     for fullname in python_decl[kind]:
 138         print(" .. auto{}:: {}".format(kind, fullname))
 139
 140 ################ And now deal with Doxygen declarations
 141 ################
 142
 143 doxy_funs = {} # {classname: {func_name: [args]} }
 144 doxy_vars = {} # {classname: [names]}
 145 doxy_type = {} # {classname: [names]}
 146
 147 # find the declarations in the XML files
 148 for arg in xml_files:
 149     if arg[-4:] != '.xml':
 150         print("Argument '{}' does not end with '.xml'".format(arg))
 151         continue
 152     #print("Parse file {}".format(arg))
 153     tree = ET.parse(arg)
 154     for elem in tree.findall(".//compounddef"):
 155         if elem.attrib["kind"] == "class":
 156             if elem.attrib["prot"] != "public":
 157                 continue
 158             if "compoundname" in elem:
 159                 raise Exception("Compound {} has no 'compoundname' child tag.".format(elem))
 160             compoundname = elem.find("compoundname").text
 161             #print("compoundname {}".format(compoundname))
 162         elif elem.attrib["kind"] == "file":
 163             compoundname = ""
 164         elif elem.attrib["kind"] == "namespace":
 165             compoundname = elem.find("compoundname").text
 166         else:
 167             print("Element {} is of kind {}".format(elem.attrib["id"], elem.attrib["kind"]))
 168
 169         for member in elem.findall('.//memberdef'):
 170             if member.attrib["prot"] != "public":
 171                 continue
 172             kind = member.attrib["kind"]
 173             name = member.find("name").text
 174             #print("kind:{} compoundname:{} name:{}".format( kind,compoundname, name))
 175             if kind == "variable":
 176                 if compoundname not in doxy_vars:
 177                     doxy_vars[compoundname] = []
 178                 doxy_vars[compoundname].append(name)
 179             elif kind == "function":
 180                 args = member.find('argsstring').text
 181                 args = re.sub(r'\)[^)]*$', ')', args) # ignore what's after the parameters (eg, '=0' or ' const')
 182
 183                 if compoundname not in doxy_funs:
 184                     doxy_funs[compoundname] = {}
 185                 if name not in doxy_funs[compoundname]:
 186                     doxy_funs[compoundname][name] = []
 187                 doxy_funs[compoundname][name].append(args)
 188             elif kind == "typedef":
 189                 if compoundname not in doxy_type:
 190                     doxy_type[compoundname] = []
 191                 doxy_type[compoundname].append(name)
 192             elif kind == "friend":
 193                 pass # Ignore friendship
 194             else:
 195                 print("member {}::{} is of kind {}".format(compoundname, name, kind))
 196
 197 # Forget about the declarations that are done in the RST
 198 with os.popen('grep doxygenfunction:: find-missing.ignore source/*rst|sed \'s/^.*doxygenfunction:: //\'|sed \'s/ *const//\'') as pse:
 199     for line in (l.strip() for l in pse):
 200         (klass, obj, args) = (None, None, None)
 201         if "(" in line:
 202             (line, args) = line.split('(', 1)
 203             args = "({}".format(args)
 204         if '::' in line:
 205             (klass, obj) = line.rsplit('::', 1)
 206         else:
 207             (klass, obj) = ("", line)
 208
 209         if klass not in doxy_funs:
 210             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 211             continue
 212         if obj not in doxy_funs[klass]:
 213             print("Warning: Object '{}' documented but not found in '{}'".format(line, klass))
 214 #            for obj in doxy_funs[klass]:
 215 #                print("  found: {}::{}".format(klass, obj))
 216         elif len(doxy_funs[klass][obj]) == 1:
 217             del doxy_funs[klass][obj]
 218         elif args not in doxy_funs[klass][obj]:
 219             print("Warning: Function {}{} not found in {}".format(obj, args, klass))
 220         else:
 221             #print("Found {} in {}".format(line, klass))
 222             doxy_funs[klass][obj].remove(args)
 223             if len(doxy_funs[klass][obj]) == 0:
 224                 del doxy_funs[klass][obj]
 225 with os.popen('grep doxygenvariable:: find-missing.ignore source/*rst|sed \'s/^.*doxygenvariable:: //\'') as pse:
 226     for line in (l.strip() for l in pse):
 227         (klass, var) = line.rsplit('::', 1)
 228
 229         if klass not in doxy_vars:
 230             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 231             continue
 232         if var not in doxy_vars[klass]:
 233             print("Warning: Object {} documented but not found in '{}'".format(line, klass))
 234         else:
 235 #            print("Found {} in {}".format(line, klass))
 236             doxy_vars[klass].remove(var)
 237             if len(doxy_vars[klass]) == 0:
 238                 del doxy_vars[klass]
 239 with os.popen('grep doxygentypedef:: find-missing.ignore source/*rst|sed \'s/^.*doxygentypedef:: //\'') as pse:
 240     for line in (l.strip() for l in pse):
 241         if '::' in line:
 242             (klass, typ) = line.rsplit('::', 1)
 243         else:
 244             (klass, typ) = ('', line)
 245
 246         if klass not in doxy_type:
 247             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 248             continue
 249         if typ not in doxy_type[klass]:
 250             print("Warning: Type {} documented but not found in '{}'".format(line, klass))
 251         else:
 252 #            print("Found {} in {}".format(line, klass))
 253             doxy_type[klass].remove(typ)
 254             if len(doxy_type[klass]) == 0:
 255                 del doxy_type[klass]
 256
 257 # Dump the undocumented Doxygen declarations
 258 for obj in sorted(doxy_funs):
 259     for meth in sorted(doxy_funs[obj]):
 260         for args in sorted(doxy_funs[obj][meth]):
 261             if obj == '':
 262                 print(".. doxygenfunction:: {}{}".format(meth, args))
 263             else:
 264                 print(".. doxygenfunction:: {}::{}{}".format(obj, meth, args))
 265
 266 for obj in doxy_vars:
 267     for meth in sorted(doxy_vars[obj]):
 268         print(".. doxygenvariable:: {}::{}".format(obj, meth))
 269
 270 for obj in doxy_type:
 271     for meth in sorted(doxy_type[obj]):
 272         if obj == '':
 273             print(".. doxygentypedef:: {}".format(meth))
 274         else:
 275             print(".. doxygentypedef:: {}::{}".format(obj, meth))