docs/find-missing.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Copyright (c) 2019-2023. The SimGrid Team. All rights reserved.
   5
   6 # This program is free software; you can redistribute it and/or modify it
   7 # under the terms of the license (GNU LGPL) which comes with this package.
   8
   9 """
  10 Search for symbols documented in both the XML files produced by Doxygen and the python modules,
  11 but not documented with breathe in the RST files.
  12
  13 This script is tailored to SimGrid own needs.
  14
  15 If you are missing some dependencies, try:  pip3 install --requirement docs/requirements.txt
  16 """
  17
  18 import os
  19 import re
  20 import sys
  21 import xml.etree.ElementTree as ET
  22 import inspect
  23
  24 xml_files = [
  25     'build/xml/classsimgrid_1_1s4u_1_1Activity.xml',
  26     'build/xml/classsimgrid_1_1s4u_1_1Actor.xml',
  27     'build/xml/classsimgrid_1_1s4u_1_1Barrier.xml',
  28     'build/xml/classsimgrid_1_1s4u_1_1Comm.xml',
  29     'build/xml/classsimgrid_1_1s4u_1_1ConditionVariable.xml',
  30     'build/xml/classsimgrid_1_1s4u_1_1Disk.xml',
  31     'build/xml/classsimgrid_1_1s4u_1_1Engine.xml',
  32     'build/xml/classsimgrid_1_1s4u_1_1Exec.xml',
  33     'build/xml/classsimgrid_1_1s4u_1_1Host.xml',
  34     'build/xml/classsimgrid_1_1s4u_1_1Io.xml',
  35     'build/xml/classsimgrid_1_1s4u_1_1Link.xml',
  36     'build/xml/classsimgrid_1_1s4u_1_1Mailbox.xml',
  37     'build/xml/classsimgrid_1_1s4u_1_1Mutex.xml',
  38     'build/xml/classsimgrid_1_1s4u_1_1NetZone.xml',
  39     'build/xml/classsimgrid_1_1s4u_1_1Semaphore.xml',
  40     'build/xml/classsimgrid_1_1s4u_1_1VirtualMachine.xml',
  41     'build/xml/classsimgrid_1_1xbt_1_1signal_3_01R_07P_8_8_8_08_4.xml',
  42     'build/xml/namespacesimgrid_1_1s4u_1_1this__actor.xml',
  43     'build/xml/actor_8h.xml',
  44     'build/xml/barrier_8h.xml',
  45     'build/xml/cond_8h.xml',
  46     'build/xml/engine_8h.xml',
  47     'build/xml/forward_8h.xml',
  48     'build/xml/host_8h.xml',
  49     'build/xml/link_8h.xml',
  50     'build/xml/mailbox_8h.xml',
  51     'build/xml/mutex_8h.xml',
  52     'build/xml/semaphore_8h.xml',
  53     'build/xml/vm_8h.xml',
  54     'build/xml/zone_8h.xml'
  55 ]
  56
  57 python_modules = [
  58     'simgrid'
  59 ]
  60 python_ignore = [
  61     'simgrid.ActorKilled'
  62 ]
  63
  64
  65 ############ Search the python elements in the source, and report undocumented ones
  66 ############
  67
  68 # data structure in which we store the declaration we find
  69 python_decl = {}
  70
  71 def handle_python_module(fullname, englobing, elm):
  72     """Recursive function exploring the python modules."""
  73
  74     def found_decl(kind, obj):
  75         """Helper function that add an object in the python_decl data structure"""
  76         if kind not in python_decl:
  77             python_decl[kind] = []
  78         python_decl[kind].append(obj)
  79
  80
  81     if fullname in python_ignore:
  82         print("Ignore Python symbol '{}' as requested.".format(fullname))
  83         return
  84
  85     if inspect.isroutine(elm) and inspect.isclass(englobing):
  86         found_decl("method", fullname)
  87 #        print('.. automethod:: {}'.format(fullname))
  88     elif inspect.isroutine(elm) and (not inspect.isclass(englobing)):
  89         found_decl("function", fullname)
  90 #        print('.. autofunction:: {}'.format(fullname))
  91     elif inspect.isdatadescriptor(elm):
  92         found_decl("attribute", fullname)
  93 #        print('.. autoattribute:: {}'.format(fullname))
  94     elif isinstance(elm, (int, str)): # We do have such a data, directly in the SimGrid top module
  95         found_decl("data", fullname)
  96 #        print('.. autodata:: {}'.format(fullname))
  97     elif inspect.ismodule(elm) or inspect.isclass(elm):
  98         for name, data in inspect.getmembers(elm):
  99             if name.startswith('__'):
 100                 continue
 101 #            print("Recurse on {}.{}".format(fullname, name))
 102             handle_python_module("{}.{}".format(fullname, name), elm, data)
 103     elif inspect.isclass(type(elm)):
 104         found_decl("enumvalue", fullname)
 105         print('.. autoenumvalue:: {}'.format(fullname))
 106     else:
 107         print('UNHANDLED TYPE {} : {!r} Type: {} Englobing: {} str: {} Members: \n{}\n'.format(fullname, elm, type(elm), englobing, str(elm), inspect.getmembers(elm)))
 108
 109 # Start the recursion on the provided Python modules
 110 for name in python_modules:
 111     try:
 112         module = __import__(name)
 113     except Exception:
 114         if os.path.exists("../lib") and "../lib" not in sys.path:
 115             print("Adding ../lib to PYTHONPATH as {} cannot be imported".format(name))
 116             sys.path.append("../lib")
 117             try:
 118                 module = __import__(name)
 119             except Exception:
 120                 print("Cannot import {}, even with PYTHONPATH=../lib".format(name))
 121                 sys.exit(1)
 122         else:
 123             print("Cannot import {}".format(name))
 124             sys.exit(1)
 125     for sub in dir(module):
 126         if sub[0] == '_':
 127             continue
 128         handle_python_module("{}.{}".format(name, sub), module, getattr(module, sub))
 129
 130 # Forget about the python declarations that were actually done in the RST
 131 for kind in python_decl:
 132     with os.popen('grep \'[[:blank:]]*auto{}::\' source/*rst|sed \'s/^.*auto{}:: //\''.format(kind, kind)) as pse:
 133         for fullname in (l.strip() for l in pse):
 134             if fullname not in python_decl[kind]:
 135                 print("Warning: {} documented but declaration not found in python.".format(fullname))
 136             else:
 137                 python_decl[kind].remove(fullname)
 138 # Dump the missing ones
 139 for kind in python_decl:
 140     for fullname in python_decl[kind]:
 141         print(" .. auto{}:: {}".format(kind, fullname))
 142
 143 ################ And now deal with Doxygen declarations
 144 ################
 145
 146 doxy_funs = {} # {classname: {func_name: [args]} }
 147 doxy_vars = {} # {classname: [names]}
 148 doxy_type = {} # {classname: [names]}
 149
 150 # find the declarations in the XML files
 151 for arg in xml_files:
 152     if arg[-4:] != '.xml':
 153         print("Argument '{}' does not end with '.xml'".format(arg))
 154         continue
 155     #print("Parse file {}".format(arg))
 156     tree = ET.parse(arg)
 157     for elem in tree.findall(".//compounddef"):
 158         if elem.attrib["kind"] == "class":
 159             if elem.attrib["prot"] != "public":
 160                 continue
 161             if "compoundname" in elem:
 162                 raise Exception("Compound {} has no 'compoundname' child tag.".format(elem))
 163             compoundname = elem.find("compoundname").text
 164             #print("compoundname {}".format(compoundname))
 165         elif elem.attrib["kind"] == "file":
 166             compoundname = ""
 167         elif elem.attrib["kind"] == "namespace":
 168             compoundname = elem.find("compoundname").text
 169         else:
 170             print("Element {} is of kind {}".format(elem.attrib["id"], elem.attrib["kind"]))
 171
 172         for member in elem.findall('.//memberdef'):
 173             if member.attrib["prot"] != "public":
 174                 continue
 175             kind = member.attrib["kind"]
 176             name = member.find("name").text
 177             #print("kind:{} compoundname:{} name:{}".format( kind,compoundname, name))
 178             if kind == "variable":
 179                 if compoundname not in doxy_vars:
 180                     doxy_vars[compoundname] = []
 181                 doxy_vars[compoundname].append(name)
 182             elif kind == "function":
 183                 args = member.find('argsstring').text
 184                 args = re.sub(r'\)[^)]*$', ')', args) # ignore what's after the parameters (eg, '=0' or ' const')
 185
 186                 if compoundname not in doxy_funs:
 187                     doxy_funs[compoundname] = {}
 188                 if name not in doxy_funs[compoundname]:
 189                     doxy_funs[compoundname][name] = []
 190                 doxy_funs[compoundname][name].append(args)
 191             elif kind == "typedef":
 192                 if compoundname not in doxy_type:
 193                     doxy_type[compoundname] = []
 194                 doxy_type[compoundname].append(name)
 195             elif kind == "friend":
 196                 pass # Ignore friendship
 197             else:
 198                 print("member {}::{} is of kind {}".format(compoundname, name, kind))
 199
 200 # Forget about the declarations that are done in the RST
 201 with os.popen('grep doxygenfunction:: find-missing.ignore source/*rst|sed \'s/^.*doxygenfunction:: //\'|sed \'s/ *const//\'') as pse:
 202     for line in (l.strip() for l in pse):
 203         (klass, obj, args) = (None, None, None)
 204         if "(" in line:
 205             (line, args) = line.split('(', 1)
 206             args = "({}".format(args)
 207         if '::' in line:
 208             (klass, obj) = line.rsplit('::', 1)
 209         else:
 210             (klass, obj) = ("", line)
 211
 212         if klass not in doxy_funs:
 213             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 214             continue
 215         if obj not in doxy_funs[klass]:
 216             print("Warning: Object '{}' documented but not found in '{}'".format(line, klass))
 217 #            for obj in doxy_funs[klass]:
 218 #                print("  found: {}::{}".format(klass, obj))
 219         elif len(doxy_funs[klass][obj]) == 1:
 220             del doxy_funs[klass][obj]
 221         elif args not in doxy_funs[klass][obj]:
 222             print("Warning: Function {}{} not found in {}".format(obj, args, klass))
 223         else:
 224             #print("Found {} in {}".format(line, klass))
 225             doxy_funs[klass][obj].remove(args)
 226             if len(doxy_funs[klass][obj]) == 0:
 227                 del doxy_funs[klass][obj]
 228 with os.popen('grep doxygenvariable:: find-missing.ignore source/*rst|sed \'s/^.*doxygenvariable:: //\'') as pse:
 229     for line in (l.strip() for l in pse):
 230         (klass, var) = line.rsplit('::', 1)
 231
 232         if klass not in doxy_vars:
 233             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 234             continue
 235         if var not in doxy_vars[klass]:
 236             print("Warning: Object {} documented but not found in '{}'".format(line, klass))
 237         else:
 238 #            print("Found {} in {}".format(line, klass))
 239             doxy_vars[klass].remove(var)
 240             if len(doxy_vars[klass]) == 0:
 241                 del doxy_vars[klass]
 242 with os.popen('grep doxygentypedef:: find-missing.ignore source/*rst|sed \'s/^.*doxygentypedef:: //\'') as pse:
 243     for line in (l.strip() for l in pse):
 244         if '::' in line:
 245             (klass, typ) = line.rsplit('::', 1)
 246         else:
 247             (klass, typ) = ('', line)
 248
 249         if klass not in doxy_type:
 250             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 251             continue
 252         if typ not in doxy_type[klass]:
 253             print("Warning: Type {} documented but not found in '{}'".format(line, klass))
 254         else:
 255 #            print("Found {} in {}".format(line, klass))
 256             doxy_type[klass].remove(typ)
 257             if len(doxy_type[klass]) == 0:
 258                 del doxy_type[klass]
 259
 260 # Dump the undocumented Doxygen declarations
 261 for obj in sorted(doxy_funs):
 262     for meth in sorted(doxy_funs[obj]):
 263         for args in sorted(doxy_funs[obj][meth]):
 264             if obj == '':
 265                 print(".. doxygenfunction:: {}{}".format(meth, args))
 266             else:
 267                 print(".. doxygenfunction:: {}::{}{}".format(obj, meth, args))
 268
 269 for obj in doxy_vars:
 270     for meth in sorted(doxy_vars[obj]):
 271         print(".. doxygenvariable:: {}::{}".format(obj, meth))
 272
 273 for obj in doxy_type:
 274     for meth in sorted(doxy_type[obj]):
 275         if obj == '':
 276             print(".. doxygentypedef:: {}".format(meth))
 277         else:
 278             print(".. doxygentypedef:: {}::{}".format(obj, meth))