498 lines
21 KiB
Python
498 lines
21 KiB
Python
import os, sys, glob, re
|
|
|
|
sys.path.append("../modules/python/src2/")
|
|
sys.path.append("../modules/java/")
|
|
|
|
import hdr_parser as hp
|
|
import rst_parser as rp
|
|
|
|
rp.show_warnings = False
|
|
rp.show_errors = False
|
|
|
|
allmodules = rp.allmodules
|
|
DOCUMENTED_MARKER = "verified"
|
|
|
|
ERROR_001_NOTACLASS = 1
|
|
ERROR_002_NOTASTRUCT = 2
|
|
ERROR_003_INCORRECTBASE = 3
|
|
ERROR_004_MISSEDNAMESPACE = 4
|
|
ERROR_005_MISSINGPYFUNC = 5
|
|
ERROR_006_INVALIDPYOLDDOC = 6
|
|
ERROR_007_INVALIDPYDOC = 7
|
|
ERROR_008_CFUNCISNOTGLOBAL = 8
|
|
ERROR_009_OVERLOADNOTFOUND = 9
|
|
ERROR_010_UNKNOWNCLASS = 10
|
|
ERROR_011_UNKNOWNFUNC = 11
|
|
|
|
do_python_crosscheck = True
|
|
errors_disabled = [ERROR_004_MISSEDNAMESPACE]
|
|
|
|
doc_signatures_whitelist = [
|
|
# templates
|
|
"Matx", "Vec", "SparseMat_", "Scalar_", "Mat_", "Ptr", "Size_", "Point_", "Rect_", "Point3_",
|
|
"DataType", "detail::RotationWarperBase", "flann::Index_", "CalonderDescriptorExtractor",
|
|
"gpu::DevMem2D_", "gpu::PtrStep_", "gpu::PtrElemStep_",
|
|
# black boxes
|
|
"CvArr", "CvFileStorage",
|
|
# other
|
|
"InputArray", "OutputArray",
|
|
]
|
|
|
|
defines = ["cvGraphEdgeIdx", "cvFree", "CV_Assert", "cvSqrt", "cvGetGraphVtx", "cvGraphVtxIdx",
|
|
"cvCaptureFromFile", "cvCaptureFromCAM", "cvCalcBackProjectPatch", "cvCalcBackProject",
|
|
"cvGetHistValue_1D", "cvGetHistValue_2D", "cvGetHistValue_3D", "cvGetHistValue_nD",
|
|
"cvQueryHistValue_1D", "cvQueryHistValue_2D", "cvQueryHistValue_3D", "cvQueryHistValue_nD",
|
|
# not a real function but behaves as function
|
|
"Mat::size",
|
|
# ugly "virtual" functions from ml module
|
|
"CvStatModel::train", "CvStatModel::predict",
|
|
# TODO:
|
|
"cvExtractSURF"
|
|
]
|
|
|
|
synonims = {
|
|
"StarDetector" : ["StarFeatureDetector"],
|
|
"MSER" : ["MserFeatureDetector"],
|
|
"GFTTDetector" : ["GoodFeaturesToTrackDetector"],
|
|
"cvCaptureFromFile" : ["cvCreateFileCapture"],
|
|
"cvCaptureFromCAM" : ["cvCreateCameraCapture"],
|
|
"cvCalcArrBackProjectPatch" : ["cvCalcBackProjectPatch"],
|
|
"cvCalcArrBackProject" : ["cvCalcBackProject"],
|
|
"InputArray" : ["_InputArray"],
|
|
"OutputArray" : ["_OutputArray"],
|
|
}
|
|
|
|
if do_python_crosscheck:
|
|
try:
|
|
import cv2
|
|
except ImportError:
|
|
print "Could not load cv2"
|
|
do_python_crosscheck = False
|
|
|
|
def get_cv2_object(name):
|
|
if name.startswith("cv2."):
|
|
name = name[4:]
|
|
if name.startswith("cv."):
|
|
name = name[3:]
|
|
if name == "Algorithm":
|
|
return cv2.Algorithm__create("Feature2D.ORB"), name
|
|
elif name == "FeatureDetector":
|
|
return cv2.FeatureDetector_create("ORB"), name
|
|
elif name == "DescriptorExtractor":
|
|
return cv2.DescriptorExtractor_create("ORB"), name
|
|
elif name == "BackgroundSubtractor":
|
|
return cv2.BackgroundSubtractorMOG(), name
|
|
elif name == "StatModel":
|
|
return cv2.KNearest(), name
|
|
else:
|
|
return getattr(cv2, name)(), name
|
|
|
|
def compareSignatures(f, s):
|
|
# function names
|
|
if f[0] != s[0]:
|
|
return False, "name mismatch"
|
|
# return type
|
|
stype = (s[1] or "void")
|
|
ftype = f[1]
|
|
stype = re.sub(r"\b(cv|std)::", "", stype)
|
|
if ftype:
|
|
ftype = re.sub(r"\b(cv|std)::", "", ftype)
|
|
if ftype and ftype != stype:
|
|
return False, "return type mismatch"
|
|
if ("\C" in f[2]) ^ ("\C" in s[2]):
|
|
return False, "const qualifier mismatch"
|
|
if ("\S" in f[2]) ^ ("\S" in s[2]):
|
|
return False, "static qualifier mismatch"
|
|
if ("\V" in f[2]) ^ ("\V" in s[2]):
|
|
return False, "virtual qualifier mismatch"
|
|
if ("\A" in f[2]) ^ ("\A" in s[2]):
|
|
return False, "abstract qualifier mismatch"
|
|
if len(f[3]) != len(s[3]):
|
|
return False, "different number of arguments"
|
|
for idx, arg in enumerate(zip(f[3], s[3])):
|
|
farg = arg[0]
|
|
sarg = arg[1]
|
|
ftype = re.sub(r"\b(cv|std)::", "", (farg[0] or ""))
|
|
stype = re.sub(r"\b(cv|std)::", "", (sarg[0] or ""))
|
|
if ftype != stype:
|
|
return False, "type of argument #" + str(idx+1) + " mismatch"
|
|
fname = farg[1] or "arg" + str(idx)
|
|
sname = sarg[1] or "arg" + str(idx)
|
|
if fname != sname:
|
|
return False, "name of argument #" + str(idx+1) + " mismatch"
|
|
fdef = re.sub(r"\b(cv|std)::", "", (farg[2] or ""))
|
|
sdef = re.sub(r"\b(cv|std)::", "", (sarg[2] or ""))
|
|
if fdef != sdef:
|
|
return False, "default value of argument #" + str(idx+1) + " mismatch"
|
|
return True, "match"
|
|
|
|
def formatSignature(s):
|
|
_str = ""
|
|
if "/V" in s[2]:
|
|
_str += "virtual "
|
|
if "/S" in s[2]:
|
|
_str += "static "
|
|
if s[1]:
|
|
_str += s[1] + " "
|
|
else:
|
|
if not bool(re.match(r"(\w+\.)*(?P<cls>\w+)\.(?P=cls)", s[0])):
|
|
_str += "void "
|
|
if s[0].startswith("cv."):
|
|
_str += s[0][3:].replace(".", "::")
|
|
else:
|
|
_str += s[0].replace(".", "::")
|
|
if len(s[3]) == 0:
|
|
_str += "()"
|
|
else:
|
|
_str += "( "
|
|
for idx, arg in enumerate(s[3]):
|
|
if idx > 0:
|
|
_str += ", "
|
|
argtype = re.sub(r"\bcv::", "", arg[0])
|
|
bidx = argtype.find('[')
|
|
if bidx < 0:
|
|
_str += argtype + " "
|
|
else:
|
|
_srt += argtype[:bidx]
|
|
if arg[1]:
|
|
_str += arg[1]
|
|
else:
|
|
_str += "arg" + str(idx)
|
|
if bidx >= 0:
|
|
_str += argtype[bidx:]
|
|
if arg[2]:
|
|
_str += "=" + re.sub(r"\bcv::", "", arg[2])
|
|
_str += " )"
|
|
if "/C" in s[2]:
|
|
_str += " const"
|
|
if "/A" in s[2]:
|
|
_str += " = 0"
|
|
return _str
|
|
|
|
|
|
def logerror(code, message, doc = None):
|
|
if code in errors_disabled:
|
|
return
|
|
if doc:
|
|
print doc["file"] + ":" + str(doc["line"]),
|
|
print "error %03d: %s" % (code, message)
|
|
#print
|
|
|
|
def process_module(module, path):
|
|
hppparser = hp.CppHeaderParser()
|
|
rstparser = rp.RstParser(hppparser)
|
|
|
|
rstparser.parse(module, path)
|
|
rst = rstparser.definitions
|
|
|
|
hdrlist = glob.glob(os.path.join(path, "include", "opencv2", module, "*.h*"))
|
|
hdrlist.extend(glob.glob(os.path.join(path, "include", "opencv2", module, "detail", "*.h*")))
|
|
|
|
if module == "gpu":
|
|
hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "devmem2d.hpp"))
|
|
hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "gpumat.hpp"))
|
|
|
|
decls = []
|
|
for hname in hdrlist:
|
|
if not "ts_gtest.h" in hname:
|
|
decls += hppparser.parse(hname, wmode=False)
|
|
|
|
funcs = []
|
|
# not really needed to hardcode all the namespaces. Normally all they are collected automatically
|
|
namespaces = ['cv', 'cv.gpu', 'cvflann', 'cvflann.anyimpl', 'cvflann.lsh', 'cv.flann', 'cv.linemod', 'cv.detail', 'cvtest', 'perf', 'cv.videostab']
|
|
classes = []
|
|
structs = []
|
|
|
|
# collect namespaces and classes/structs
|
|
for decl in decls:
|
|
if decl[0].startswith("const"):
|
|
pass
|
|
elif decl[0].startswith("class") or decl[0].startswith("struct"):
|
|
if decl[0][0] == 'c':
|
|
classes.append(decl)
|
|
else:
|
|
structs.append(decl)
|
|
dotIdx = decl[0].rfind('.')
|
|
if dotIdx > 0:
|
|
namespace = decl[0][decl[0].find(' ')+1:dotIdx]
|
|
if not [c for c in classes if c[0].endswith(namespace)] and not [s for s in structs if s[0].endswith(namespace)]:
|
|
if namespace not in namespaces:
|
|
namespaces.append(namespace)
|
|
else:
|
|
funcs.append(decl)
|
|
|
|
clsnamespaces = []
|
|
# process classes
|
|
for cl in classes:
|
|
name = cl[0][cl[0].find(' ')+1:]
|
|
if name.find('.') < 0 and not name.startswith("Cv"):
|
|
logerror(ERROR_004_MISSEDNAMESPACE, "class " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
|
|
clsnamespaces.append(name)
|
|
if do_python_crosscheck and not name.startswith("cv.") and name.startswith("Cv"):
|
|
clsnamespaces.append("cv." + name[2:])
|
|
if name.startswith("cv."):
|
|
name = name[3:]
|
|
name = name.replace(".", "::")
|
|
sns = synonims.get(name, [])
|
|
sns.append(name)
|
|
for name in sns:
|
|
doc = rst.get(name)
|
|
if not doc:
|
|
#TODO: class is not documented
|
|
continue
|
|
doc[DOCUMENTED_MARKER] = True
|
|
# verify class marker
|
|
if not doc.get("isclass"):
|
|
logerror(ERROR_001_NOTACLASS, "class " + name + " is not marked as \"class\" in documentation", doc)
|
|
else:
|
|
# verify base
|
|
signature = doc.get("class", "")
|
|
signature = signature.replace(" public ", " ")
|
|
namespaceIdx = signature.rfind("::")
|
|
|
|
signature = ("class " + signature).strip()
|
|
hdrsignature = ("class " + name + " " + cl[1]).replace(".", "::").replace("cv::","").strip()
|
|
if signature != hdrsignature:
|
|
logerror(ERROR_003_INCORRECTBASE, "invalid base class documentation\ndocumented: " + signature + "\nactual: " + hdrsignature, doc)
|
|
|
|
# process structs
|
|
for st in structs:
|
|
name = st[0][st[0].find(' ')+1:]
|
|
if name.find('.') < 0 and not name.startswith("Cv"):
|
|
logerror(ERROR_004_MISSEDNAMESPACE, "struct " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
|
|
clsnamespaces.append(name)
|
|
if name.startswith("cv."):
|
|
name = name[3:]
|
|
name = name.replace(".", "::")
|
|
doc = rst.get(name)
|
|
if not doc:
|
|
#TODO: struct is not documented
|
|
continue
|
|
doc[DOCUMENTED_MARKER] = True
|
|
# verify struct marker
|
|
if not doc.get("isstruct"):
|
|
logerror(ERROR_002_NOTASTRUCT, "struct " + name + " is not marked as \"struct\" in documentation", doc)
|
|
else:
|
|
# verify base
|
|
signature = doc.get("class", "")
|
|
signature = signature.replace(", public ", " ").replace(" public ", " ")
|
|
signature = signature.replace(", protected ", " ").replace(" protected ", " ")
|
|
signature = signature.replace(", private ", " ").replace(" private ", " ")
|
|
signature = ("struct " + signature).strip()
|
|
hdrsignature = (st[0] + " " + st[1]).replace("struct cv.", "struct ").replace(".", "::").strip()
|
|
if signature != hdrsignature:
|
|
logerror(ERROR_003_INCORRECTBASE, "invalid base struct documentation\ndocumented: " + signature + "\nactual: " + hdrsignature, doc)
|
|
print st, doc
|
|
|
|
# process functions and methods
|
|
flookup = {}
|
|
for fn in funcs:
|
|
name = fn[0]
|
|
parent = None
|
|
namespace = None
|
|
for cl in clsnamespaces:
|
|
if name.startswith(cl + "."):
|
|
if cl.startswith(parent or ""):
|
|
parent = cl
|
|
if parent:
|
|
name = name[len(parent) + 1:]
|
|
for nm in namespaces:
|
|
if parent.startswith(nm + "."):
|
|
if nm.startswith(namespace or ""):
|
|
namespace = nm
|
|
if namespace:
|
|
parent = parent[len(namespace) + 1:]
|
|
else:
|
|
for nm in namespaces:
|
|
if name.startswith(nm + "."):
|
|
if nm.startswith(namespace or ""):
|
|
namespace = nm
|
|
if namespace:
|
|
name = name[len(namespace) + 1:]
|
|
#print namespace, parent, name, fn[0]
|
|
if not namespace and not parent and not name.startswith("cv") and not name.startswith("CV_"):
|
|
logerror(ERROR_004_MISSEDNAMESPACE, "function " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
|
|
else:
|
|
fdescr = (namespace, parent, name, fn)
|
|
flookup_entry = flookup.get(fn[0], [])
|
|
flookup_entry.append(fdescr)
|
|
flookup[fn[0]] = flookup_entry
|
|
|
|
if do_python_crosscheck:
|
|
for name, doc in rst.iteritems():
|
|
decls = doc.get("decls")
|
|
if not decls:
|
|
continue
|
|
for signature in decls:
|
|
if signature[0] == "Python1":
|
|
pname = signature[1][:signature[1].find('(')]
|
|
try:
|
|
fn = getattr(cv2.cv, pname[3:])
|
|
docstr = "cv." + fn.__doc__
|
|
except AttributeError:
|
|
logerror(ERROR_005_MISSINGPYFUNC, "could not load documented function: cv2." + pname, doc)
|
|
continue
|
|
docstring = docstr
|
|
sign = signature[1]
|
|
signature.append(DOCUMENTED_MARKER)
|
|
# convert old signature to pydoc style
|
|
if docstring.endswith("*"):
|
|
docstring = docstring[:-1]
|
|
s = None
|
|
while s != sign:
|
|
s = sign
|
|
sign = re.sub(r"^(.*\(.*)\(.*?\)(.*\) *->)", "\\1_\\2", sign)
|
|
s = None
|
|
while s != sign:
|
|
s = sign
|
|
sign = re.sub(r"\s*,\s*([^,]+)\s*=\s*[^,]+\s*(( \[.*\])?)\)", " [, \\1\\2])", sign)
|
|
sign = re.sub(r"\(\s*([^,]+)\s*=\s*[^,]+\s*(( \[.*\])?)\)", "([\\1\\2])", sign)
|
|
|
|
sign = re.sub(r"\)\s*->\s*", ") -> ", sign)
|
|
sign = sign.replace("-> convexHull", "-> CvSeq")
|
|
sign = sign.replace("-> lines", "-> CvSeq")
|
|
sign = sign.replace("-> boundingRects", "-> CvSeq")
|
|
sign = sign.replace("-> contours", "-> CvSeq")
|
|
sign = sign.replace("-> retval", "-> int")
|
|
sign = sign.replace("-> detectedObjects", "-> CvSeqOfCvAvgComp")
|
|
|
|
def retvalRplace(match):
|
|
m = match.group(1)
|
|
m = m.replace("CvScalar", "scalar")
|
|
m = m.replace("CvMemStorage", "memstorage")
|
|
m = m.replace("ROIplImage", "image")
|
|
m = m.replace("IplImage", "image")
|
|
m = m.replace("ROCvMat", "mat")
|
|
m = m.replace("CvMat", "mat")
|
|
m = m.replace("double", "float")
|
|
m = m.replace("CvSubdiv2DPoint", "point")
|
|
m = m.replace("CvBox2D", "Box2D")
|
|
m = m.replace("IplConvKernel", "kernel")
|
|
m = m.replace("CvHistogram", "hist")
|
|
m = m.replace("CvSize", "width,height")
|
|
m = m.replace("cvmatnd", "matND")
|
|
m = m.replace("CvSeqOfCvConvexityDefect", "convexityDefects")
|
|
mm = m.split(',')
|
|
if len(mm) > 1:
|
|
return "(" + ", ".join(mm) + ")"
|
|
else:
|
|
return m
|
|
|
|
docstring = re.sub(r"(?<=-> )(.*)$", retvalRplace, docstring)
|
|
docstring = docstring.replace("( [, ", "([")
|
|
|
|
if sign != docstring:
|
|
logerror(ERROR_006_INVALIDPYOLDDOC, "old-style documentation differs from pydoc\npydoc: " + docstring + "\nfixup: " + sign + "\ncvdoc: " + signature[1], doc)
|
|
elif signature[0] == "Python2":
|
|
pname = signature[1][4:signature[1].find('(')]
|
|
cvname = "cv." + pname
|
|
parent = None
|
|
for cl in clsnamespaces:
|
|
if cvname.startswith(cl + "."):
|
|
if cl.startswith(parent or ""):
|
|
parent = cl
|
|
try:
|
|
if parent:
|
|
instance, clsname = get_cv2_object(parent)
|
|
fn = getattr(instance, cvname[len(parent)+1:])
|
|
docstr = fn.__doc__
|
|
docprefix = "cv2." + clsname + "."
|
|
else:
|
|
fn = getattr(cv2, pname)
|
|
docstr = fn.__doc__
|
|
docprefix = "cv2."
|
|
except AttributeError:
|
|
if parent:
|
|
logerror(ERROR_005_MISSINGPYFUNC, "could not load documented member of " + parent + " class: cv2." + pname, doc)
|
|
else:
|
|
logerror(ERROR_005_MISSINGPYFUNC, "could not load documented function cv2." + pname, doc)
|
|
signature.append(DOCUMENTED_MARKER) # stop subsequent errors
|
|
continue
|
|
docstrings = [docprefix + s.replace("([, ", "([") for s in docstr.split(" or ")]
|
|
if not signature[1] in docstrings:
|
|
pydocs = "\npydoc: ".join(docstrings)
|
|
logerror(ERROR_007_INVALIDPYDOC, "documentation differs from pydoc\npydoc: " + pydocs + "\ncvdoc: " + signature[1], doc)
|
|
signature.append(DOCUMENTED_MARKER)
|
|
|
|
# verify C/C++ signatures
|
|
for name, doc in rst.iteritems():
|
|
decls = doc.get("decls")
|
|
if not decls:
|
|
continue
|
|
for signature in decls:
|
|
if signature[0] == "C" or signature[0] == "C++":
|
|
if "template" in (signature[2][1] or ""):
|
|
# TODO find a way to validate templates
|
|
signature.append(DOCUMENTED_MARKER)
|
|
continue
|
|
fd = flookup.get(signature[2][0])
|
|
if not fd:
|
|
if signature[2][0].startswith("cv."):
|
|
fd = flookup.get(signature[2][0][3:])
|
|
if not fd:
|
|
continue
|
|
else:
|
|
signature[2][0] = signature[2][0][3:]
|
|
if signature[0] == "C":
|
|
ffd = [f for f in fd if not f[0] and not f[1]] # filter out C++ stuff
|
|
if not ffd:
|
|
if fd[0][1]:
|
|
logerror(ERROR_008_CFUNCISNOTGLOBAL, "function " + fd[0][2] + " is documented as C function but is actually member of " + fd[0][1] + " class", doc)
|
|
elif fd[0][0]:
|
|
logerror(ERROR_008_CFUNCISNOTGLOBAL, "function " + fd[0][2] + " is documented as C function but is actually placed in " + fd[0][0] + " namespace", doc)
|
|
fd = ffd
|
|
error = None
|
|
for f in fd:
|
|
match, error = compareSignatures(signature[2], f[3])
|
|
if match:
|
|
signature.append(DOCUMENTED_MARKER)
|
|
break
|
|
if signature[-1] != DOCUMENTED_MARKER:
|
|
candidates = "\n\t".join([formatSignature(f[3]) for f in fd])
|
|
logerror(ERROR_009_OVERLOADNOTFOUND, signature[0] + " function " + signature[2][0].replace(".","::") + " is documented but misses in headers (" + error + ").\nDocumented as:\n\t" + signature[1] + "\nCandidates are:\n\t" + candidates, doc)
|
|
signature.append(DOCUMENTED_MARKER) # to stop subsequent error on this function
|
|
|
|
# verify that all signatures was found in the library headers
|
|
for name, doc in rst.iteritems():
|
|
# if doc.get(DOCUMENTED_MARKER, False):
|
|
# continue # this class/struct was found
|
|
if not doc.get(DOCUMENTED_MARKER, False) and (doc.get("isclass", False) or doc.get("isstruct", False)):
|
|
if name in doc_signatures_whitelist:
|
|
continue
|
|
logerror(ERROR_010_UNKNOWNCLASS, "class/struct " + name + " is mentioned in documentation but is not found in OpenCV headers", doc)
|
|
for d in doc.get("decls", []):
|
|
if d[-1] != DOCUMENTED_MARKER:
|
|
if d[0] == "C" or d[0] =="C++" or (do_python_crosscheck and d[0].startswith("Python")):
|
|
if d[0][0] == 'C':
|
|
sname = d[2][0][3:].replace(".", "::")
|
|
if sname in defines:
|
|
#TODO: need to find a way to verify #define's
|
|
continue
|
|
else:
|
|
sname = d[1][:d[1].find("(")]
|
|
prefixes = [x for x in doc_signatures_whitelist if sname.startswith(x)]
|
|
if prefixes:
|
|
# TODO: member of template class
|
|
continue
|
|
logerror(ERROR_011_UNKNOWNFUNC, d[0] + " function " + sname + " is documented but is not found in OpenCV headers. It is documented as:\n\t" + d[1], doc)
|
|
# end of process_module
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) < 2:
|
|
print "Usage:\n", os.path.basename(sys.argv[0]), " <module path>"
|
|
exit(0)
|
|
|
|
modules = sys.argv[1:]
|
|
if modules[0] == "all":
|
|
modules = allmodules
|
|
|
|
for module in modules:
|
|
selfpath = os.path.dirname(os.path.abspath(sys.argv[0]))
|
|
module_path = os.path.join(selfpath, "..", "modules", module)
|
|
|
|
if not os.path.isdir(module_path):
|
|
print "Module \"" + module + "\" could not be found."
|
|
exit(1)
|
|
|
|
process_module(module, module_path)
|