diff --git a/.known-issues/doc/duplicate.conf b/.known-issues/doc/duplicate.conf deleted file mode 100644 index b738dfcb2be..00000000000 --- a/.known-issues/doc/duplicate.conf +++ /dev/null @@ -1,39 +0,0 @@ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]file_system[/\\]index.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]peripherals[/\\]dma.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]audio[/\\]dmic.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]networking[/\\]net_if.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]networking[/\\]ieee802154.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]networking[/\\]sockets.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]uuid.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]sdp.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]rfcomm.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]hci_raw.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]hci_drivers.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]gatt.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]gap.rst):(?P[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$ -^Declaration is \'.*\'\.$ diff --git a/.known-issues/doc/misc.conf b/.known-issues/doc/misc.conf deleted file mode 100644 index 6cb79467703..00000000000 --- a/.known-issues/doc/misc.conf +++ /dev/null @@ -1,14 +0,0 @@ -# multiple section 'index' -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]custom-doxygen[/\\]mainpage.md):[0-9]+: warning: multiple use of section label 'index' for main page, \(first occurrence: .*$ -# -^(?P([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]gatt.rst):(?P[0-9]+): WARNING: Error in declarator$ -^[ \t]*If.*:$ -^[ \t]*Invalid C declaration: .* \[error at [0-9]+\]$ -^[ \t]*ATOMIC_DEFINE.*$ -^[- \t]*\^$ -^[ \t]*If.*:$ -^[ \t]*Error in declarator or parameters$ -^[ \t]*Invalid C declaration: .* \[error at [0-9]+\]$ -^[ \t]*ATOMIC_DEFINE.*$ -^[- \t]*\^$ -^[ \t]*$ diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index c11fd190469..c5f025bb98a 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -97,7 +97,6 @@ set(DOXY_OUT ${CMAKE_CURRENT_BINARY_DIR}/doxygen) set(RST_OUT ${CMAKE_CURRENT_BINARY_DIR}/rst) set(DOC_LOG ${CMAKE_CURRENT_BINARY_DIR}/doc.log) set(DOXY_LOG ${CMAKE_CURRENT_BINARY_DIR}/doxy.log) -set(SPHINX_LOG ${CMAKE_CURRENT_BINARY_DIR}/sphinx.log) set(DOC_WARN ${CMAKE_CURRENT_BINARY_DIR}/doc.warnings) set(CONTENT_OUTPUTS ${CMAKE_CURRENT_BINARY_DIR}/extracted-content.txt) @@ -201,9 +200,7 @@ else() set(SEP :) endif() -set(KI_SCRIPT ${ZEPHYR_BASE}/scripts/filter-known-issues.py) set(FIX_TEX_SCRIPT ${ZEPHYR_BASE}/doc/_scripts/fix_tex.py) -set(CONF_DIR ${ZEPHYR_BASE}/.known-issues/doc) # # Generated Kconfig .rst documents @@ -330,7 +327,7 @@ set(SPHINX_BUILD_HTML_COMMAND ${CMAKE_COMMAND} -E env ZEPHYR_BASE=${ZEPHYR_BASE} ZEPHYR_BUILD=${CMAKE_CURRENT_BINARY_DIR} - ${SPHINXBUILD} -w ${SPHINX_LOG} -N -t ${DOC_TAG} -b html ${ALLSPHINXOPTS} ${RST_OUT}/doc ${SPHINX_OUTPUT_DIR_HTML}) + ${SPHINXBUILD} -W -N -t ${DOC_TAG} -b html ${ALLSPHINXOPTS} ${RST_OUT}/doc ${SPHINX_OUTPUT_DIR_HTML}) # The sphinx-html target is provided as a convenience for incremental # re-builds of content files without regenerating the entire docs @@ -355,9 +352,6 @@ add_dependencies(sphinx-html content) add_custom_target( html COMMAND ${SPHINX_BUILD_HTML_COMMAND} - # Merge the Doxygen and Sphinx logs into a single file - COMMAND ${CMAKE_COMMAND} -P ${ZEPHYR_BASE}/cmake/util/fmerge.cmake ${DOC_LOG} ${DOXY_LOG} ${SPHINX_LOG} - COMMAND ${PYTHON_EXECUTABLE} ${KI_SCRIPT} --config-dir ${CONF_DIR} --errors ${DOC_WARN} --warnings ${DOC_WARN} ${DOC_LOG} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMENT "Generating HTML documentation with ${SPHINXBUILD} ${SPHINXOPTS}" ${SPHINX_USES_TERMINAL} @@ -369,7 +363,7 @@ add_custom_target( set(SPHINX_BUILD_LATEX_COMMAND ${CMAKE_COMMAND} -E env ZEPHYR_BUILD=${CMAKE_CURRENT_BINARY_DIR} - ${SPHINXBUILD} -w ${SPHINX_LOG} -N -t ${DOC_TAG} -b latex -t svgconvert ${ALLSPHINXOPTS} ${RST_OUT}/doc ${SPHINX_OUTPUT_DIR_LATEX}) + ${SPHINXBUILD} -W -N -t ${DOC_TAG} -b latex -t svgconvert ${ALLSPHINXOPTS} ${RST_OUT}/doc ${SPHINX_OUTPUT_DIR_LATEX}) # The sphinx-latex target works similarly to sphinx-html, and carries # the same warnings. @@ -386,9 +380,6 @@ add_dependencies(sphinx-latex content) add_custom_command( OUTPUT ${SPHINX_OUTPUT_DIR_LATEX}/zephyr.tex COMMAND ${SPHINX_BUILD_LATEX_COMMAND} - # Merge the Doxygen and Sphinx logs into a single file - COMMAND ${CMAKE_COMMAND} -P ${ZEPHYR_BASE}/cmake/util/fmerge.cmake ${DOC_LOG} ${DOXY_LOG} ${SPHINX_LOG} - COMMAND ${PYTHON_EXECUTABLE} ${KI_SCRIPT} --config-dir ${CONF_DIR} --errors ${DOC_WARN} --warnings ${DOC_WARN} ${DOC_LOG} COMMAND ${PYTHON_EXECUTABLE} ${FIX_TEX_SCRIPT} ${SPHINX_OUTPUT_DIR_LATEX}/zephyr.tex WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMENT "Generating LaTeX documentation" diff --git a/doc/README.rst b/doc/README.rst index e32665fbd04..62c22d5e46d 100644 --- a/doc/README.rst +++ b/doc/README.rst @@ -197,24 +197,20 @@ there: Filtering expected warnings *************************** -Alas, there are some known issues with the doxygen/Sphinx/Breathe -processing that generates warnings for some constructs, in particular -around unnamed structures in nested unions or structs. -While these issues are being considered for fixing in -Sphinx/Breathe, we've added a post-processing filter on the output of -the documentation build process to check for "expected" messages from the -generation process output. +There are some known issues with Sphinx/Breathe that generate Sphinx warnings +even though the input is valid C code. While these issues are being considered +for fixing we have created a Sphinx extension that allows to filter them out +based on a set of regular expressions. The extension is named +``zephyr.warnings_filter`` and it is located at +``doc/_extensions/zephyr/warnings_filter.py``. The warnings to be filtered out +can be added to the ``doc/known-warnings.txt`` file. -The output from the Sphinx build is processed by the python script -``scripts/filter-known-issues.py`` together with a set of filter -configuration files in the ``.known-issues/doc`` folder. (This -filtering is done as part of the ``doc/CMakeLists.txt`` CMake listfile.) +The most common warning reported by Sphinx/Breathe is related to duplicate C +declarations. This warning may be caused by different Sphinx/Breathe issues: -If you're contributing components included in the Zephyr API -documentation and run across these warnings, you can include filtering -them out as "expected" warnings by adding a conf file to the -``.known-issues/doc`` folder, following the example of other conf files -found there. +- Multiple declarations of the same object are not supported +- Different objects (e.g. a struct and a function) can not share the same name +- Nested elements (e.g. in a struct or union) can not share the same name Developer-mode Document Building ******************************** diff --git a/doc/conf.py b/doc/conf.py index 408e274841d..4165210fe07 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -82,6 +82,7 @@ extensions = [ "zephyr.dtcompatible-role", "zephyr.link-roles", "sphinx_tabs.tabs", + "zephyr.warnings_filter", ] # Only use SVG converter when it is really needed, e.g. LaTeX. @@ -189,6 +190,11 @@ c_id_attributes = cpp_id_attributes html_redirect_pages = redirects.REDIRECTS +# -- Options for zephyr.warnings_filter ----------------------------------- + +warnings_filter_config = str(ZEPHYR_BASE / "doc" / "known-warnings.txt") +warnings_filter_silent = False + # -- Linkcheck options ---------------------------------------------------- extlinks = { diff --git a/doc/known-warnings.txt b/doc/known-warnings.txt new file mode 100644 index 00000000000..4e9601ae457 --- /dev/null +++ b/doc/known-warnings.txt @@ -0,0 +1,16 @@ +# Each line should contain the regular expression of a known Sphinx warning +# that should be filtered out +.*Duplicate C declaration.*\n.*'\.\. c:struct:: dma_config'.* +.*Duplicate C declaration.*\n.*'\.\. c:struct:: zsock_fd_set'.* +.*Duplicate C declaration.*\n.*'\.\. c:struct:: net_if_mcast_monitor'.* +.*Duplicate C declaration.*\n.*'\.\. c:struct:: fs_statvfs'.* +.*Duplicate C declaration.*\n.*'\.\. c:function:: .*dmic_trigger.*'.* +.*Duplicate C declaration.*\n.*'\.\. c:var:: uint16_t id'.* +.*Duplicate C declaration.*\n.*'\.\. c:function:: .*net_if_ipv4_addr_mask_cmp.*'.* +.*Duplicate C declaration.*\n.*'\.\. c:function:: .*net_if_ipv4_is_addr_bcast.*'.* +.*Duplicate C declaration.*\n.*'\.\. c:function:: .*net_if_ipv4_addr_lookup.*'.* +.*Duplicate C declaration.*\n.*'\.\. c:function:: .*net_if_ipv6_addr_lookup.*'.* +.*Duplicate C declaration.*\n.*'\.\. c:function:: .*net_if_ipv6_maddr_lookup.*'.* +.*Duplicate C declaration.*\n.*'\.\. c:None:: .*struct in_addr.*'.* +.*Duplicate C declaration.*\n.*'\.\. c:None:: .*struct in6_addr.*'.* +.*Duplicate C declaration.*\n.*'\.\. c:None:: .*struct net_if.*'.* diff --git a/scripts/filter-known-issues.py b/scripts/filter-known-issues.py deleted file mode 100755 index cb197c6dcce..00000000000 --- a/scripts/filter-known-issues.py +++ /dev/null @@ -1,249 +0,0 @@ -#! /usr/bin/env python3 -""" -Filters a file, classifying output in errors, warnings and discarding -the rest. - -Given a set of regular expressions read from files named *.conf in the -given configuration path(s), of the format: - - # - # Comments for multiline regex 1... - # - MULTILINEPYTHONREGEX - MULTILINEPYTHONREGEX - MULTILINEPYTHONREGEX - # - # Comments for multiline regex 2... - # - #WARNING - MULTILINEPYTHONREGEX2 - MULTILINEPYTHONREGEX2 - -Anything matched by MULTILINEPYTHONREGEX will be considered something -to be filtered out and not printed. - -Anything matched by MULTILINEPYHONREGEX with a #WARNING tag in the -comment means (optional) means that it describes something that is -considered to be a warning. Print it to stderr. - -Anything leftover is considred to be errors, printed to stdout. - -""" -import argparse -import logging -import mmap -import os -import re -import sre_constants -import sys -import traceback - -exclude_regexs = [] - -# first is a list of one or more comment lines -# followed by a list of non-comments which describe a multiline regex -config_regex = \ - b"(?P(^\\s*#.*\n)+)" \ - b"(?P(^[^#].*\n)+)" - - -def config_import_file(filename): - """ - Imports regular expression from any file *.conf in the given path, - format as given in the main doc - """ - try: - with open(filename, "rb") as f: - mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) - # That regex basically selects any block of - # lines that is not a comment block. The - # finditer() finds all the blocks and selects - # the bits of mmapped-file that comprises - # each--we compile it into a regex and append. - for m in re.finditer(config_regex, mm, re.MULTILINE): - origin = "%s:%s-%s" % (filename, m.start(), m.end()) - gd = m.groupdict() - comment = gd['comment'] - regex = gd['regex'] - try: - r = re.compile(regex, re.MULTILINE) - except sre_constants.error as e: - logging.error("%s: bytes %d-%d: bad regex: %s", - filename, m.start(), m.end(), e) - raise - logging.debug("%s: found regex at bytes %d-%d: %s", - filename, m.start(), m.end(), regex) - if b'#WARNING' in comment: - exclude_regexs.append((r, origin, ('warning',))) - else: - exclude_regexs.append((r, origin, ())) - logging.debug("%s: loaded", filename) - except Exception as e: - logging.error("E: %s: can't load config file: %s" % (filename, e)) - raise - - -def config_import_path(path): - """ - Imports regular expression from any file *.conf in the given path - """ - file_regex = re.compile(r".*\.conf$") - try: - for dirpath, _, filenames in os.walk(path): - for _filename in sorted(filenames): - filename = os.path.join(dirpath, _filename) - if not file_regex.search(_filename): - logging.debug("%s: ignored", filename) - continue - config_import_file(filename) - except Exception as e: - raise Exception( - "E: %s: can't load config files: %s %s" % - (path, e, traceback.format_exc())) - - -def config_import(paths): - """ - Imports regular expression from any file *.conf in the list of paths. - - If a path is "" or None, the list of paths until then is flushed - and only the new ones are considered. - """ - _paths = [] - # Go over the list, flush it if the user gave an empty path ("") - for path in paths: - if path == "" or path is None: - logging.debug("flushing current config list: %s", _paths) - _paths = [] - else: - _paths.append(path) - logging.debug("config list: %s", _paths) - for path in _paths: - config_import_path(path) - - -arg_parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) -arg_parser.add_argument("-v", "--verbosity", action="count", default=0, - help="increase verbosity") -arg_parser.add_argument("-q", "--quiet", action="count", default=0, - help="decrease verbosity") -arg_parser.add_argument("-e", "--errors", action="store", default=None, - help="file where to store errors") -arg_parser.add_argument("-w", "--warnings", action="store", default=None, - help="file where to store warnings") -arg_parser.add_argument("-c", "--config-dir", action="append", nargs="?", - default=[".known-issues/"], - help="configuration directory (multiple can be " - "given; if none given, clears the current list) " - "%(default)s") -arg_parser.add_argument("FILENAMEs", nargs="+", - help="files to filter") -args = arg_parser.parse_args() - -logging.basicConfig(level=40 - 10 * (args.verbosity - args.quiet), - format="%(levelname)s: %(message)s") - -path = ".known-issues/" -logging.debug("Reading configuration from directory `%s`", path) -config_import(args.config_dir) - -exclude_ranges = [] - -if args.warnings: - warnings = open(args.warnings, "w") -else: - warnings = None -if args.errors: - errors = open(args.errors, "w") -else: - errors = None - -def report_error(data): - sys.stdout.write(data.decode('utf-8')) - if errors: - errors.write(data.decode('utf-8')) - - -def report_warning(data): - sys.stderr.write(data.decode('utf-8')) - if warnings: - warnings.write(data.decode('utf-8')) - -for filename in args.FILENAMEs: - if os.stat(filename).st_size == 0: - continue # skip empty log files - try: - with open(filename, "r+b") as f: - logging.info("%s: filtering", filename) - # Yeah, this should be more protected in case of exception - # and such, but this is a short running program... - mm = mmap.mmap(f.fileno(), 0) - for ex, origin, flags in exclude_regexs: - logging.info("%s: searching from %s: %s", - filename, origin, ex.pattern) - for m in re.finditer(ex.pattern, mm, re.MULTILINE): - logging.info("%s: %s-%s: match from from %s %s", - filename, m.start(), m.end(), origin, flags) - if 'warning' in flags: - exclude_ranges.append((m.start(), m.end(), True)) - else: - exclude_ranges.append((m.start(), m.end(), False)) - - exclude_ranges = sorted(exclude_ranges, key=lambda r: r[0]) - logging.warning( - "%s: ranges excluded: %s", - filename, - exclude_ranges) - - # Decide what to do with what has been filtered; warnings - # go to stderr and warnings file, errors to stdout, what - # is ignored is just dumped. - offset = 0 - for b, e, warning in exclude_ranges: - mm.seek(offset) - if b > offset: - # We have something not caught by a filter, an error - logging.info("%s: error range (%d, %d), from %d %dB", - filename, offset, b, offset, b - offset) - report_error(mm.read(b - offset)) - mm.seek(b) - if warning: # A warning, print it - mm.seek(b) - logging.info("%s: warning range (%d, %d), from %d %dB", - filename, b, e, offset, e - b) - report_warning(mm.read(e - b)) - else: # Exclude, ignore it - d = b - offset - logging.info("%s: exclude range (%d, %d), from %d %dB", - filename, b, e, offset, d) - offset = e - mm.seek(offset) - if len(mm) != offset: - logging.info("%s: error final range from %d %dB", - filename, offset, len(mm)) - report_error(mm.read(len(mm) - offset - 1)) - del mm - except Exception as e: - logging.error("%s: cannot load: %s", filename, e) - raise - -if warnings or errors: - if warnings: - warnings.flush() - if errors: - errors.flush() - if ((os.path.isfile(args.warnings) and os.path.getsize(args.warnings) > 0) or - (os.path.isfile(args.errors) and os.path.getsize(args.errors) > 0)): - print('''\n\n ---- New errors/warnings not tracked as .known-issues/, \ -please fix them ----\n''') - if args.warnings: - print(open(args.warnings, "r").read()) - if args.errors and (args.errors != args.warnings): - print(open(args.errors, "r").read()) - else: - print("\n\nNo new errors/warnings.\n") - - print('''\nTo see *all* new error/warnings you must make/ninja clean and \ -rebuild from scratch.''')