doc: use warnings_filter

Replace current filter setup with the warnings_filter Sphinx extension.
Note that current regexes have been simplified to make them more
readable while keeping the most relevant information.

Sphinx warnings are now treated as errors (-W).

Signed-off-by: Gerard Marull-Paretas <gerard.marull@nordicsemi.no>
This commit is contained in:
Gerard Marull-Paretas 2021-04-06 00:02:03 +02:00 committed by Carles Cufí
parent 74783c1976
commit 96c7d10a15
7 changed files with 36 additions and 329 deletions

View file

@ -1,39 +0,0 @@
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]file_system[/\\]index.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]peripherals[/\\]dma.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]audio[/\\]dmic.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]networking[/\\]net_if.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]networking[/\\]ieee802154.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]networking[/\\]sockets.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]uuid.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]sdp.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]rfcomm.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]hci_raw.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]hci_drivers.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]gatt.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]gap.rst):(?P<lineno>[0-9]+): WARNING: Duplicate C declaration, also defined at (.*)\.$
^Declaration is \'.*\'\.$

View file

@ -1,14 +0,0 @@
# multiple section 'index'
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]custom-doxygen[/\\]mainpage.md):[0-9]+: warning: multiple use of section label 'index' for main page, \(first occurrence: .*$
#
^(?P<filename>([\-:\\/\w\.])+[/\\]doc[/\\]reference[/\\]bluetooth[/\\]gatt.rst):(?P<lineno>[0-9]+): WARNING: Error in declarator$
^[ \t]*If.*:$
^[ \t]*Invalid C declaration: .* \[error at [0-9]+\]$
^[ \t]*ATOMIC_DEFINE.*$
^[- \t]*\^$
^[ \t]*If.*:$
^[ \t]*Error in declarator or parameters$
^[ \t]*Invalid C declaration: .* \[error at [0-9]+\]$
^[ \t]*ATOMIC_DEFINE.*$
^[- \t]*\^$
^[ \t]*$

View file

@ -97,7 +97,6 @@ set(DOXY_OUT ${CMAKE_CURRENT_BINARY_DIR}/doxygen)
set(RST_OUT ${CMAKE_CURRENT_BINARY_DIR}/rst)
set(DOC_LOG ${CMAKE_CURRENT_BINARY_DIR}/doc.log)
set(DOXY_LOG ${CMAKE_CURRENT_BINARY_DIR}/doxy.log)
set(SPHINX_LOG ${CMAKE_CURRENT_BINARY_DIR}/sphinx.log)
set(DOC_WARN ${CMAKE_CURRENT_BINARY_DIR}/doc.warnings)
set(CONTENT_OUTPUTS ${CMAKE_CURRENT_BINARY_DIR}/extracted-content.txt)
@ -201,9 +200,7 @@ else()
set(SEP :)
endif()
set(KI_SCRIPT ${ZEPHYR_BASE}/scripts/filter-known-issues.py)
set(FIX_TEX_SCRIPT ${ZEPHYR_BASE}/doc/_scripts/fix_tex.py)
set(CONF_DIR ${ZEPHYR_BASE}/.known-issues/doc)
#
# Generated Kconfig .rst documents
@ -330,7 +327,7 @@ set(SPHINX_BUILD_HTML_COMMAND
${CMAKE_COMMAND} -E env
ZEPHYR_BASE=${ZEPHYR_BASE}
ZEPHYR_BUILD=${CMAKE_CURRENT_BINARY_DIR}
${SPHINXBUILD} -w ${SPHINX_LOG} -N -t ${DOC_TAG} -b html ${ALLSPHINXOPTS} ${RST_OUT}/doc ${SPHINX_OUTPUT_DIR_HTML})
${SPHINXBUILD} -W -N -t ${DOC_TAG} -b html ${ALLSPHINXOPTS} ${RST_OUT}/doc ${SPHINX_OUTPUT_DIR_HTML})
# The sphinx-html target is provided as a convenience for incremental
# re-builds of content files without regenerating the entire docs
@ -355,9 +352,6 @@ add_dependencies(sphinx-html content)
add_custom_target(
html
COMMAND ${SPHINX_BUILD_HTML_COMMAND}
# Merge the Doxygen and Sphinx logs into a single file
COMMAND ${CMAKE_COMMAND} -P ${ZEPHYR_BASE}/cmake/util/fmerge.cmake ${DOC_LOG} ${DOXY_LOG} ${SPHINX_LOG}
COMMAND ${PYTHON_EXECUTABLE} ${KI_SCRIPT} --config-dir ${CONF_DIR} --errors ${DOC_WARN} --warnings ${DOC_WARN} ${DOC_LOG}
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
COMMENT "Generating HTML documentation with ${SPHINXBUILD} ${SPHINXOPTS}"
${SPHINX_USES_TERMINAL}
@ -369,7 +363,7 @@ add_custom_target(
set(SPHINX_BUILD_LATEX_COMMAND
${CMAKE_COMMAND} -E env
ZEPHYR_BUILD=${CMAKE_CURRENT_BINARY_DIR}
${SPHINXBUILD} -w ${SPHINX_LOG} -N -t ${DOC_TAG} -b latex -t svgconvert ${ALLSPHINXOPTS} ${RST_OUT}/doc ${SPHINX_OUTPUT_DIR_LATEX})
${SPHINXBUILD} -W -N -t ${DOC_TAG} -b latex -t svgconvert ${ALLSPHINXOPTS} ${RST_OUT}/doc ${SPHINX_OUTPUT_DIR_LATEX})
# The sphinx-latex target works similarly to sphinx-html, and carries
# the same warnings.
@ -386,9 +380,6 @@ add_dependencies(sphinx-latex content)
add_custom_command(
OUTPUT ${SPHINX_OUTPUT_DIR_LATEX}/zephyr.tex
COMMAND ${SPHINX_BUILD_LATEX_COMMAND}
# Merge the Doxygen and Sphinx logs into a single file
COMMAND ${CMAKE_COMMAND} -P ${ZEPHYR_BASE}/cmake/util/fmerge.cmake ${DOC_LOG} ${DOXY_LOG} ${SPHINX_LOG}
COMMAND ${PYTHON_EXECUTABLE} ${KI_SCRIPT} --config-dir ${CONF_DIR} --errors ${DOC_WARN} --warnings ${DOC_WARN} ${DOC_LOG}
COMMAND ${PYTHON_EXECUTABLE} ${FIX_TEX_SCRIPT} ${SPHINX_OUTPUT_DIR_LATEX}/zephyr.tex
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
COMMENT "Generating LaTeX documentation"

View file

@ -197,24 +197,20 @@ there:
Filtering expected warnings
***************************
Alas, there are some known issues with the doxygen/Sphinx/Breathe
processing that generates warnings for some constructs, in particular
around unnamed structures in nested unions or structs.
While these issues are being considered for fixing in
Sphinx/Breathe, we've added a post-processing filter on the output of
the documentation build process to check for "expected" messages from the
generation process output.
There are some known issues with Sphinx/Breathe that generate Sphinx warnings
even though the input is valid C code. While these issues are being considered
for fixing we have created a Sphinx extension that allows to filter them out
based on a set of regular expressions. The extension is named
``zephyr.warnings_filter`` and it is located at
``doc/_extensions/zephyr/warnings_filter.py``. The warnings to be filtered out
can be added to the ``doc/known-warnings.txt`` file.
The output from the Sphinx build is processed by the python script
``scripts/filter-known-issues.py`` together with a set of filter
configuration files in the ``.known-issues/doc`` folder. (This
filtering is done as part of the ``doc/CMakeLists.txt`` CMake listfile.)
The most common warning reported by Sphinx/Breathe is related to duplicate C
declarations. This warning may be caused by different Sphinx/Breathe issues:
If you're contributing components included in the Zephyr API
documentation and run across these warnings, you can include filtering
them out as "expected" warnings by adding a conf file to the
``.known-issues/doc`` folder, following the example of other conf files
found there.
- Multiple declarations of the same object are not supported
- Different objects (e.g. a struct and a function) can not share the same name
- Nested elements (e.g. in a struct or union) can not share the same name
Developer-mode Document Building
********************************

View file

@ -82,6 +82,7 @@ extensions = [
"zephyr.dtcompatible-role",
"zephyr.link-roles",
"sphinx_tabs.tabs",
"zephyr.warnings_filter",
]
# Only use SVG converter when it is really needed, e.g. LaTeX.
@ -189,6 +190,11 @@ c_id_attributes = cpp_id_attributes
html_redirect_pages = redirects.REDIRECTS
# -- Options for zephyr.warnings_filter -----------------------------------
warnings_filter_config = str(ZEPHYR_BASE / "doc" / "known-warnings.txt")
warnings_filter_silent = False
# -- Linkcheck options ----------------------------------------------------
extlinks = {

16
doc/known-warnings.txt Normal file
View file

@ -0,0 +1,16 @@
# Each line should contain the regular expression of a known Sphinx warning
# that should be filtered out
.*Duplicate C declaration.*\n.*'\.\. c:struct:: dma_config'.*
.*Duplicate C declaration.*\n.*'\.\. c:struct:: zsock_fd_set'.*
.*Duplicate C declaration.*\n.*'\.\. c:struct:: net_if_mcast_monitor'.*
.*Duplicate C declaration.*\n.*'\.\. c:struct:: fs_statvfs'.*
.*Duplicate C declaration.*\n.*'\.\. c:function:: .*dmic_trigger.*'.*
.*Duplicate C declaration.*\n.*'\.\. c:var:: uint16_t id'.*
.*Duplicate C declaration.*\n.*'\.\. c:function:: .*net_if_ipv4_addr_mask_cmp.*'.*
.*Duplicate C declaration.*\n.*'\.\. c:function:: .*net_if_ipv4_is_addr_bcast.*'.*
.*Duplicate C declaration.*\n.*'\.\. c:function:: .*net_if_ipv4_addr_lookup.*'.*
.*Duplicate C declaration.*\n.*'\.\. c:function:: .*net_if_ipv6_addr_lookup.*'.*
.*Duplicate C declaration.*\n.*'\.\. c:function:: .*net_if_ipv6_maddr_lookup.*'.*
.*Duplicate C declaration.*\n.*'\.\. c:None:: .*struct in_addr.*'.*
.*Duplicate C declaration.*\n.*'\.\. c:None:: .*struct in6_addr.*'.*
.*Duplicate C declaration.*\n.*'\.\. c:None:: .*struct net_if.*'.*

View file

@ -1,249 +0,0 @@
#! /usr/bin/env python3
"""
Filters a file, classifying output in errors, warnings and discarding
the rest.
Given a set of regular expressions read from files named *.conf in the
given configuration path(s), of the format:
#
# Comments for multiline regex 1...
#
MULTILINEPYTHONREGEX
MULTILINEPYTHONREGEX
MULTILINEPYTHONREGEX
#
# Comments for multiline regex 2...
#
#WARNING
MULTILINEPYTHONREGEX2
MULTILINEPYTHONREGEX2
Anything matched by MULTILINEPYTHONREGEX will be considered something
to be filtered out and not printed.
Anything matched by MULTILINEPYHONREGEX with a #WARNING tag in the
comment means (optional) means that it describes something that is
considered to be a warning. Print it to stderr.
Anything leftover is considred to be errors, printed to stdout.
"""
import argparse
import logging
import mmap
import os
import re
import sre_constants
import sys
import traceback
exclude_regexs = []
# first is a list of one or more comment lines
# followed by a list of non-comments which describe a multiline regex
config_regex = \
b"(?P<comment>(^\\s*#.*\n)+)" \
b"(?P<regex>(^[^#].*\n)+)"
def config_import_file(filename):
"""
Imports regular expression from any file *.conf in the given path,
format as given in the main doc
"""
try:
with open(filename, "rb") as f:
mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
# That regex basically selects any block of
# lines that is not a comment block. The
# finditer() finds all the blocks and selects
# the bits of mmapped-file that comprises
# each--we compile it into a regex and append.
for m in re.finditer(config_regex, mm, re.MULTILINE):
origin = "%s:%s-%s" % (filename, m.start(), m.end())
gd = m.groupdict()
comment = gd['comment']
regex = gd['regex']
try:
r = re.compile(regex, re.MULTILINE)
except sre_constants.error as e:
logging.error("%s: bytes %d-%d: bad regex: %s",
filename, m.start(), m.end(), e)
raise
logging.debug("%s: found regex at bytes %d-%d: %s",
filename, m.start(), m.end(), regex)
if b'#WARNING' in comment:
exclude_regexs.append((r, origin, ('warning',)))
else:
exclude_regexs.append((r, origin, ()))
logging.debug("%s: loaded", filename)
except Exception as e:
logging.error("E: %s: can't load config file: %s" % (filename, e))
raise
def config_import_path(path):
"""
Imports regular expression from any file *.conf in the given path
"""
file_regex = re.compile(r".*\.conf$")
try:
for dirpath, _, filenames in os.walk(path):
for _filename in sorted(filenames):
filename = os.path.join(dirpath, _filename)
if not file_regex.search(_filename):
logging.debug("%s: ignored", filename)
continue
config_import_file(filename)
except Exception as e:
raise Exception(
"E: %s: can't load config files: %s %s" %
(path, e, traceback.format_exc()))
def config_import(paths):
"""
Imports regular expression from any file *.conf in the list of paths.
If a path is "" or None, the list of paths until then is flushed
and only the new ones are considered.
"""
_paths = []
# Go over the list, flush it if the user gave an empty path ("")
for path in paths:
if path == "" or path is None:
logging.debug("flushing current config list: %s", _paths)
_paths = []
else:
_paths.append(path)
logging.debug("config list: %s", _paths)
for path in _paths:
config_import_path(path)
arg_parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
arg_parser.add_argument("-v", "--verbosity", action="count", default=0,
help="increase verbosity")
arg_parser.add_argument("-q", "--quiet", action="count", default=0,
help="decrease verbosity")
arg_parser.add_argument("-e", "--errors", action="store", default=None,
help="file where to store errors")
arg_parser.add_argument("-w", "--warnings", action="store", default=None,
help="file where to store warnings")
arg_parser.add_argument("-c", "--config-dir", action="append", nargs="?",
default=[".known-issues/"],
help="configuration directory (multiple can be "
"given; if none given, clears the current list) "
"%(default)s")
arg_parser.add_argument("FILENAMEs", nargs="+",
help="files to filter")
args = arg_parser.parse_args()
logging.basicConfig(level=40 - 10 * (args.verbosity - args.quiet),
format="%(levelname)s: %(message)s")
path = ".known-issues/"
logging.debug("Reading configuration from directory `%s`", path)
config_import(args.config_dir)
exclude_ranges = []
if args.warnings:
warnings = open(args.warnings, "w")
else:
warnings = None
if args.errors:
errors = open(args.errors, "w")
else:
errors = None
def report_error(data):
sys.stdout.write(data.decode('utf-8'))
if errors:
errors.write(data.decode('utf-8'))
def report_warning(data):
sys.stderr.write(data.decode('utf-8'))
if warnings:
warnings.write(data.decode('utf-8'))
for filename in args.FILENAMEs:
if os.stat(filename).st_size == 0:
continue # skip empty log files
try:
with open(filename, "r+b") as f:
logging.info("%s: filtering", filename)
# Yeah, this should be more protected in case of exception
# and such, but this is a short running program...
mm = mmap.mmap(f.fileno(), 0)
for ex, origin, flags in exclude_regexs:
logging.info("%s: searching from %s: %s",
filename, origin, ex.pattern)
for m in re.finditer(ex.pattern, mm, re.MULTILINE):
logging.info("%s: %s-%s: match from from %s %s",
filename, m.start(), m.end(), origin, flags)
if 'warning' in flags:
exclude_ranges.append((m.start(), m.end(), True))
else:
exclude_ranges.append((m.start(), m.end(), False))
exclude_ranges = sorted(exclude_ranges, key=lambda r: r[0])
logging.warning(
"%s: ranges excluded: %s",
filename,
exclude_ranges)
# Decide what to do with what has been filtered; warnings
# go to stderr and warnings file, errors to stdout, what
# is ignored is just dumped.
offset = 0
for b, e, warning in exclude_ranges:
mm.seek(offset)
if b > offset:
# We have something not caught by a filter, an error
logging.info("%s: error range (%d, %d), from %d %dB",
filename, offset, b, offset, b - offset)
report_error(mm.read(b - offset))
mm.seek(b)
if warning: # A warning, print it
mm.seek(b)
logging.info("%s: warning range (%d, %d), from %d %dB",
filename, b, e, offset, e - b)
report_warning(mm.read(e - b))
else: # Exclude, ignore it
d = b - offset
logging.info("%s: exclude range (%d, %d), from %d %dB",
filename, b, e, offset, d)
offset = e
mm.seek(offset)
if len(mm) != offset:
logging.info("%s: error final range from %d %dB",
filename, offset, len(mm))
report_error(mm.read(len(mm) - offset - 1))
del mm
except Exception as e:
logging.error("%s: cannot load: %s", filename, e)
raise
if warnings or errors:
if warnings:
warnings.flush()
if errors:
errors.flush()
if ((os.path.isfile(args.warnings) and os.path.getsize(args.warnings) > 0) or
(os.path.isfile(args.errors) and os.path.getsize(args.errors) > 0)):
print('''\n\n ---- New errors/warnings not tracked as .known-issues/, \
please fix them ----\n''')
if args.warnings:
print(open(args.warnings, "r").read())
if args.errors and (args.errors != args.warnings):
print(open(args.errors, "r").read())
else:
print("\n\nNo new errors/warnings.\n")
print('''\nTo see *all* new error/warnings you must make/ninja clean and \
rebuild from scratch.''')