Add a simple sphinx.Parser to handle yaml files and add the
the code to handle Netlink specs. All other yaml files are
ignored.
The code was written in a way that parsing yaml for different
subsystems and even for different parts of Netlink are easy.
All it takes to have a different parser is to add an
import line similar to:
from netlink_yml_parser import YnlDocGenerator
adding the corresponding parser somewhere at the extension:
netlink_parser = YnlDocGenerator()
And then add a logic inside parse() to handle different
doc outputs, depending on the file location, similar to:
if "/netlink/specs/" in fname:
msg = self.netlink_parser.parse_yaml_file(fname)
Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
---
Documentation/sphinx/parser_yaml.py | 76 +++++++++++++++++++++++++++++
1 file changed, 76 insertions(+)
create mode 100755 Documentation/sphinx/parser_yaml.py
diff --git a/Documentation/sphinx/parser_yaml.py b/Documentation/sphinx/parser_yaml.py
new file mode 100755
index 000000000000..635945e1c5ba
--- /dev/null
+++ b/Documentation/sphinx/parser_yaml.py
@@ -0,0 +1,76 @@
+"""
+Sphinx extension for processing YAML files
+"""
+
+import os
+import re
+import sys
+
+from pprint import pformat
+
+from docutils.parsers.rst import Parser as RSTParser
+from docutils.statemachine import ViewList
+
+from sphinx.util import logging
+from sphinx.parsers import Parser
+
+srctree = os.path.abspath(os.environ["srctree"])
+sys.path.insert(0, os.path.join(srctree, "tools/net/ynl/pyynl"))
+
+from netlink_yml_parser import YnlDocGenerator # pylint: disable=C0413
+
+logger = logging.getLogger(__name__)
+
+class YamlParser(Parser):
+ """Custom parser for YAML files."""
+
+ # Need at least two elements on this set
+ supported = ('yaml', 'yml')
+
+ netlink_parser = YnlDocGenerator()
+
+ def do_parse(self, inputstring, document, msg):
+ """Parse YAML and generate a document tree."""
+
+ self.setup_parse(inputstring, document)
+
+ result = ViewList()
+
+ try:
+ # Parse message with RSTParser
+ for i, line in enumerate(msg.split('\n')):
+ result.append(line, document.current_source, i)
+
+ rst_parser = RSTParser()
+ rst_parser.parse('\n'.join(result), document)
+
+ except Exception as e:
+ document.reporter.error("YAML parsing error: %s" % pformat(e))
+
+ self.finish_parse()
+
+ # Overrides docutils.parsers.Parser. See sphinx.parsers.RSTParser
+ def parse(self, inputstring, document):
+ """Check if a YAML is meant to be parsed."""
+
+ fname = document.current_source
+
+ # Handle netlink yaml specs
+ if "/netlink/specs/" in fname:
+ msg = self.netlink_parser.parse_yaml_file(fname)
+ self.do_parse(inputstring, document, msg)
+
+ # All other yaml files are ignored
+
+def setup(app):
+ """Setup function for the Sphinx extension."""
+
+ # Add YAML parser
+ app.add_source_parser(YamlParser)
+ app.add_source_suffix('.yaml', 'yaml')
+
+ return {
+ 'version': '1.0',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
--
2.49.0Mauro Carvalho Chehab <mchehab+huawei@kernel.org> writes:
> Add a simple sphinx.Parser to handle yaml files and add the
> the code to handle Netlink specs. All other yaml files are
> ignored.
>
> The code was written in a way that parsing yaml for different
> subsystems and even for different parts of Netlink are easy.
>
> All it takes to have a different parser is to add an
> import line similar to:
>
> from netlink_yml_parser import YnlDocGenerator
>
> adding the corresponding parser somewhere at the extension:
>
> netlink_parser = YnlDocGenerator()
>
> And then add a logic inside parse() to handle different
> doc outputs, depending on the file location, similar to:
>
> if "/netlink/specs/" in fname:
> msg = self.netlink_parser.parse_yaml_file(fname)
>
> Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
> ---
> Documentation/sphinx/parser_yaml.py | 76 +++++++++++++++++++++++++++++
> 1 file changed, 76 insertions(+)
> create mode 100755 Documentation/sphinx/parser_yaml.py
>
> diff --git a/Documentation/sphinx/parser_yaml.py b/Documentation/sphinx/parser_yaml.py
> new file mode 100755
> index 000000000000..635945e1c5ba
> --- /dev/null
> +++ b/Documentation/sphinx/parser_yaml.py
> @@ -0,0 +1,76 @@
> +"""
> +Sphinx extension for processing YAML files
> +"""
> +
> +import os
> +import re
> +import sys
> +
> +from pprint import pformat
> +
> +from docutils.parsers.rst import Parser as RSTParser
> +from docutils.statemachine import ViewList
> +
> +from sphinx.util import logging
> +from sphinx.parsers import Parser
> +
> +srctree = os.path.abspath(os.environ["srctree"])
> +sys.path.insert(0, os.path.join(srctree, "tools/net/ynl/pyynl"))
> +
> +from netlink_yml_parser import YnlDocGenerator # pylint: disable=C0413
> +
> +logger = logging.getLogger(__name__)
> +
> +class YamlParser(Parser):
> + """Custom parser for YAML files."""
Would be good to say that this is a common YAML parser that calls
different subsystems, e.g. how you described it in the commit message.
> +
> + # Need at least two elements on this set
I think you can drop this comment. It's not that it must be two
elements, it's that supported needs to be a list and the python syntax
to force parsing as a list would be ('item', )
> + supported = ('yaml', 'yml')
> +
> + netlink_parser = YnlDocGenerator()
> +
> + def do_parse(self, inputstring, document, msg):
Maybe a better name for this is parse_rst?
> + """Parse YAML and generate a document tree."""
Also update comment.
> +
> + self.setup_parse(inputstring, document)
> +
> + result = ViewList()
> +
> + try:
> + # Parse message with RSTParser
> + for i, line in enumerate(msg.split('\n')):
> + result.append(line, document.current_source, i)
This has the effect of associating line numbers from the generated ReST
with the source .yaml file, right? So errors will be reported against
the wrong place in the file. Is there any way to show the cause of the
error in the intermediate ReST?
As an example if I modify tc.yaml like this:
diff --git a/Documentation/netlink/specs/tc.yaml b/Documentation/netlink/specs/tc.yaml
index 4cc1f6a45001..c36d86d2dc72 100644
--- a/Documentation/netlink/specs/tc.yaml
+++ b/Documentation/netlink/specs/tc.yaml
@@ -4044,7 +4044,9 @@ operations:
- chain
-
name: getchain
- doc: Get / dump tc chain information.
+ doc: |
+ Get / dump tc chain information.
+ .. bogus-directive::
attribute-set: attrs
fixed-header: tcmsg
do:
This is the resuting error which will be really hard to track down:
/home/donaldh/net-next/Documentation/netlink/specs/tc.yaml:216: ERROR: Unexpected indentation. [docutils]
> +
> + rst_parser = RSTParser()
> + rst_parser.parse('\n'.join(result), document)
> +
> + except Exception as e:
> + document.reporter.error("YAML parsing error: %s" % pformat(e))
> +
> + self.finish_parse()
> +
> + # Overrides docutils.parsers.Parser. See sphinx.parsers.RSTParser
> + def parse(self, inputstring, document):
> + """Check if a YAML is meant to be parsed."""
> +
> + fname = document.current_source
> +
> + # Handle netlink yaml specs
> + if "/netlink/specs/" in fname:
> + msg = self.netlink_parser.parse_yaml_file(fname)
> + self.do_parse(inputstring, document, msg)
> +
> + # All other yaml files are ignored
> +
> +def setup(app):
> + """Setup function for the Sphinx extension."""
> +
> + # Add YAML parser
> + app.add_source_parser(YamlParser)
> + app.add_source_suffix('.yaml', 'yaml')
> +
> + return {
> + 'version': '1.0',
> + 'parallel_read_safe': True,
> + 'parallel_write_safe': True,
> + }
Em Tue, 17 Jun 2025 13:35:50 +0100
Donald Hunter <donald.hunter@gmail.com> escreveu:
> Mauro Carvalho Chehab <mchehab+huawei@kernel.org> writes:
>
> > Add a simple sphinx.Parser to handle yaml files and add the
> > the code to handle Netlink specs. All other yaml files are
> > ignored.
> >
> > The code was written in a way that parsing yaml for different
> > subsystems and even for different parts of Netlink are easy.
> >
> > All it takes to have a different parser is to add an
> > import line similar to:
> >
> > from netlink_yml_parser import YnlDocGenerator
> >
> > adding the corresponding parser somewhere at the extension:
> >
> > netlink_parser = YnlDocGenerator()
> >
> > And then add a logic inside parse() to handle different
> > doc outputs, depending on the file location, similar to:
> >
> > if "/netlink/specs/" in fname:
> > msg = self.netlink_parser.parse_yaml_file(fname)
> >
> > Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
> > ---
> > Documentation/sphinx/parser_yaml.py | 76 +++++++++++++++++++++++++++++
> > 1 file changed, 76 insertions(+)
> > create mode 100755 Documentation/sphinx/parser_yaml.py
> >
> > diff --git a/Documentation/sphinx/parser_yaml.py b/Documentation/sphinx/parser_yaml.py
> > new file mode 100755
> > index 000000000000..635945e1c5ba
> > --- /dev/null
> > +++ b/Documentation/sphinx/parser_yaml.py
> > @@ -0,0 +1,76 @@
> > +"""
> > +Sphinx extension for processing YAML files
> > +"""
> > +
> > +import os
> > +import re
> > +import sys
> > +
> > +from pprint import pformat
> > +
> > +from docutils.parsers.rst import Parser as RSTParser
> > +from docutils.statemachine import ViewList
> > +
> > +from sphinx.util import logging
> > +from sphinx.parsers import Parser
> > +
> > +srctree = os.path.abspath(os.environ["srctree"])
> > +sys.path.insert(0, os.path.join(srctree, "tools/net/ynl/pyynl"))
> > +
> > +from netlink_yml_parser import YnlDocGenerator # pylint: disable=C0413
> > +
> > +logger = logging.getLogger(__name__)
> > +
> > +class YamlParser(Parser):
> > + """Custom parser for YAML files."""
>
> Would be good to say that this is a common YAML parser that calls
> different subsystems, e.g. how you described it in the commit message.
Makes sense. Will fix at the next version.
>
> > +
> > + # Need at least two elements on this set
>
> I think you can drop this comment. It's not that it must be two
> elements, it's that supported needs to be a list and the python syntax
> to force parsing as a list would be ('item', )
Ah, ok.
> > + supported = ('yaml', 'yml')
> > +
> > + netlink_parser = YnlDocGenerator()
> > +
> > + def do_parse(self, inputstring, document, msg):
>
> Maybe a better name for this is parse_rst?
Ok.
>
> > + """Parse YAML and generate a document tree."""
>
> Also update comment.
Ok.
> > +
> > + self.setup_parse(inputstring, document)
> > +
> > + result = ViewList()
> > +
> > + try:
> > + # Parse message with RSTParser
> > + for i, line in enumerate(msg.split('\n')):
> > + result.append(line, document.current_source, i)
>
> This has the effect of associating line numbers from the generated ReST
> with the source .yaml file, right? So errors will be reported against
> the wrong place in the file. Is there any way to show the cause of the
> error in the intermediate ReST?
Yes, but this will require modifying the parser. I prefer merging this
series without such change, and then having a separate changeset
addressing it.
There are two ways we can do that:
1. The parser can add a ReST comment with the line number. This
is what it is done by kerneldoc.py Sphinx extension:
lineoffset = 0
line_regex = re.compile(r"^\.\. LINENO ([0-9]+)$")
for line in lines:
match = line_regex.search(line)
if match:
lineoffset = int(match.group(1)) - 1 # sphinx counts lines from 0
else:
doc = str(env.srcdir) + "/" + env.docname + ":" + str(self.lineno)
result.append(line, doc + ": " + filename, lineoffset)
lineoffset += 1
I kept the same way after its conversion to Python, as right now,
it supports both a Python class and a command lin command. I may
eventually clean it up in the future.
2. making the parser return a tuple. At kernel_abi.py, as the parser
returns content from multiple files, such tuple is:
(rst_output, filename, line_number)
and the code for it is (cleaned up):
for msg, f, ln in kernel_abi.doc(show_file=show_file,
show_symbols=show_symbols,
filter_path=abi_type):
lines = statemachine.string2lines(msg, tab_width,
convert_whitespace=True)
for line in lines:
content.append(line, f, ln - 1) # sphinx counts lines from 0
(2) is cleaner and faster, but (1) is easier to implement on an
already-existing code.
> As an example if I modify tc.yaml like this:
>
> diff --git a/Documentation/netlink/specs/tc.yaml b/Documentation/netlink/specs/tc.yaml
> index 4cc1f6a45001..c36d86d2dc72 100644
> --- a/Documentation/netlink/specs/tc.yaml
> +++ b/Documentation/netlink/specs/tc.yaml
> @@ -4044,7 +4044,9 @@ operations:
> - chain
> -
> name: getchain
> - doc: Get / dump tc chain information.
> + doc: |
> + Get / dump tc chain information.
> + .. bogus-directive::
> attribute-set: attrs
> fixed-header: tcmsg
> do:
>
> This is the resuting error which will be really hard to track down:
>
> /home/donaldh/net-next/Documentation/netlink/specs/tc.yaml:216: ERROR: Unexpected indentation. [docutils]
>
> > +
> > + rst_parser = RSTParser()
> > + rst_parser.parse('\n'.join(result), document)
> > +
> > + except Exception as e:
> > + document.reporter.error("YAML parsing error: %s" % pformat(e))
> > +
> > + self.finish_parse()
> > +
> > + # Overrides docutils.parsers.Parser. See sphinx.parsers.RSTParser
> > + def parse(self, inputstring, document):
> > + """Check if a YAML is meant to be parsed."""
> > +
> > + fname = document.current_source
> > +
> > + # Handle netlink yaml specs
> > + if "/netlink/specs/" in fname:
> > + msg = self.netlink_parser.parse_yaml_file(fname)
> > + self.do_parse(inputstring, document, msg)
> > +
> > + # All other yaml files are ignored
> > +
> > +def setup(app):
> > + """Setup function for the Sphinx extension."""
> > +
> > + # Add YAML parser
> > + app.add_source_parser(YamlParser)
> > + app.add_source_suffix('.yaml', 'yaml')
> > +
> > + return {
> > + 'version': '1.0',
> > + 'parallel_read_safe': True,
> > + 'parallel_write_safe': True,
> > + }
Em Tue, 17 Jun 2025 15:40:49 +0200
Mauro Carvalho Chehab <mchehab+huawei@kernel.org> escreveu:
> > > + # Parse message with RSTParser
> > > + for i, line in enumerate(msg.split('\n')):
> > > + result.append(line, document.current_source, i)
> >
> > This has the effect of associating line numbers from the generated ReST
> > with the source .yaml file, right? So errors will be reported against
> > the wrong place in the file. Is there any way to show the cause of the
> > error in the intermediate ReST?
>
> Yes, but this will require modifying the parser. I prefer merging this
> series without such change, and then having a separate changeset
> addressing it.
>
> There are two ways we can do that:
>
> 1. The parser can add a ReST comment with the line number. This
> is what it is done by kerneldoc.py Sphinx extension:
>
> lineoffset = 0
> line_regex = re.compile(r"^\.\. LINENO ([0-9]+)$")
> for line in lines:
> match = line_regex.search(line)
> if match:
> lineoffset = int(match.group(1)) - 1 # sphinx counts lines from 0
> else:
> doc = str(env.srcdir) + "/" + env.docname + ":" + str(self.lineno)
> result.append(line, doc + ": " + filename, lineoffset)
> lineoffset += 1
>
> I kept the same way after its conversion to Python, as right now,
> it supports both a Python class and a command lin command. I may
> eventually clean it up in the future.
>
> 2. making the parser return a tuple. At kernel_abi.py, as the parser
> returns content from multiple files, such tuple is:
>
> (rst_output, filename, line_number)
>
> and the code for it is (cleaned up):
>
> for msg, f, ln in kernel_abi.doc(show_file=show_file,
> show_symbols=show_symbols,
> filter_path=abi_type):
>
> lines = statemachine.string2lines(msg, tab_width,
> convert_whitespace=True)
>
> for line in lines:
> content.append(line, f, ln - 1) # sphinx counts lines from 0
>
> (2) is cleaner and faster, but (1) is easier to implement on an
> already-existing code.
The logic below implements (1). This seems to be the easiest way for
pyyaml. I will submit as 2 separate patches at the end of the next
version.
Please notice that I didn't check yet for the "quality" of the
line numbers. Some tweaks could be needed later on.
Regards,
Mauro
---
From 750daebebadcd156b5fe9b516f4fae4bd42b9d2c Mon Sep 17 00:00:00 2001
From: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
Date: Tue, 17 Jun 2025 17:54:03 +0200
Subject: [PATCH] docs: parser_yaml.py: add support for line numbers from the
parser
Instead of printing line numbers from the temp converted ReST
file, get them from the original source.
Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
diff --git a/Documentation/sphinx/parser_yaml.py b/Documentation/sphinx/parser_yaml.py
index 635945e1c5ba..15c642fc0bd5 100755
--- a/Documentation/sphinx/parser_yaml.py
+++ b/Documentation/sphinx/parser_yaml.py
@@ -29,6 +29,8 @@ class YamlParser(Parser):
netlink_parser = YnlDocGenerator()
+ re_lineno = re.compile(r"\.\. LINENO ([0-9]+)$")
+
def do_parse(self, inputstring, document, msg):
"""Parse YAML and generate a document tree."""
@@ -38,8 +40,14 @@ class YamlParser(Parser):
try:
# Parse message with RSTParser
- for i, line in enumerate(msg.split('\n')):
- result.append(line, document.current_source, i)
+ lineoffset = 0;
+ for line in msg.split('\n'):
+ match = self.re_lineno.match(line)
+ if match:
+ lineoffset = int(match.group(1))
+ continue
+
+ result.append(line, document.current_source, lineoffset)
rst_parser = RSTParser()
rst_parser.parse('\n'.join(result), document)
From 15c1f9db30f3abdce110e19788d87f9fe1417781 Mon Sep 17 00:00:00 2001
From: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
Date: Tue, 17 Jun 2025 17:28:04 +0200
Subject: [PATCH] tools: netlink_yml_parser.py: add line numbers to parsed data
When something goes wrong, we want Sphinx error to point to the
right line number from the original source, not from the
processed ReST data.
Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
diff --git a/tools/net/ynl/pyynl/netlink_yml_parser.py b/tools/net/ynl/pyynl/netlink_yml_parser.py
index 866551726723..a9d8ab6f2639 100755
--- a/tools/net/ynl/pyynl/netlink_yml_parser.py
+++ b/tools/net/ynl/pyynl/netlink_yml_parser.py
@@ -20,6 +20,16 @@
from typing import Any, Dict, List
import yaml
+LINE_STR = '__lineno__'
+
+class NumberedSafeLoader(yaml.SafeLoader):
+ """Override the SafeLoader class to add line number to parsed data"""
+
+ def construct_mapping(self, node):
+ mapping = super().construct_mapping(node)
+ mapping[LINE_STR] = node.start_mark.line
+
+ return mapping
class RstFormatters:
"""RST Formatters"""
@@ -127,6 +137,11 @@ class RstFormatters:
"""Return a formatted label"""
return f".. _{title}:\n\n"
+ @staticmethod
+ def rst_lineno(lineno: int) -> str:
+ """Return a lineno comment"""
+ return f".. LINENO {lineno}\n"
+
class YnlDocGenerator:
"""YAML Netlink specs Parser"""
@@ -144,6 +159,9 @@ class YnlDocGenerator:
"""Parse 'do' section and return a formatted string"""
lines = []
for key in do_dict.keys():
+ if key == LINE_STR:
+ lines.append(self.fmt.rst_lineno(do_dict[key]))
+ continue
lines.append(self.fmt.rst_paragraph(self.fmt.bold(key), level + 1))
if key in ['request', 'reply']:
lines.append(self.parse_do_attributes(do_dict[key], level + 1) + "\n")
@@ -174,6 +192,10 @@ class YnlDocGenerator:
lines.append(self.fmt.rst_paragraph(operation["doc"]) + "\n")
for key in operation.keys():
+ if key == LINE_STR:
+ lines.append(self.fmt.rst_lineno(operation[key]))
+ continue
+
if key in preprocessed:
# Skip the special fields
continue
@@ -233,6 +255,9 @@ class YnlDocGenerator:
for definition in defs:
lines.append(self.fmt.rst_section(namespace, 'definition', definition["name"]))
for k in definition.keys():
+ if k == LINE_STR:
+ lines.append(self.fmt.rst_lineno(definition[k]))
+ continue
if k in preprocessed + ignored:
continue
lines.append(self.fmt.rst_fields(k, self.fmt.sanitize(definition[k]), 0))
@@ -268,6 +293,9 @@ class YnlDocGenerator:
lines.append(self.fmt.rst_subsubsection(attr_line))
for k in attr.keys():
+ if k == LINE_STR:
+ lines.append(self.fmt.rst_lineno(attr[k]))
+ continue
if k in preprocessed + ignored:
continue
if k in linkable:
@@ -306,6 +334,8 @@ class YnlDocGenerator:
lines = []
# Main header
+ lineno = obj.get('__lineno__', 0)
+ lines.append(self.fmt.rst_lineno(lineno))
family = obj['name']
@@ -354,7 +384,7 @@ class YnlDocGenerator:
def parse_yaml_file(self, filename: str) -> str:
"""Transform the YAML specified by filename into an RST-formatted string"""
with open(filename, "r", encoding="utf-8") as spec_file:
- yaml_data = yaml.safe_load(spec_file)
- content = self.parse_yaml(yaml_data)
+ numbered_yaml = yaml.load(spec_file, Loader=NumberedSafeLoader)
+ content = self.parse_yaml(numbered_yaml)
return content
On Tue, 17 Jun 2025 at 17:00, Mauro Carvalho Chehab
<mchehab+huawei@kernel.org> wrote:
> >
> > (2) is cleaner and faster, but (1) is easier to implement on an
> > already-existing code.
>
> The logic below implements (1). This seems to be the easiest way for
> pyyaml. I will submit as 2 separate patches at the end of the next
> version.
>
> Please notice that I didn't check yet for the "quality" of the
> line numbers. Some tweaks could be needed later on.
Thanks for working on this. I suppose we might be able to work on an
evolution from (1) to (2) in a followup piece of work?
> Regards,
> Mauro
>
> ---
>
> From 750daebebadcd156b5fe9b516f4fae4bd42b9d2c Mon Sep 17 00:00:00 2001
> From: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
> Date: Tue, 17 Jun 2025 17:54:03 +0200
> Subject: [PATCH] docs: parser_yaml.py: add support for line numbers from the
> parser
>
> Instead of printing line numbers from the temp converted ReST
> file, get them from the original source.
>
> Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
>
> diff --git a/Documentation/sphinx/parser_yaml.py b/Documentation/sphinx/parser_yaml.py
> index 635945e1c5ba..15c642fc0bd5 100755
> --- a/Documentation/sphinx/parser_yaml.py
> +++ b/Documentation/sphinx/parser_yaml.py
> @@ -29,6 +29,8 @@ class YamlParser(Parser):
>
> netlink_parser = YnlDocGenerator()
>
> + re_lineno = re.compile(r"\.\. LINENO ([0-9]+)$")
> +
> def do_parse(self, inputstring, document, msg):
> """Parse YAML and generate a document tree."""
>
> @@ -38,8 +40,14 @@ class YamlParser(Parser):
>
> try:
> # Parse message with RSTParser
> - for i, line in enumerate(msg.split('\n')):
> - result.append(line, document.current_source, i)
> + lineoffset = 0;
> + for line in msg.split('\n'):
> + match = self.re_lineno.match(line)
> + if match:
> + lineoffset = int(match.group(1))
> + continue
> +
> + result.append(line, document.current_source, lineoffset)
>
> rst_parser = RSTParser()
> rst_parser.parse('\n'.join(result), document)
>
> From 15c1f9db30f3abdce110e19788d87f9fe1417781 Mon Sep 17 00:00:00 2001
> From: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
> Date: Tue, 17 Jun 2025 17:28:04 +0200
> Subject: [PATCH] tools: netlink_yml_parser.py: add line numbers to parsed data
>
> When something goes wrong, we want Sphinx error to point to the
> right line number from the original source, not from the
> processed ReST data.
>
> Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
>
> diff --git a/tools/net/ynl/pyynl/netlink_yml_parser.py b/tools/net/ynl/pyynl/netlink_yml_parser.py
> index 866551726723..a9d8ab6f2639 100755
> --- a/tools/net/ynl/pyynl/netlink_yml_parser.py
> +++ b/tools/net/ynl/pyynl/netlink_yml_parser.py
> @@ -20,6 +20,16 @@
> from typing import Any, Dict, List
> import yaml
>
> +LINE_STR = '__lineno__'
> +
> +class NumberedSafeLoader(yaml.SafeLoader):
> + """Override the SafeLoader class to add line number to parsed data"""
> +
> + def construct_mapping(self, node):
> + mapping = super().construct_mapping(node)
> + mapping[LINE_STR] = node.start_mark.line
> +
> + return mapping
>
> class RstFormatters:
> """RST Formatters"""
> @@ -127,6 +137,11 @@ class RstFormatters:
> """Return a formatted label"""
> return f".. _{title}:\n\n"
>
> + @staticmethod
> + def rst_lineno(lineno: int) -> str:
> + """Return a lineno comment"""
> + return f".. LINENO {lineno}\n"
> +
> class YnlDocGenerator:
> """YAML Netlink specs Parser"""
>
> @@ -144,6 +159,9 @@ class YnlDocGenerator:
> """Parse 'do' section and return a formatted string"""
> lines = []
> for key in do_dict.keys():
> + if key == LINE_STR:
> + lines.append(self.fmt.rst_lineno(do_dict[key]))
> + continue
> lines.append(self.fmt.rst_paragraph(self.fmt.bold(key), level + 1))
> if key in ['request', 'reply']:
> lines.append(self.parse_do_attributes(do_dict[key], level + 1) + "\n")
> @@ -174,6 +192,10 @@ class YnlDocGenerator:
> lines.append(self.fmt.rst_paragraph(operation["doc"]) + "\n")
>
> for key in operation.keys():
> + if key == LINE_STR:
> + lines.append(self.fmt.rst_lineno(operation[key]))
> + continue
> +
> if key in preprocessed:
> # Skip the special fields
> continue
> @@ -233,6 +255,9 @@ class YnlDocGenerator:
> for definition in defs:
> lines.append(self.fmt.rst_section(namespace, 'definition', definition["name"]))
> for k in definition.keys():
> + if k == LINE_STR:
> + lines.append(self.fmt.rst_lineno(definition[k]))
> + continue
> if k in preprocessed + ignored:
> continue
> lines.append(self.fmt.rst_fields(k, self.fmt.sanitize(definition[k]), 0))
> @@ -268,6 +293,9 @@ class YnlDocGenerator:
> lines.append(self.fmt.rst_subsubsection(attr_line))
>
> for k in attr.keys():
> + if k == LINE_STR:
> + lines.append(self.fmt.rst_lineno(attr[k]))
> + continue
> if k in preprocessed + ignored:
> continue
> if k in linkable:
> @@ -306,6 +334,8 @@ class YnlDocGenerator:
> lines = []
>
> # Main header
> + lineno = obj.get('__lineno__', 0)
> + lines.append(self.fmt.rst_lineno(lineno))
>
> family = obj['name']
>
> @@ -354,7 +384,7 @@ class YnlDocGenerator:
> def parse_yaml_file(self, filename: str) -> str:
> """Transform the YAML specified by filename into an RST-formatted string"""
> with open(filename, "r", encoding="utf-8") as spec_file:
> - yaml_data = yaml.safe_load(spec_file)
> - content = self.parse_yaml(yaml_data)
> + numbered_yaml = yaml.load(spec_file, Loader=NumberedSafeLoader)
> + content = self.parse_yaml(numbered_yaml)
>
> return content
>
Em Tue, 17 Jun 2025 18:23:22 +0100 Donald Hunter <donald.hunter@gmail.com> escreveu: > On Tue, 17 Jun 2025 at 17:00, Mauro Carvalho Chehab > <mchehab+huawei@kernel.org> wrote: > > > > > > (2) is cleaner and faster, but (1) is easier to implement on an > > > already-existing code. > > > > The logic below implements (1). This seems to be the easiest way for > > pyyaml. I will submit as 2 separate patches at the end of the next > > version. > > > > Please notice that I didn't check yet for the "quality" of the > > line numbers. Some tweaks could be needed later on. > > Thanks for working on this. I suppose we might be able to work on an > evolution from (1) to (2) in a followup piece of work? Yes, it shouldn't be hard for you to migrate to (2) in the future. Currently, the parser is stateless, but, as there's now a class, IMO the best would be to store the lines as an array of tuples inside the YnlDocGenerator class, like this: self.lines = [ (line_number1, message_string1), (line_number2, message_string2), (line_number3, message_string3), (line_number4, message_string4), ... ] This way, the parse_yaml_file() method would just return self.lines. Thanks, Mauro
© 2016 - 2025 Red Hat, Inc.