[v5] Don't generate netlink .rst files inside $(srctree)

[PATCH v5 05/15] tools: ynl_gen_rst.py: make the index parser more generic

Posted by Mauro Carvalho Chehab 7 months, 3 weeks ago

It is not a good practice to store build-generated files
inside $(srctree), as one may be using O=<BUILDDIR> and even
have the Kernel on a read-only directory.

Change the YAML generation for netlink files to allow it
to parse data based on the source or on the object tree.

Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
---
 tools/net/ynl/pyynl/ynl_gen_rst.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/tools/net/ynl/pyynl/ynl_gen_rst.py b/tools/net/ynl/pyynl/ynl_gen_rst.py
index 7bfb8ceeeefc..b1e5acafb998 100755
--- a/tools/net/ynl/pyynl/ynl_gen_rst.py
+++ b/tools/net/ynl/pyynl/ynl_gen_rst.py
@@ -365,6 +365,7 @@ def parse_arguments() -> argparse.Namespace:
 
     parser.add_argument("-v", "--verbose", action="store_true")
     parser.add_argument("-o", "--output", help="Output file name")
+    parser.add_argument("-d", "--input_dir", help="YAML input directory")
 
     # Index and input are mutually exclusive
     group = parser.add_mutually_exclusive_group()
@@ -405,11 +406,14 @@ def write_to_rstfile(content: str, filename: str) -> None:
     """Write the generated content into an RST file"""
     logging.debug("Saving RST file to %s", filename)
 
+    dir = os.path.dirname(filename)
+    os.makedirs(dir, exist_ok=True)
+
     with open(filename, "w", encoding="utf-8") as rst_file:
         rst_file.write(content)
 
 
-def generate_main_index_rst(output: str) -> None:
+def generate_main_index_rst(output: str, index_dir: str) -> None:
     """Generate the `networking_spec/index` content and write to the file"""
     lines = []
 
@@ -418,12 +422,18 @@ def generate_main_index_rst(output: str) -> None:
     lines.append(rst_title("Netlink Family Specifications"))
     lines.append(rst_toctree(1))
 
-    index_dir = os.path.dirname(output)
-    logging.debug("Looking for .rst files in %s", index_dir)
+    index_fname = os.path.basename(output)
+    base, ext = os.path.splitext(index_fname)
+
+    if not index_dir:
+        index_dir = os.path.dirname(output)
+
+    logging.debug(f"Looking for {ext} files in %s", index_dir)
     for filename in sorted(os.listdir(index_dir)):
-        if not filename.endswith(".rst") or filename == "index.rst":
+        if not filename.endswith(ext) or filename == index_fname:
             continue
-        lines.append(f"   {filename.replace('.rst', '')}\n")
+        base, ext = os.path.splitext(filename)
+        lines.append(f"   {base}\n")
 
     logging.debug("Writing an index file at %s", output)
     write_to_rstfile("".join(lines), output)
@@ -447,7 +457,7 @@ def main() -> None:
 
     if args.index:
         # Generate the index RST file
-        generate_main_index_rst(args.output)
+        generate_main_index_rst(args.output, args.input_dir)
 
 
 if __name__ == "__main__":
-- 
2.49.0

Re: [PATCH v5 05/15] tools: ynl_gen_rst.py: make the index parser more generic

Posted by Donald Hunter 7 months, 3 weeks ago

Mauro Carvalho Chehab <mchehab+huawei@kernel.org> writes:

> It is not a good practice to store build-generated files
> inside $(srctree), as one may be using O=<BUILDDIR> and even
> have the Kernel on a read-only directory.
>
> Change the YAML generation for netlink files to allow it
> to parse data based on the source or on the object tree.
>
> Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>

Please drop this patch. It introduces changes that affect interim
patches and the functionality all gets dropped later in the series.

> ---
>  tools/net/ynl/pyynl/ynl_gen_rst.py | 22 ++++++++++++++++------
>  1 file changed, 16 insertions(+), 6 deletions(-)
>
> diff --git a/tools/net/ynl/pyynl/ynl_gen_rst.py b/tools/net/ynl/pyynl/ynl_gen_rst.py
> index 7bfb8ceeeefc..b1e5acafb998 100755
> --- a/tools/net/ynl/pyynl/ynl_gen_rst.py
> +++ b/tools/net/ynl/pyynl/ynl_gen_rst.py
> @@ -365,6 +365,7 @@ def parse_arguments() -> argparse.Namespace:
>  
>      parser.add_argument("-v", "--verbose", action="store_true")
>      parser.add_argument("-o", "--output", help="Output file name")
> +    parser.add_argument("-d", "--input_dir", help="YAML input directory")
>  
>      # Index and input are mutually exclusive
>      group = parser.add_mutually_exclusive_group()
> @@ -405,11 +406,14 @@ def write_to_rstfile(content: str, filename: str) -> None:
>      """Write the generated content into an RST file"""
>      logging.debug("Saving RST file to %s", filename)
>  
> +    dir = os.path.dirname(filename)
> +    os.makedirs(dir, exist_ok=True)
> +
>      with open(filename, "w", encoding="utf-8") as rst_file:
>          rst_file.write(content)
>  
>  
> -def generate_main_index_rst(output: str) -> None:
> +def generate_main_index_rst(output: str, index_dir: str) -> None:
>      """Generate the `networking_spec/index` content and write to the file"""
>      lines = []
>  
> @@ -418,12 +422,18 @@ def generate_main_index_rst(output: str) -> None:
>      lines.append(rst_title("Netlink Family Specifications"))
>      lines.append(rst_toctree(1))
>  
> -    index_dir = os.path.dirname(output)
> -    logging.debug("Looking for .rst files in %s", index_dir)
> +    index_fname = os.path.basename(output)
> +    base, ext = os.path.splitext(index_fname)
> +
> +    if not index_dir:
> +        index_dir = os.path.dirname(output)
> +
> +    logging.debug(f"Looking for {ext} files in %s", index_dir)
>      for filename in sorted(os.listdir(index_dir)):
> -        if not filename.endswith(".rst") or filename == "index.rst":
> +        if not filename.endswith(ext) or filename == index_fname:
>              continue
> -        lines.append(f"   {filename.replace('.rst', '')}\n")
> +        base, ext = os.path.splitext(filename)
> +        lines.append(f"   {base}\n")
>  
>      logging.debug("Writing an index file at %s", output)
>      write_to_rstfile("".join(lines), output)
> @@ -447,7 +457,7 @@ def main() -> None:
>  
>      if args.index:
>          # Generate the index RST file
> -        generate_main_index_rst(args.output)
> +        generate_main_index_rst(args.output, args.input_dir)
>  
>  
>  if __name__ == "__main__":

Re: [PATCH v5 05/15] tools: ynl_gen_rst.py: make the index parser more generic

Posted by Simon Horman 7 months, 3 weeks ago

On Tue, Jun 17, 2025 at 10:02:02AM +0200, Mauro Carvalho Chehab wrote:
> It is not a good practice to store build-generated files
> inside $(srctree), as one may be using O=<BUILDDIR> and even
> have the Kernel on a read-only directory.
> 
> Change the YAML generation for netlink files to allow it
> to parse data based on the source or on the object tree.
> 
> Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
> ---
>  tools/net/ynl/pyynl/ynl_gen_rst.py | 22 ++++++++++++++++------
>  1 file changed, 16 insertions(+), 6 deletions(-)
> 
> diff --git a/tools/net/ynl/pyynl/ynl_gen_rst.py b/tools/net/ynl/pyynl/ynl_gen_rst.py
> index 7bfb8ceeeefc..b1e5acafb998 100755
> --- a/tools/net/ynl/pyynl/ynl_gen_rst.py
> +++ b/tools/net/ynl/pyynl/ynl_gen_rst.py
> @@ -365,6 +365,7 @@ def parse_arguments() -> argparse.Namespace:
>  
>      parser.add_argument("-v", "--verbose", action="store_true")
>      parser.add_argument("-o", "--output", help="Output file name")
> +    parser.add_argument("-d", "--input_dir", help="YAML input directory")
>  
>      # Index and input are mutually exclusive
>      group = parser.add_mutually_exclusive_group()
> @@ -405,11 +406,14 @@ def write_to_rstfile(content: str, filename: str) -> None:
>      """Write the generated content into an RST file"""
>      logging.debug("Saving RST file to %s", filename)
>  
> +    dir = os.path.dirname(filename)
> +    os.makedirs(dir, exist_ok=True)
> +
>      with open(filename, "w", encoding="utf-8") as rst_file:
>          rst_file.write(content)

Hi Mauro,

With this patch applied I see the following, which did not happen before.

$ make -C tools/net/ynl
...
Traceback (most recent call last):
  File ".../tools/net/ynl/generated/../pyynl/ynl_gen_rst.py", line 464, in <module>
    main()
    ~~~~^^
  File ".../tools/net/ynl/generated/../pyynl/ynl_gen_rst.py", line 456, in main
    write_to_rstfile(content, args.output)
    ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^
  File ".../tools/net/ynl/generated/../pyynl/ynl_gen_rst.py", line 410, in write_to_rstfile
    os.makedirs(dir, exist_ok=True)
    ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "<frozen os>", line 227, in makedirs
FileNotFoundError: [Errno 2] No such file or directory: ''
make[1]: *** [Makefile:55: conntrack.rst] Error 1

-- 
pw-bot: cr

Re: [PATCH v5 05/15] tools: ynl_gen_rst.py: make the index parser more generic

Posted by Mauro Carvalho Chehab 7 months, 3 weeks ago

Em Tue, 17 Jun 2025 12:59:27 +0100
Simon Horman <horms@kernel.org> escreveu:

> On Tue, Jun 17, 2025 at 10:02:02AM +0200, Mauro Carvalho Chehab wrote:
> > It is not a good practice to store build-generated files
> > inside $(srctree), as one may be using O=<BUILDDIR> and even
> > have the Kernel on a read-only directory.
> > 
> > Change the YAML generation for netlink files to allow it
> > to parse data based on the source or on the object tree.
> > 
> > Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
> > ---
> >  tools/net/ynl/pyynl/ynl_gen_rst.py | 22 ++++++++++++++++------
> >  1 file changed, 16 insertions(+), 6 deletions(-)
> > 
> > diff --git a/tools/net/ynl/pyynl/ynl_gen_rst.py b/tools/net/ynl/pyynl/ynl_gen_rst.py
> > index 7bfb8ceeeefc..b1e5acafb998 100755
> > --- a/tools/net/ynl/pyynl/ynl_gen_rst.py
> > +++ b/tools/net/ynl/pyynl/ynl_gen_rst.py
> > @@ -365,6 +365,7 @@ def parse_arguments() -> argparse.Namespace:
> >  
> >      parser.add_argument("-v", "--verbose", action="store_true")
> >      parser.add_argument("-o", "--output", help="Output file name")
> > +    parser.add_argument("-d", "--input_dir", help="YAML input directory")
> >  
> >      # Index and input are mutually exclusive
> >      group = parser.add_mutually_exclusive_group()
> > @@ -405,11 +406,14 @@ def write_to_rstfile(content: str, filename: str) -> None:
> >      """Write the generated content into an RST file"""
> >      logging.debug("Saving RST file to %s", filename)
> >  
> > +    dir = os.path.dirname(filename)
> > +    os.makedirs(dir, exist_ok=True)
> > +
> >      with open(filename, "w", encoding="utf-8") as rst_file:
> >          rst_file.write(content)  
> 
> Hi Mauro,
> 
> With this patch applied I see the following, which did not happen before.

Thanks! this was an intermediate step. I'll just drop this patch and
fix conflicts at the next version.

> 
> $ make -C tools/net/ynl
> ...
> Traceback (most recent call last):
>   File ".../tools/net/ynl/generated/../pyynl/ynl_gen_rst.py", line 464, in <module>
>     main()
>     ~~~~^^
>   File ".../tools/net/ynl/generated/../pyynl/ynl_gen_rst.py", line 456, in main
>     write_to_rstfile(content, args.output)
>     ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^
>   File ".../tools/net/ynl/generated/../pyynl/ynl_gen_rst.py", line 410, in write_to_rstfile
>     os.makedirs(dir, exist_ok=True)
>     ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
>   File "<frozen os>", line 227, in makedirs
> FileNotFoundError: [Errno 2] No such file or directory: ''
> make[1]: *** [Makefile:55: conntrack.rst] Error 1
> 



Thanks,
Mauro

Re: [PATCH v5 05/15] tools: ynl_gen_rst.py: make the index parser more generic

Posted by Simon Horman 7 months, 3 weeks ago

On Wed, Jun 18, 2025 at 08:57:35AM +0200, Mauro Carvalho Chehab wrote:
> Em Tue, 17 Jun 2025 12:59:27 +0100
> Simon Horman <horms@kernel.org> escreveu:
> 
> > On Tue, Jun 17, 2025 at 10:02:02AM +0200, Mauro Carvalho Chehab wrote:
> > > It is not a good practice to store build-generated files
> > > inside $(srctree), as one may be using O=<BUILDDIR> and even
> > > have the Kernel on a read-only directory.
> > > 
> > > Change the YAML generation for netlink files to allow it
> > > to parse data based on the source or on the object tree.
> > > 
> > > Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
> > > ---
> > >  tools/net/ynl/pyynl/ynl_gen_rst.py | 22 ++++++++++++++++------
> > >  1 file changed, 16 insertions(+), 6 deletions(-)
> > > 
> > > diff --git a/tools/net/ynl/pyynl/ynl_gen_rst.py b/tools/net/ynl/pyynl/ynl_gen_rst.py
> > > index 7bfb8ceeeefc..b1e5acafb998 100755
> > > --- a/tools/net/ynl/pyynl/ynl_gen_rst.py
> > > +++ b/tools/net/ynl/pyynl/ynl_gen_rst.py
> > > @@ -365,6 +365,7 @@ def parse_arguments() -> argparse.Namespace:
> > >  
> > >      parser.add_argument("-v", "--verbose", action="store_true")
> > >      parser.add_argument("-o", "--output", help="Output file name")
> > > +    parser.add_argument("-d", "--input_dir", help="YAML input directory")
> > >  
> > >      # Index and input are mutually exclusive
> > >      group = parser.add_mutually_exclusive_group()
> > > @@ -405,11 +406,14 @@ def write_to_rstfile(content: str, filename: str) -> None:
> > >      """Write the generated content into an RST file"""
> > >      logging.debug("Saving RST file to %s", filename)
> > >  
> > > +    dir = os.path.dirname(filename)
> > > +    os.makedirs(dir, exist_ok=True)
> > > +
> > >      with open(filename, "w", encoding="utf-8") as rst_file:
> > >          rst_file.write(content)  
> > 
> > Hi Mauro,
> > 
> > With this patch applied I see the following, which did not happen before.
> 
> Thanks! this was an intermediate step. I'll just drop this patch and
> fix conflicts at the next version.

Likewise, thanks.