[PATCH v2] scripts: `make rust-analyzer` for out-of-tree modules

Vinay Varma posted 1 patch 2 years, 7 months ago
There is a newer version of this series
Makefile                          | 12 +++++++-----
rust/Makefile                     |  6 ++++--
scripts/generate_rust_analyzer.py | 31 ++++++++++++++++++-------------
3 files changed, 29 insertions(+), 20 deletions(-)
[PATCH v2] scripts: `make rust-analyzer` for out-of-tree modules
Posted by Vinay Varma 2 years, 7 months ago
Adds support for out-of-tree rust modules to use the `rust-analyzer`
make target to generate the rust-project.json file.

The change involves adding an optional parameter `external_src` to the
`generate_rust_analyzer.py` which expects the path to the out-of-tree
module's source directory. When this parameter is passed, I have chosen
not to add the non-core modules (samples and drivers) into the result
since these are not expected to be used in third party modules. Related
changes are also made to the Makefile and rust/Makefile allowing the
`rust-analyzer` target to be used for out-of-tree modules as well.

Link: https://github.com/Rust-for-Linux/linux/pull/914
Link: https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2

Signed-off-by: Vinay Varma <varmavinaym@gmail.com>
---
 Makefile                          | 12 +++++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 31 ++++++++++++++++++-------------
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/Makefile b/Makefile
index f41ec8c8426b..a055a316d2a4 100644
--- a/Makefile
+++ b/Makefile
@@ -1831,11 +1831,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-	$(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------
 
@@ -1888,6 +1883,7 @@ help:
 	@echo  '  modules         - default target, build the module(s)'
 	@echo  '  modules_install - install the module'
 	@echo  '  clean           - remove generated files in module directory only'
+	@echo  '  rust-analyzer	  - generate rust-project.json rust-analyzer support file'
 	@echo  ''
 
 endif # KBUILD_EXTMOD
@@ -2022,6 +2018,12 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
 	$(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+	$(Q)$(MAKE) $(build)=rust $@
+
+
 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------
 
diff --git a/rust/Makefile b/rust/Makefile
index 8f598a904f38..41c1435cd8d4 100644
--- a/rust/Makefile
+++ b/rust/Makefile
@@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 	$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-	$(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-		$(RUST_LIB_SRC) > $(objtree)/rust-project.json
+	$(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+		$(abs_srctree) $(abs_objtree) \
+		$(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+		$(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
 
 $(obj)/core.o: private skip_clippy = 1
 $(obj)/core.o: private skip_flags = -Dunreachable_pub
diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
index ecc7ea9a4dcf..1792f379ee4e 100755
--- a/scripts/generate_rust_analyzer.py
+++ b/scripts/generate_rust_analyzer.py
@@ -6,10 +6,11 @@
 import argparse
 import json
 import logging
+import os
 import pathlib
 import sys
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",
@@ -95,25 +96,28 @@ def generate_crates(srctree, objtree, sysroot_src):
         "exclude_dirs": [],
     }
 
+    def is_root_crate(build_file, target):
+        return os.path.exists(build_file) and target in open(build_file).read()
+
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    for folder in ("samples", "drivers"):
+    for folder in ("samples", "drivers") if external_src is None else [external_src]:
         for path in (srctree / folder).rglob("*.rs"):
             logging.info("Checking %s", path)
             name = path.name.replace(".rs", "")
 
             # Skip those that are not crate roots.
-            if f"{name}.o" not in open(path.parent / "Makefile").read():
-                continue
+            if is_root_crate(path.parent / "Makefile", f"{name}.o") or \
+               is_root_crate(path.parent / "Kbuild", f"{name}.o"):
 
-            logging.info("Adding %s", name)
-            append_crate(
-                name,
-                path,
-                ["core", "alloc", "kernel"],
-                cfg=cfg,
-            )
+                logging.info("Adding %s", name)
+                append_crate(
+                    name,
+                    path,
+                    ["core", "alloc", "kernel"],
+                    cfg=cfg,
+                )
 
     return crates
 
@@ -123,6 +127,7 @@ def main():
     parser.add_argument("srctree", type=pathlib.Path)
     parser.add_argument("objtree", type=pathlib.Path)
     parser.add_argument("sysroot_src", type=pathlib.Path)
+    parser.add_argument("exttree", type=pathlib.Path, nargs='?')
     args = parser.parse_args()
 
     logging.basicConfig(
@@ -131,7 +136,7 @@ def main():
     )
 
     rust_project = {
-        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
         "sysroot_src": str(args.sysroot_src),
     }
 
-- 
2.39.0
Re: [PATCH v2] scripts: `make rust-analyzer` for out-of-tree modules
Posted by Miguel Ojeda 2 years, 6 months ago
Hi Vinay,

On Sat, Jan 21, 2023 at 6:25 AM Vinay Varma <varmavinaym@gmail.com> wrote:
>
> Adds support for out-of-tree rust modules to use the `rust-analyzer`
> make target to generate the rust-project.json file.

Could you please address (some of) Masahiro's and Alice's feedback in
a new version?

Thanks!

Cheers,
Miguel
[PATCH v3] scripts: `make rust-analyzer` for out-of-tree modules
Posted by Vinay Varma 2 years, 6 months ago
Adds support for out-of-tree rust modules to use the `rust-analyzer`
make target to generate the rust-project.json file.

The change involves adding an optional parameter `external_src` to the
`generate_rust_analyzer.py` which expects the path to the out-of-tree
module's source directory. When this parameter is passed, I have chosen
not to add the non-core modules (samples and drivers) into the result
since these are not expected to be used in third party modules. Related
changes are also made to the Makefile and rust/Makefile allowing the
`rust-analyzer` target to be used for out-of-tree modules as well.

Link: https://github.com/Rust-for-Linux/linux/pull/914
Link: https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2

Signed-off-by: Vinay Varma <varmavinaym@gmail.com>
---
 Makefile                          | 11 ++++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 31 ++++++++++++++++++-------------
 3 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/Makefile b/Makefile
index 3f6628780eb2..ee7b61dc6075 100644
--- a/Makefile
+++ b/Makefile
@@ -1860,11 +1860,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-	$(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------
 
@@ -1917,6 +1912,7 @@ help:
 	@echo  '  modules         - default target, build the module(s)'
 	@echo  '  modules_install - install the module'
 	@echo  '  clean           - remove generated files in module directory only'
+	@echo  '  rust-analyzer	  - generate rust-project.json rust-analyzer support file'
 	@echo  ''
 
 endif # KBUILD_EXTMOD
@@ -2053,6 +2049,11 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
 	$(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+	$(Q)$(MAKE) $(build)=rust $@
+
 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------
 
diff --git a/rust/Makefile b/rust/Makefile
index 8f598a904f38..41c1435cd8d4 100644
--- a/rust/Makefile
+++ b/rust/Makefile
@@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 	$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-	$(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-		$(RUST_LIB_SRC) > $(objtree)/rust-project.json
+	$(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+		$(abs_srctree) $(abs_objtree) \
+		$(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+		$(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
 
 $(obj)/core.o: private skip_clippy = 1
 $(obj)/core.o: private skip_flags = -Dunreachable_pub
diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
index ecc7ea9a4dcf..741ccd6efe2b 100755
--- a/scripts/generate_rust_analyzer.py
+++ b/scripts/generate_rust_analyzer.py
@@ -6,10 +6,11 @@
 import argparse
 import json
 import logging
+import os
 import pathlib
 import sys
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",
@@ -95,25 +96,28 @@ def generate_crates(srctree, objtree, sysroot_src):
         "exclude_dirs": [],
     }
 
+    def is_root_crate(build_file, target):
+        return os.path.exists(build_file) and target in open(build_file).read()
+
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    for folder in ("samples", "drivers"):
+    for folder in ("samples", "drivers") if external_src is None else [external_src]:
         for path in (srctree / folder).rglob("*.rs"):
             logging.info("Checking %s", path)
             name = path.name.replace(".rs", "")
 
             # Skip those that are not crate roots.
-            if f"{name}.o" not in open(path.parent / "Makefile").read():
-                continue
+            if is_root_crate(path.parent / "Makefile", f"{name}.o") or \
+               is_root_crate(path.parent / "Kbuild", f"{name}.o"):
 
-            logging.info("Adding %s", name)
-            append_crate(
-                name,
-                path,
-                ["core", "alloc", "kernel"],
-                cfg=cfg,
-            )
+                logging.info("Adding %s", name)
+                append_crate(
+                    name,
+                    path,
+                    ["core", "alloc", "kernel"],
+                    cfg=cfg,
+                )
 
     return crates
 
@@ -123,6 +127,7 @@ def main():
     parser.add_argument("srctree", type=pathlib.Path)
     parser.add_argument("objtree", type=pathlib.Path)
     parser.add_argument("sysroot_src", type=pathlib.Path)
+    parser.add_argument("exttree", type=pathlib.Path, nargs="?")
     args = parser.parse_args()
 
     logging.basicConfig(
@@ -131,7 +136,7 @@ def main():
     )
 
     rust_project = {
-        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
         "sysroot_src": str(args.sysroot_src),
     }
 
-- 
2.39.2
[PATCH v4] scripts: `make rust-analyzer` for out-of-tree modules
Posted by Vinay Varma 2 years, 6 months ago
Adds support for out-of-tree rust modules to use the `rust-analyzer`
make target to generate the rust-project.json file.

The change involves adding an optional parameter `external_src` to the
`generate_rust_analyzer.py` which expects the path to the out-of-tree
module's source directory. When this parameter is passed, I have chosen
not to add the non-core modules (samples and drivers) into the result
since these are not expected to be used in third party modules. Related
changes are also made to the Makefile and rust/Makefile allowing the
`rust-analyzer` target to be used for out-of-tree modules as well.

Link: https://github.com/Rust-for-Linux/linux/pull/914
Link: https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2

Signed-off-by: Vinay Varma <varmavinaym@gmail.com>
---
 Makefile                          | 11 +++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 36 +++++++++++++++++++------------
 3 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/Makefile b/Makefile
index 3f6628780eb2..ee7b61dc6075 100644
--- a/Makefile
+++ b/Makefile
@@ -1860,11 +1860,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-	$(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------
 
@@ -1917,6 +1912,7 @@ help:
 	@echo  '  modules         - default target, build the module(s)'
 	@echo  '  modules_install - install the module'
 	@echo  '  clean           - remove generated files in module directory only'
+	@echo  '  rust-analyzer	  - generate rust-project.json rust-analyzer support file'
 	@echo  ''
 
 endif # KBUILD_EXTMOD
@@ -2053,6 +2049,11 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
 	$(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+	$(Q)$(MAKE) $(build)=rust $@
+
 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------
 
diff --git a/rust/Makefile b/rust/Makefile
index 8f598a904f38..41c1435cd8d4 100644
--- a/rust/Makefile
+++ b/rust/Makefile
@@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 	$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-	$(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-		$(RUST_LIB_SRC) > $(objtree)/rust-project.json
+	$(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+		$(abs_srctree) $(abs_objtree) \
+		$(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+		$(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
 
 $(obj)/core.o: private skip_clippy = 1
 $(obj)/core.o: private skip_flags = -Dunreachable_pub
diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
index ecc7ea9a4dcf..7a1a958b1f8a 100755
--- a/scripts/generate_rust_analyzer.py
+++ b/scripts/generate_rust_analyzer.py
@@ -6,10 +6,11 @@
 import argparse
 import json
 import logging
+import os
 import pathlib
 import sys
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",
@@ -95,25 +96,31 @@ def generate_crates(srctree, objtree, sysroot_src):
         "exclude_dirs": [],
     }
 
+    def is_root_crate(build_file, target):
+        return os.path.exists(build_file) and target in open(build_file).read()
+
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    for folder in ("samples", "drivers"):
-        for path in (srctree / folder).rglob("*.rs"):
+    extra_dirs = map(lambda dir: srctree / dir, ("samples", "drivers"))
+    if external_src is not None:
+        extra_dirs = [external_src]
+    for folder in extra_dirs:
+        for path in folder.rglob("*.rs"):
             logging.info("Checking %s", path)
             name = path.name.replace(".rs", "")
 
             # Skip those that are not crate roots.
-            if f"{name}.o" not in open(path.parent / "Makefile").read():
-                continue
+            if is_root_crate(path.parent / "Makefile", f"{name}.o") or \
+               is_root_crate(path.parent / "Kbuild", f"{name}.o"):
 
-            logging.info("Adding %s", name)
-            append_crate(
-                name,
-                path,
-                ["core", "alloc", "kernel"],
-                cfg=cfg,
-            )
+                logging.info("Adding %s", name)
+                append_crate(
+                    name,
+                    path,
+                    ["core", "alloc", "kernel"],
+                    cfg=cfg,
+                )
 
     return crates
 
@@ -123,6 +130,7 @@ def main():
     parser.add_argument("srctree", type=pathlib.Path)
     parser.add_argument("objtree", type=pathlib.Path)
     parser.add_argument("sysroot_src", type=pathlib.Path)
+    parser.add_argument("exttree", type=pathlib.Path, nargs="?")
     args = parser.parse_args()
 
     logging.basicConfig(
@@ -131,7 +139,7 @@ def main():
     )
 
     rust_project = {
-        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
         "sysroot_src": str(args.sysroot_src),
     }
 
-- 
2.39.2
Re: [PATCH v2] scripts: `make rust-analyzer` for out-of-tree modules
Posted by Björn Roy Baron 2 years, 7 months ago
------- Original Message -------
On Saturday, January 21st, 2023 at 06:25, Vinay Varma <varmavinaym@gmail.com> wrote:


> Adds support for out-of-tree rust modules to use the `rust-analyzer`
> make target to generate the rust-project.json file.
>
> The change involves adding an optional parameter `external_src` to the
> `generate_rust_analyzer.py` which expects the path to the out-of-tree
> module's source directory. When this parameter is passed, I have chosen
> not to add the non-core modules (samples and drivers) into the result
> since these are not expected to be used in third party modules. Related
> changes are also made to the Makefile and rust/Makefile allowing the
> `rust-analyzer` target to be used for out-of-tree modules as well.
>
> Link: https://github.com/Rust-for-Linux/linux/pull/914
> Link: https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2
>
> Signed-off-by: Vinay Varma varmavinaym@gmail.com

"make -C ../rust-for-linux M=$(pwd) rust-analyzer" produces a rust-project.json file in the source dir of the out of tree kernel module with a crate definition for the kernel module. In addition rust-analyzer is able to resolve definitions from the kernel crate as expected.

Tested-by: Björn Roy Baron <bjorn3_gh@protonmail.com>

>
> ---
> Makefile | 12 +++++++-----
> rust/Makefile | 6 ++++--
> scripts/generate_rust_analyzer.py | 31 ++++++++++++++++++-------------
> 3 files changed, 29 insertions(+), 20 deletions(-)
>
> diff --git a/Makefile b/Makefile
> index f41ec8c8426b..a055a316d2a4 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -1831,11 +1831,6 @@ rustfmt:
> rustfmtcheck: rustfmt_flags = --check
> rustfmtcheck: rustfmt
>
> -# IDE support targets
> -PHONY += rust-analyzer
> -rust-analyzer:
> - $(Q)$(MAKE) $(build)=rust $@
> -
> # Misc
> # ---------------------------------------------------------------------------
>
> @@ -1888,6 +1883,7 @@ help:
> @echo ' modules - default target, build the module(s)'
> @echo ' modules_install - install the module'
> @echo ' clean - remove generated files in module directory only'
> + @echo ' rust-analyzer - generate rust-project.json rust-analyzer support file'
> @echo ''
>
> endif # KBUILD_EXTMOD
> @@ -2022,6 +2018,12 @@ quiet_cmd_tags = GEN $@
> tags TAGS cscope gtags: FORCE
> $(call cmd,tags)
>
> +# IDE support targets
> +PHONY += rust-analyzer
> +rust-analyzer:
> + $(Q)$(MAKE) $(build)=rust $@
> +
> +
> # Script to generate missing namespace dependencies
> # ---------------------------------------------------------------------------
>
> diff --git a/rust/Makefile b/rust/Makefile
> index 8f598a904f38..41c1435cd8d4 100644
> --- a/rust/Makefile
> +++ b/rust/Makefile
> @@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
> $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
>
> rust-analyzer:
> - $(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
> - $(RUST_LIB_SRC) > $(objtree)/rust-project.json
>
> + $(Q)$(srctree)/scripts/generate_rust_analyzer.py \
> + $(abs_srctree) $(abs_objtree) \
> + $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
>
> + $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
>
> $(obj)/core.o: private skip_clippy = 1
> $(obj)/core.o: private skip_flags = -Dunreachable_pub
> diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
> index ecc7ea9a4dcf..1792f379ee4e 100755
> --- a/scripts/generate_rust_analyzer.py
> +++ b/scripts/generate_rust_analyzer.py
> @@ -6,10 +6,11 @@
> import argparse
> import json
> import logging
> +import os
> import pathlib
> import sys
>
> -def generate_crates(srctree, objtree, sysroot_src):
> +def generate_crates(srctree, objtree, sysroot_src, external_src):
> # Generate the configuration list.
> cfg = []
> with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
> @@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
> [],
> is_proc_macro=True,
> )
> - crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
> + crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
>
> append_crate(
> "build_error",
> @@ -95,25 +96,28 @@ def generate_crates(srctree, objtree, sysroot_src):
> "exclude_dirs": [],
> }
>
> + def is_root_crate(build_file, target):
> + return os.path.exists(build_file) and target in open(build_file).read()
> +
> # Then, the rest outside of `rust/`.
> #
> # We explicitly mention the top-level folders we want to cover.
> - for folder in ("samples", "drivers"):
> + for folder in ("samples", "drivers") if external_src is None else [external_src]:
> for path in (srctree / folder).rglob("*.rs"):
> logging.info("Checking %s", path)
> name = path.name.replace(".rs", "")
>
> # Skip those that are not crate roots.
> - if f"{name}.o" not in open(path.parent / "Makefile").read():
> - continue
> + if is_root_crate(path.parent / "Makefile", f"{name}.o") or \
> + is_root_crate(path.parent / "Kbuild", f"{name}.o"):
>
> - logging.info("Adding %s", name)
> - append_crate(
> - name,
> - path,
> - ["core", "alloc", "kernel"],
> - cfg=cfg,
> - )
> + logging.info("Adding %s", name)
> + append_crate(
> + name,
> + path,
> + ["core", "alloc", "kernel"],
> + cfg=cfg,
> + )
>
> return crates
>
> @@ -123,6 +127,7 @@ def main():
> parser.add_argument("srctree", type=pathlib.Path)
> parser.add_argument("objtree", type=pathlib.Path)
> parser.add_argument("sysroot_src", type=pathlib.Path)
> + parser.add_argument("exttree", type=pathlib.Path, nargs='?')
> args = parser.parse_args()
>
> logging.basicConfig(
> @@ -131,7 +136,7 @@ def main():
> )
>
> rust_project = {
> - "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
> + "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
> "sysroot_src": str(args.sysroot_src),
> }
>
> --
> 2.39.0

Cheers,
Björn

(resend as I forgot to do reply to all)
Re: [PATCH v2] scripts: `make rust-analyzer` for out-of-tree modules
Posted by Masahiro Yamada 2 years, 7 months ago
On Sat, Jan 21, 2023 at 2:25 PM Vinay Varma <varmavinaym@gmail.com> wrote:
>
> Adds support for out-of-tree rust modules to use the `rust-analyzer`
> make target to generate the rust-project.json file.
>
> The change involves adding an optional parameter `external_src` to the
> `generate_rust_analyzer.py` which expects the path to the out-of-tree
> module's source directory. When this parameter is passed, I have chosen
> not to add the non-core modules (samples and drivers) into the result
> since these are not expected to be used in third party modules. Related
> changes are also made to the Makefile and rust/Makefile allowing the
> `rust-analyzer` target to be used for out-of-tree modules as well.
>
> Link: https://github.com/Rust-for-Linux/linux/pull/914
> Link: https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2
>
> Signed-off-by: Vinay Varma <varmavinaym@gmail.com>
> ---
>  Makefile                          | 12 +++++++-----
>  rust/Makefile                     |  6 ++++--
>  scripts/generate_rust_analyzer.py | 31 ++++++++++++++++++-------------
>  3 files changed, 29 insertions(+), 20 deletions(-)
>
> diff --git a/Makefile b/Makefile
> index f41ec8c8426b..a055a316d2a4 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -1831,11 +1831,6 @@ rustfmt:
>  rustfmtcheck: rustfmt_flags = --check
>  rustfmtcheck: rustfmt
>
> -# IDE support targets
> -PHONY += rust-analyzer
> -rust-analyzer:
> -       $(Q)$(MAKE) $(build)=rust $@
> -
>  # Misc
>  # ---------------------------------------------------------------------------
>
> @@ -1888,6 +1883,7 @@ help:
>         @echo  '  modules         - default target, build the module(s)'
>         @echo  '  modules_install - install the module'
>         @echo  '  clean           - remove generated files in module directory only'
> +       @echo  '  rust-analyzer   - generate rust-project.json rust-analyzer support file'
>         @echo  ''
>
>  endif # KBUILD_EXTMOD
> @@ -2022,6 +2018,12 @@ quiet_cmd_tags = GEN     $@
>  tags TAGS cscope gtags: FORCE
>         $(call cmd,tags)
>
> +# IDE support targets
> +PHONY += rust-analyzer
> +rust-analyzer:
> +       $(Q)$(MAKE) $(build)=rust $@
> +
> +


Extra empty line.


>  # Script to generate missing namespace dependencies
>  # ---------------------------------------------------------------------------
>
> diff --git a/rust/Makefile b/rust/Makefile
> index 8f598a904f38..41c1435cd8d4 100644
> --- a/rust/Makefile
> +++ b/rust/Makefile
> @@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
>         $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
>
>  rust-analyzer:
> -       $(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
> -               $(RUST_LIB_SRC) > $(objtree)/rust-project.json
> +       $(Q)$(srctree)/scripts/generate_rust_analyzer.py \
> +               $(abs_srctree) $(abs_objtree) \
> +               $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
> +               $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json



This is equivalent to:

  >  $(extmod_prefix)/rust-project.json



See the rule of 'compile_commands.json'.







>  $(obj)/core.o: private skip_clippy = 1
>  $(obj)/core.o: private skip_flags = -Dunreachable_pub
> diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
> index ecc7ea9a4dcf..1792f379ee4e 100755
> --- a/scripts/generate_rust_analyzer.py
> +++ b/scripts/generate_rust_analyzer.py
> @@ -6,10 +6,11 @@
>  import argparse
>  import json
>  import logging
> +import os
>  import pathlib
>  import sys
>
> -def generate_crates(srctree, objtree, sysroot_src):
> +def generate_crates(srctree, objtree, sysroot_src, external_src):
>      # Generate the configuration list.
>      cfg = []
>      with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
> @@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
>          [],
>          is_proc_macro=True,
>      )
> -    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
> +    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
>
>      append_crate(
>          "build_error",
> @@ -95,25 +96,28 @@ def generate_crates(srctree, objtree, sysroot_src):
>          "exclude_dirs": [],
>      }
>
> +    def is_root_crate(build_file, target):
> +        return os.path.exists(build_file) and target in open(build_file).read()
> +
>      # Then, the rest outside of `rust/`.
>      #
>      # We explicitly mention the top-level folders we want to cover.


Huh, not maintainable, unfortunately.





> -    for folder in ("samples", "drivers"):
> +    for folder in ("samples", "drivers") if external_src is None else [external_src]:
>          for path in (srctree / folder).rglob("*.rs"):



It is odd to add 'srctree' prefix to external module sources.



I think rust-project.json is a similar concept to
compile_commands.json, but it was implemented
in a different way.







-- 
Best Regards
Masahiro Yamada
Re: [PATCH v2] scripts: `make rust-analyzer` for out-of-tree modules
Posted by Vinay Varma 2 years, 6 months ago
Sorry, got caught up with another project and lost track of this thread.
I have updated the patch and replied to some of the threads inline.
On Sat, Jan 21, 2023 at 06:18:01PM +0900, Masahiro Yamada wrote:
> On Sat, Jan 21, 2023 at 2:25 PM Vinay Varma <varmavinaym@gmail.com> wrote:
> >
> > Adds support for out-of-tree rust modules to use the `rust-analyzer`
> > make target to generate the rust-project.json file.
> >
> > The change involves adding an optional parameter `external_src` to the
> > `generate_rust_analyzer.py` which expects the path to the out-of-tree
> > module's source directory. When this parameter is passed, I have chosen
> > not to add the non-core modules (samples and drivers) into the result
> > since these are not expected to be used in third party modules. Related
> > changes are also made to the Makefile and rust/Makefile allowing the
> > `rust-analyzer` target to be used for out-of-tree modules as well.
> >
> > Link: https://github.com/Rust-for-Linux/linux/pull/914
> > Link: https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2
> >
> > Signed-off-by: Vinay Varma <varmavinaym@gmail.com>
> > ---
> >  Makefile                          | 12 +++++++-----
> >  rust/Makefile                     |  6 ++++--
> >  scripts/generate_rust_analyzer.py | 31 ++++++++++++++++++-------------
> >  3 files changed, 29 insertions(+), 20 deletions(-)
> >
> > diff --git a/Makefile b/Makefile
> > index f41ec8c8426b..a055a316d2a4 100644
> > --- a/Makefile
> > +++ b/Makefile
> > @@ -1831,11 +1831,6 @@ rustfmt:
> >  rustfmtcheck: rustfmt_flags = --check
> >  rustfmtcheck: rustfmt
> >
> > -# IDE support targets
> > -PHONY += rust-analyzer
> > -rust-analyzer:
> > -       $(Q)$(MAKE) $(build)=rust $@
> > -
> >  # Misc
> >  # ---------------------------------------------------------------------------
> >
> > @@ -1888,6 +1883,7 @@ help:
> >         @echo  '  modules         - default target, build the module(s)'
> >         @echo  '  modules_install - install the module'
> >         @echo  '  clean           - remove generated files in module directory only'
> > +       @echo  '  rust-analyzer   - generate rust-project.json rust-analyzer support file'
> >         @echo  ''
> >
> >  endif # KBUILD_EXTMOD
> > @@ -2022,6 +2018,12 @@ quiet_cmd_tags = GEN     $@
> >  tags TAGS cscope gtags: FORCE
> >         $(call cmd,tags)
> >
> > +# IDE support targets
> > +PHONY += rust-analyzer
> > +rust-analyzer:
> > +       $(Q)$(MAKE) $(build)=rust $@
> > +
> > +
> 
> 
> Extra empty line.
> 
> 
> >  # Script to generate missing namespace dependencies
> >  # ---------------------------------------------------------------------------
> >
> > diff --git a/rust/Makefile b/rust/Makefile
> > index 8f598a904f38..41c1435cd8d4 100644
> > --- a/rust/Makefile
> > +++ b/rust/Makefile
> > @@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
> >         $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
> >
> >  rust-analyzer:
> > -       $(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
> > -               $(RUST_LIB_SRC) > $(objtree)/rust-project.json
> > +       $(Q)$(srctree)/scripts/generate_rust_analyzer.py \
> > +               $(abs_srctree) $(abs_objtree) \
> > +               $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
> > +               $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
> 
> 
> 
> This is equivalent to:
> 
>   >  $(extmod_prefix)/rust-project.json
> 
> 
> 
> See the rule of 'compile_commands.json'.
> 
> 
> 
> 
> 
> 
> 
> >  $(obj)/core.o: private skip_clippy = 1
> >  $(obj)/core.o: private skip_flags = -Dunreachable_pub
> > diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
> > index ecc7ea9a4dcf..1792f379ee4e 100755
> > --- a/scripts/generate_rust_analyzer.py
> > +++ b/scripts/generate_rust_analyzer.py
> > @@ -6,10 +6,11 @@
> >  import argparse
> >  import json
> >  import logging
> > +import os
> >  import pathlib
> >  import sys
> >
> > -def generate_crates(srctree, objtree, sysroot_src):
> > +def generate_crates(srctree, objtree, sysroot_src, external_src):
> >      # Generate the configuration list.
> >      cfg = []
> >      with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
> > @@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
> >          [],
> >          is_proc_macro=True,
> >      )
> > -    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
> > +    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
> >
> >      append_crate(
> >          "build_error",
> > @@ -95,25 +96,28 @@ def generate_crates(srctree, objtree, sysroot_src):
> >          "exclude_dirs": [],
> >      }
> >
> > +    def is_root_crate(build_file, target):
> > +        return os.path.exists(build_file) and target in open(build_file).read()
> > +
> >      # Then, the rest outside of `rust/`.
> >      #
> >      # We explicitly mention the top-level folders we want to cover.
> 
> 
> Huh, not maintainable, unfortunately.
> 
> 
> 
> 
> 
> > -    for folder in ("samples", "drivers"):
> > +    for folder in ("samples", "drivers") if external_src is None else [external_src]:
> >          for path in (srctree / folder).rglob("*.rs"):
> 
> 
> 
> It is odd to add 'srctree' prefix to external module sources.
For external module sources, external_src is an absolute path and hence
srctree is ignored in this call.
> 
> 
> 
> I think rust-project.json is a similar concept to
> compile_commands.json, but it was implemented
> in a different way.
> 
I have not included the changes mentioned to refactor the rust-analyzer
target along the lines of how compile_commands.json has been solved
since it was beyond the scope of this changeset. However, I can take
this up as a follow up changeset.
> 
> 
> 
> 
> 
> 
> -- 
> Best Regards
> Masahiro Yamada
Re: [PATCH v2] scripts: `make rust-analyzer` for out-of-tree modules
Posted by Miguel Ojeda 2 years, 6 months ago
On Tue, Mar 7, 2023 at 4:28 PM Vinay Varma <varmavinaym@gmail.com> wrote:
>
> Sorry, got caught up with another project and lost track of this thread.
> I have updated the patch and replied to some of the threads inline.

No problem -- thanks!

Note that we may need a rebase after
https://lore.kernel.org/rust-for-linux/20230307120736.75492-1-nmi@metaspace.dk/
(or equivalent) lands in `rust-fixes`. So perhaps wait for that, then
rebase (especially if we go with the EAFP style), and then others may
want to give it a spin for a re-test.

When you rebase, please consider putting `f"{name}.o"` inside the
function to avoid repetition, and perhaps consider inverting the
condition to avoid indenting the file further (and reducing the diff).
It makes the "Skip ..." comment more understandable (otherwise, you
may want to change the comment to "Only process ..." or similar
instead of using "Skip ...").

Cheers,
Miguel
Re: [PATCH v2] scripts: `make rust-analyzer` for out-of-tree modules
Posted by Miguel Ojeda 2 years, 5 months ago
On Tue, Mar 7, 2023 at 8:55 PM Miguel Ojeda
<miguel.ojeda.sandonis@gmail.com> wrote:
>
> No problem -- thanks!
>
> Note that we may need a rebase after
> https://lore.kernel.org/rust-for-linux/20230307120736.75492-1-nmi@metaspace.dk/
> (or equivalent) lands in `rust-fixes`. So perhaps wait for that, then
> rebase (especially if we go with the EAFP style), and then others may
> want to give it a spin for a re-test.
>
> When you rebase, please consider putting `f"{name}.o"` inside the
> function to avoid repetition, and perhaps consider inverting the
> condition to avoid indenting the file further (and reducing the diff).
> It makes the "Skip ..." comment more understandable (otherwise, you
> may want to change the comment to "Only process ..." or similar
> instead of using "Skip ...").

The EAFP style change landed in `rust-fixes` at
https://github.com/Rust-for-Linux/linux/commit/5c7548d5a25306dcdb97689479be81cacc8ce596.

Cheers,
Miguel
[PATCH v5] scripts: `make rust-analyzer` for out-of-tree modules
Posted by Vinay Varma 2 years, 5 months ago
Adds support for out-of-tree rust modules to use the `rust-analyzer`
make target to generate the rust-project.json file.

The change involves adding an optional parameter `external_src` to the
`generate_rust_analyzer.py` which expects the path to the out-of-tree
module's source directory. When this parameter is passed, I have chosen
not to add the non-core modules (samples and drivers) into the result
since these are not expected to be used in third party modules. Related
changes are also made to the Makefile and rust/Makefile allowing the
`rust-analyzer` target to be used for out-of-tree modules as well.

Link: https://github.com/Rust-for-Linux/linux/pull/914
Link: https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2

Signed-off-by: Vinay Varma <varmavinaym@gmail.com>
---
 Makefile                          | 11 ++++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 27 ++++++++++++++++++---------
 3 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/Makefile b/Makefile
index ef4e96b9cd5b..1c57360e14f2 100644
--- a/Makefile
+++ b/Makefile
@@ -1856,11 +1856,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-	$(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------
 
@@ -1921,6 +1916,7 @@ help:
 	@echo  '  modules         - default target, build the module(s)'
 	@echo  '  modules_install - install the module'
 	@echo  '  clean           - remove generated files in module directory only'
+	@echo  '  rust-analyzer	  - generate rust-project.json rust-analyzer support file'
 	@echo  ''
 
 endif # KBUILD_EXTMOD
@@ -2058,6 +2054,11 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
 	$(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+	$(Q)$(MAKE) $(build)=rust $@
+
 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------
 
diff --git a/rust/Makefile b/rust/Makefile
index aef85e9e8eeb..45c92acd13bc 100644
--- a/rust/Makefile
+++ b/rust/Makefile
@@ -360,8 +360,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 	$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-	$(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-		$(RUST_LIB_SRC) > $(objtree)/rust-project.json
+	$(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+		$(abs_srctree) $(abs_objtree) \
+		$(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+		$(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
 
 redirect-intrinsics = \
 	__eqsf2 __gesf2 __lesf2 __nesf2 __unordsf2 \
diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
index 946e250c1b2a..848fa1ad92ba 100755
--- a/scripts/generate_rust_analyzer.py
+++ b/scripts/generate_rust_analyzer.py
@@ -6,10 +6,11 @@
 import argparse
 import json
 import logging
+import os
 import pathlib
 import sys
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",
@@ -95,19 +96,26 @@ def generate_crates(srctree, objtree, sysroot_src):
         "exclude_dirs": [],
     }
 
+    def is_root_crate(build_file, target):
+        try:
+            return f"{target}.o" in open(build_file).read()
+        except FileNotFoundError:
+            return False
+
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    for folder in ("samples", "drivers"):
-        for path in (srctree / folder).rglob("*.rs"):
+    extra_dirs = map(lambda dir: srctree / dir, ("samples", "drivers"))
+    if external_src is not None:
+        extra_dirs = [external_src]
+    for folder in extra_dirs:
+        for path in folder.rglob("*.rs"):
             logging.info("Checking %s", path)
             name = path.name.replace(".rs", "")
 
             # Skip those that are not crate roots.
-            try:
-                if f"{name}.o" not in open(path.parent / "Makefile").read():
-                    continue
-            except FileNotFoundError:
+            if not is_root_crate(path.parent / "Makefile", name) and \
+               not is_root_crate(path.parent / "Kbuild", name):
                 continue
 
             logging.info("Adding %s", name)
@@ -126,6 +134,7 @@ def main():
     parser.add_argument("srctree", type=pathlib.Path)
     parser.add_argument("objtree", type=pathlib.Path)
     parser.add_argument("sysroot_src", type=pathlib.Path)
+    parser.add_argument("exttree", type=pathlib.Path, nargs="?")
     args = parser.parse_args()
 
     logging.basicConfig(
@@ -134,7 +143,7 @@ def main():
     )
 
     rust_project = {
-        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
         "sysroot_src": str(args.sysroot_src),
     }
 
-- 
2.40.0
Re: [PATCH v5] scripts: `make rust-analyzer` for out-of-tree modules
Posted by Miguel Ojeda 2 years, 1 month ago
On Tue, Apr 11, 2023 at 11:19 AM Vinay Varma <varmavinaym@gmail.com> wrote:
>
> Adds support for out-of-tree rust modules to use the `rust-analyzer`
> make target to generate the rust-project.json file.
>
> The change involves adding an optional parameter `external_src` to the
> `generate_rust_analyzer.py` which expects the path to the out-of-tree
> module's source directory. When this parameter is passed, I have chosen
> not to add the non-core modules (samples and drivers) into the result
> since these are not expected to be used in third party modules. Related
> changes are also made to the Makefile and rust/Makefile allowing the
> `rust-analyzer` target to be used for out-of-tree modules as well.
>
> Link: https://github.com/Rust-for-Linux/linux/pull/914
> Link: https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2
>
> Signed-off-by: Vinay Varma <varmavinaym@gmail.com>

Applied to `rust-next`, thanks! I tested it and the generation of the
file seems to work on the sample out-of-tree module. We can do the
other improvements later on. I removed the newline above your SoB, by
the way.

Cheers,
Miguel