From nobody Fri Dec 19 16:05:25 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9AF142673BD; Tue, 8 Apr 2025 10:09:56 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1744106996; cv=none; b=c6j+564gTXm7E4bL5UwgJfGocnjyGQ4sF99Q6bXUOlE6vUm42X+wAaABRlJH/V9JpPeLGOAyFyk+NyQ4Yq/M7BIjpPeu+pZjWZxIXWZ5myuQOIZGPWsEP4aTvVCQgaSEQg7owbZGCWCEfwIoIdmCrqeOTBvvPNoq35hcTkYKFeA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1744106996; c=relaxed/simple; bh=MVRuaiqzwNT41sqHa7MqwAn2eO+Z5qjUAPM1cDdmHsA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=nKLzf6ZYiuhTDi0+GLvsBlZbfPtVuriod9Y+fPjWTv9KrbARLXkK0/tTT2MYuskQHAu7obznHLbbt27o5XnW/KKRxzNyhgYDLrzSM9Oo4c0nUsmM663JFgjlKdnzlHE43Dcmc0pBCgggwTIZm+Zu5D6exVdKvRijv+fKud8wIMM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=AUz/Ozyf; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="AUz/Ozyf" Received: by smtp.kernel.org (Postfix) with ESMTPSA id D7AD3C4CEF5; Tue, 8 Apr 2025 10:09:55 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1744106996; bh=MVRuaiqzwNT41sqHa7MqwAn2eO+Z5qjUAPM1cDdmHsA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=AUz/OzyfgFxI4AJ3JQd72trrR1cJ+kLo7xlTZLaAgQTQZ+qmFvZxcCU7hhlw6SF4V 39WoFttJMesOd5HRcIqijDrd4pHbl75n/UUo0XE3EVYGOGVM1TCHq8sF5lk+7RQV2p HQ2NwC1flRfxxcG17wuIMx2PgaIUpyWk8ipLlbFnYeg/JvwELncppfZoq0xRYtwqHd 3c4MppF9oi1WZno/9Ynapzpj62bt7/pT6HBZPnHQscXWT28csOEossGphkq7edFdQm 0IrbmcWQ7yyP9dDlJvSpnCfS+ebJlCd6NSpL8NrSjr51Htrpe9aoyqFClYjjNy0EIY MroabyA47rexw== Received: from mchehab by mail.kernel.org with local (Exim 4.98.2) (envelope-from ) id 1u25tt-00000008RVg-0fct; Tue, 08 Apr 2025 18:09:49 +0800 From: Mauro Carvalho Chehab To: Linux Doc Mailing List , Jonathan Corbet Cc: Mauro Carvalho Chehab , linux-kernel@vger.kernel.org Subject: [PATCH v3 09/33] scripts/kernel-doc.py: move KernelFiles class to a separate file Date: Tue, 8 Apr 2025 18:09:12 +0800 Message-ID: <80bc855e128a9ff0a11df5afe9ba71775dfc9a0f.1744106241.git.mchehab+huawei@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: Mauro Carvalho Chehab Content-Type: text/plain; charset="utf-8" The KernelFiles class is the main dispatcher which parses each source file. In preparation for letting kerneldoc Sphinx extension to import Python libraries, move regex ancillary classes to a separate file. Signed-off-by: Mauro Carvalho Chehab --- scripts/kernel-doc.py | 220 +-------------------------- scripts/lib/kdoc/kdoc_files.py | 270 +++++++++++++++++++++++++++++++++ 2 files changed, 271 insertions(+), 219 deletions(-) create mode 100755 scripts/lib/kdoc/kdoc_files.py diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index f030a36a165b..d09ada2d862a 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -119,6 +119,7 @@ sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) =20 from kdoc_parser import KernelDoc, type_param from kdoc_re import Re +from kdoc_files import KernelFiles =20 function_pointer =3D Re(r"([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)", cache=3DFalse) =20 @@ -143,225 +144,6 @@ type_member =3D Re(r"\&([_\w]+)(\.|->)([_\w]+)", cach= e=3DFalse) type_fallback =3D Re(r"\&([_\w]+)", cache=3DFalse) type_member_func =3D type_member + Re(r"\(\)", cache=3DFalse) =20 -class GlobSourceFiles: - """ - Parse C source code file names and directories via an Interactor. - - """ - - def __init__(self, srctree=3DNone, valid_extensions=3DNone): - """ - Initialize valid extensions with a tuple. - - If not defined, assume default C extensions (.c and .h) - - It would be possible to use python's glob function, but it is - very slow, and it is not interactive. So, it would wait to read all - directories before actually do something. - - So, let's use our own implementation. - """ - - if not valid_extensions: - self.extensions =3D (".c", ".h") - else: - self.extensions =3D valid_extensions - - self.srctree =3D srctree - - def _parse_dir(self, dirname): - """Internal function to parse files recursively""" - - with os.scandir(dirname) as obj: - for entry in obj: - name =3D os.path.join(dirname, entry.name) - - if entry.is_dir(): - yield from self._parse_dir(name) - - if not entry.is_file(): - continue - - basename =3D os.path.basename(name) - - if not basename.endswith(self.extensions): - continue - - yield name - - def parse_files(self, file_list, file_not_found_cb): - for fname in file_list: - if self.srctree: - f =3D os.path.join(self.srctree, fname) - else: - f =3D fname - - if os.path.isdir(f): - yield from self._parse_dir(f) - elif os.path.isfile(f): - yield f - elif file_not_found_cb: - file_not_found_cb(fname) - - -class KernelFiles(): - - def parse_file(self, fname): - - doc =3D KernelDoc(self.config, fname) - doc.run() - - return doc - - def process_export_file(self, fname): - try: - with open(fname, "r", encoding=3D"utf8", - errors=3D"backslashreplace") as fp: - for line in fp: - KernelDoc.process_export(self.config.function_table, l= ine) - - except IOError: - print(f"Error: Cannot open fname {fname}", fname=3Dsys.stderr) - self.config.errors +=3D 1 - - def file_not_found_cb(self, fname): - self.config.log.error("Cannot find file %s", fname) - self.config.errors +=3D 1 - - def __init__(self, files=3DNone, verbose=3DFalse, out_style=3DNone, - werror=3DFalse, wreturn=3DFalse, wshort_desc=3DFalse, - wcontents_before_sections=3DFalse, - logger=3DNone, modulename=3DNone, export_file=3DNone): - """Initialize startup variables and parse all files""" - - - if not verbose: - verbose =3D bool(os.environ.get("KBUILD_VERBOSE", 0)) - - if not modulename: - modulename =3D "Kernel API" - - dt =3D datetime.now() - if os.environ.get("KBUILD_BUILD_TIMESTAMP", None): - # use UTC TZ - to_zone =3D tz.gettz('UTC') - dt =3D dt.astimezone(to_zone) - - if not werror: - kcflags =3D os.environ.get("KCFLAGS", None) - if kcflags: - match =3D re.search(r"(\s|^)-Werror(\s|$)/", kcflags) - if match: - werror =3D True - - # reading this variable is for backwards compat just in case - # someone was calling it with the variable from outside the - # kernel's build system - kdoc_werror =3D os.environ.get("KDOC_WERROR", None) - if kdoc_werror: - werror =3D kdoc_werror - - # Set global config data used on all files - self.config =3D argparse.Namespace - - self.config.verbose =3D verbose - self.config.werror =3D werror - self.config.wreturn =3D wreturn - self.config.wshort_desc =3D wshort_desc - self.config.wcontents_before_sections =3D wcontents_before_sections - self.config.modulename =3D modulename - - self.config.function_table =3D set() - self.config.source_map =3D {} - - if not logger: - self.config.log =3D logging.getLogger("kernel-doc") - else: - self.config.log =3D logger - - self.config.kernel_version =3D os.environ.get("KERNELVERSION", - "unknown kernel versio= n'") - self.config.src_tree =3D os.environ.get("SRCTREE", None) - - self.out_style =3D out_style - self.export_file =3D export_file - - # Initialize internal variables - - self.config.errors =3D 0 - self.results =3D [] - - self.file_list =3D files - self.files =3D set() - - def parse(self): - """ - Parse all files - """ - - glob =3D GlobSourceFiles(srctree=3Dself.config.src_tree) - - # Let's use a set here to avoid duplicating files - - for fname in glob.parse_files(self.file_list, self.file_not_found_= cb): - if fname in self.files: - continue - - self.files.add(fname) - - res =3D self.parse_file(fname) - self.results.append((res.fname, res.entries)) - - if not self.files: - sys.exit(1) - - # If a list of export files was provided, parse EXPORT_SYMBOL* - # from the ones not already parsed - - if self.export_file: - files =3D self.files - - glob =3D GlobSourceFiles(srctree=3Dself.config.src_tree) - - for fname in glob.parse_files(self.export_file, - self.file_not_found_cb): - if fname not in files: - files.add(fname) - - self.process_export_file(fname) - - def out_msg(self, fname, name, arg): - # TODO: filter out unwanted parts - - return self.out_style.msg(fname, name, arg) - - def msg(self, enable_lineno=3DFalse, export=3DFalse, internal=3DFalse, - symbol=3DNone, nosymbol=3DNone): - - function_table =3D self.config.function_table - - if symbol: - for s in symbol: - function_table.add(s) - - # Output none mode: only warnings will be shown - if not self.out_style: - return - - self.out_style.set_config(self.config) - - self.out_style.set_filter(export, internal, symbol, nosymbol, - function_table, enable_lineno) - - for fname, arg_tuple in self.results: - for name, arg in arg_tuple: - if self.out_msg(fname, name, arg): - ln =3D arg.get("ln", 0) - dtype =3D arg.get('type', "") - - self.config.log.warning("%s:%d Can't handle %s", - fname, ln, dtype) - =20 class OutputFormat: # output mode. diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py new file mode 100755 index 000000000000..8bcdc7ead984 --- /dev/null +++ b/scripts/lib/kdoc/kdoc_files.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025: Mauro Carvalho Chehab . +# +# pylint: disable=3DR0903,R0913,R0914,R0917 + +# TODO: implement warning filtering + +""" +Parse lernel-doc tags on multiple kernel source files. +""" + +import argparse +import logging +import os +import re +import sys +from datetime import datetime + +from dateutil import tz + +from kdoc_parser import KernelDoc + + +class GlobSourceFiles: + """ + Parse C source code file names and directories via an Interactor. + """ + + def __init__(self, srctree=3DNone, valid_extensions=3DNone): + """ + Initialize valid extensions with a tuple. + + If not defined, assume default C extensions (.c and .h) + + It would be possible to use python's glob function, but it is + very slow, and it is not interactive. So, it would wait to read all + directories before actually do something. + + So, let's use our own implementation. + """ + + if not valid_extensions: + self.extensions =3D (".c", ".h") + else: + self.extensions =3D valid_extensions + + self.srctree =3D srctree + + def _parse_dir(self, dirname): + """Internal function to parse files recursively""" + + with os.scandir(dirname) as obj: + for entry in obj: + name =3D os.path.join(dirname, entry.name) + + if entry.is_dir(): + yield from self._parse_dir(name) + + if not entry.is_file(): + continue + + basename =3D os.path.basename(name) + + if not basename.endswith(self.extensions): + continue + + yield name + + def parse_files(self, file_list, file_not_found_cb): + """ + Define an interator to parse all source files from file_list, + handling directories if any + """ + + for fname in file_list: + if self.srctree: + f =3D os.path.join(self.srctree, fname) + else: + f =3D fname + + if os.path.isdir(f): + yield from self._parse_dir(f) + elif os.path.isfile(f): + yield f + elif file_not_found_cb: + file_not_found_cb(fname) + + +class KernelFiles(): + """ + Parse lernel-doc tags on multiple kernel source files. + """ + + def parse_file(self, fname): + """ + Parse a single Kernel source. + """ + + doc =3D KernelDoc(self.config, fname) + doc.run() + + return doc + + def process_export_file(self, fname): + """ + Parses EXPORT_SYMBOL* macros from a single Kernel source file. + """ + try: + with open(fname, "r", encoding=3D"utf8", + errors=3D"backslashreplace") as fp: + for line in fp: + KernelDoc.process_export(self.config.function_table, l= ine) + + except IOError: + print(f"Error: Cannot open fname {fname}", fname=3Dsys.stderr) + self.config.errors +=3D 1 + + def file_not_found_cb(self, fname): + """ + Callback to warn if a file was not found. + """ + + self.config.log.error("Cannot find file %s", fname) + self.config.errors +=3D 1 + + def __init__(self, files=3DNone, verbose=3DFalse, out_style=3DNone, + werror=3DFalse, wreturn=3DFalse, wshort_desc=3DFalse, + wcontents_before_sections=3DFalse, + logger=3DNone, modulename=3DNone, export_file=3DNone): + """ + Initialize startup variables and parse all files + """ + + if not verbose: + verbose =3D bool(os.environ.get("KBUILD_VERBOSE", 0)) + + if not modulename: + modulename =3D "Kernel API" + + dt =3D datetime.now() + if os.environ.get("KBUILD_BUILD_TIMESTAMP", None): + # use UTC TZ + to_zone =3D tz.gettz('UTC') + dt =3D dt.astimezone(to_zone) + + if not werror: + kcflags =3D os.environ.get("KCFLAGS", None) + if kcflags: + match =3D re.search(r"(\s|^)-Werror(\s|$)/", kcflags) + if match: + werror =3D True + + # reading this variable is for backwards compat just in case + # someone was calling it with the variable from outside the + # kernel's build system + kdoc_werror =3D os.environ.get("KDOC_WERROR", None) + if kdoc_werror: + werror =3D kdoc_werror + + # Set global config data used on all files + self.config =3D argparse.Namespace + + self.config.verbose =3D verbose + self.config.werror =3D werror + self.config.wreturn =3D wreturn + self.config.wshort_desc =3D wshort_desc + self.config.wcontents_before_sections =3D wcontents_before_sections + self.config.modulename =3D modulename + + self.config.function_table =3D set() + self.config.source_map =3D {} + + if not logger: + self.config.log =3D logging.getLogger("kernel-doc") + else: + self.config.log =3D logger + + self.config.kernel_version =3D os.environ.get("KERNELVERSION", + "unknown kernel versio= n'") + self.config.src_tree =3D os.environ.get("SRCTREE", None) + + self.out_style =3D out_style + self.export_file =3D export_file + + # Initialize internal variables + + self.config.errors =3D 0 + self.results =3D [] + + self.file_list =3D files + self.files =3D set() + + def parse(self): + """ + Parse all files + """ + + glob =3D GlobSourceFiles(srctree=3Dself.config.src_tree) + + # Let's use a set here to avoid duplicating files + + for fname in glob.parse_files(self.file_list, self.file_not_found_= cb): + if fname in self.files: + continue + + self.files.add(fname) + + res =3D self.parse_file(fname) + self.results.append((res.fname, res.entries)) + + if not self.files: + sys.exit(1) + + # If a list of export files was provided, parse EXPORT_SYMBOL* + # from the ones not already parsed + + if self.export_file: + files =3D self.files + + glob =3D GlobSourceFiles(srctree=3Dself.config.src_tree) + + for fname in glob.parse_files(self.export_file, + self.file_not_found_cb): + if fname not in files: + files.add(fname) + + self.process_export_file(fname) + + def out_msg(self, fname, name, arg): + """ + Output messages from a file name using the output style filtering. + + If output type was not handled by the syler, return False. + """ + + # NOTE: we can add rules here to filter out unwanted parts, + # although OutputFormat.msg already does that. + + return self.out_style.msg(fname, name, arg) + + def msg(self, enable_lineno=3DFalse, export=3DFalse, internal=3DFalse, + symbol=3DNone, nosymbol=3DNone): + """ + Interacts over the kernel-doc results and output messages. + """ + + function_table =3D self.config.function_table + + if symbol: + for s in symbol: + function_table.add(s) + + # Output none mode: only warnings will be shown + if not self.out_style: + return + + self.out_style.set_config(self.config) + + self.out_style.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno) + + for fname, arg_tuple in self.results: + for name, arg in arg_tuple: + if self.out_msg(fname, name, arg): + ln =3D arg.get("ln", 0) + dtype =3D arg.get('type', "") + + self.config.log.warning("%s:%d Can't handle %s", + fname, ln, dtype) --=20 2.49.0