From nobody Tue Apr 7 19:37:27 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 10C713AD53C; Thu, 12 Mar 2026 07:12:34 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1773299554; cv=none; b=rGfXrhJq3/msn+Vv9e0pKewi9fy+A5EII0XEo3VmXNSZS6nDk1Sgut8qCl1D9r3L9NkXnk5AWGZE2ycmU3KytthWTVH8NmZFhAzk8JI/xE23w+fA4cUqdXIm8TE4DZq+d8tgA69xIl8hcbQrvwGx5poHayvkyGKEnc5BnGkuZQg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1773299554; c=relaxed/simple; bh=Znhgh5VpMegTURFUzFsOSwJlqZmT0mMIAtISYNXYSXM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=Mol0A1MnoIMvNSU6ttph7BYbG0c17jkV/kqA6jikNiXpFTna8J89c1DhBoLxEbU9m3wCCR/9ocJvFp9fuCp4uEJPIXoftdiZ3bfSibO6SLTa8vQdBcfklorRKaJrMX6JSweaal4liBLNAwqcfoF5LgCQQrGo5lOCXW4rRneBjOM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=S8Ra4+cy; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="S8Ra4+cy" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 64416C2BCB3; Thu, 12 Mar 2026 07:12:33 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1773299553; bh=Znhgh5VpMegTURFUzFsOSwJlqZmT0mMIAtISYNXYSXM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=S8Ra4+cyC4cAuvdogezwt+o/z6U138dgnco6Lr6PABVV2eDL3RarhuTdCZqqKAqQF hmL0ErRTpCt/L8WNZJ+vvwfiXRfybStNmQJZFnnV1+n1O0gxsjb9XZL0eRnODSZKzg X/GjSdZKqtJ1qJ3s8QGk5FSeDlL7G3FPjArfDicI+SHJmTuh8dtdaGEyVLvNW1FRal DpoDkfvroqXjNbearS+lZi603mR+jtZeiHwJXo6qDGdJXpd4FWxhVRfK3fGWROGcQG V7gmY8ufafEM20tmB0bW4IHkZarmkT1P9x4N1nhUEQikNGsL4hEfSZfHZmRraOy1OW umwd3OxIpT3Nw== Received: from mchehab by mail.kernel.org with local (Exim 4.99.1) (envelope-from ) id 1w0aDf-000000077gK-1lGt; Thu, 12 Mar 2026 08:12:31 +0100 From: Mauro Carvalho Chehab To: Jonathan Corbet , Linux Doc Mailing List Cc: Mauro Carvalho Chehab , linux-hardening@vger.kernel.org, linux-kernel@vger.kernel.org, Aleksandr Loktionov , Mauro Carvalho Chehab , Randy Dunlap Subject: [PATCH v2 05/20] docs: kdoc_re: add a C tokenizer Date: Thu, 12 Mar 2026 08:12:13 +0100 Message-ID: <8541ffa469647db1a7154f274fb2d55b4c127dcb.1773297828.git.mchehab+huawei@kernel.org> X-Mailer: git-send-email 2.53.0 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Sender: Mauro Carvalho Chehab Handling C code purely using regular expressions doesn't work well. Add a C tokenizer to help doing it the right way. The tokenizer was written using as basis the Python re documentation tokenizer example from: https://docs.python.org/3/library/re.html#writing-a-tokenizer Signed-off-by: Mauro Carvalho Chehab Message-ID: --- tools/lib/python/kdoc/kdoc_re.py | 234 +++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_= re.py index 085b89a4547c..7bed4e9a8810 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -141,6 +141,240 @@ class KernRe: =20 return self.last_match.groups() =20 +class TokType(): + + @staticmethod + def __str__(val): + """Return the name of an enum value""" + return TokType._name_by_val.get(val, f"UNKNOWN({val})") + +class CToken(): + """ + Data class to define a C token. + """ + + # Tokens that can be used by the parser. Works like an C enum. + + COMMENT =3D 0 #: A standard C or C99 comment, including delimiter. + STRING =3D 1 #: A string, including quotation marks. + CHAR =3D 2 #: A character, including apostophes. + NUMBER =3D 3 #: A number. + PUNC =3D 4 #: A puntuation mark: ``;`` / ``,`` / ``.``. + BEGIN =3D 5 #: A begin character: ``{`` / ``[`` / ``(``. + END =3D 6 #: A end character: ``}`` / ``]`` / ``)``. + CPP =3D 7 #: A preprocessor macro. + HASH =3D 8 #: The hash character - useful to handle other macro= s. + OP =3D 9 #: A C operator (add, subtract, ...). + STRUCT =3D 10 #: A ``struct`` keyword. + UNION =3D 11 #: An ``union`` keyword. + ENUM =3D 12 #: A ``struct`` keyword. + TYPEDEF =3D 13 #: A ``typedef`` keyword. + NAME =3D 14 #: A name. Can be an ID or a type. + SPACE =3D 15 #: Any space characters, including new lines + + MISMATCH =3D 255 #: an error indicator: should never happen in practi= ce. + + # Dict to convert from an enum interger into a string. + _name_by_val =3D {v: k for k, v in dict(vars()).items() if isinstance(= v, int)} + + # Dict to convert from string to an enum-like integer value. + _name_to_val =3D {k: v for v, k in _name_by_val.items()} + + @staticmethod + def to_name(val): + """Convert from an integer value from CToken enum into a string""" + + return CToken._name_by_val.get(val, f"UNKNOWN({val})") + + @staticmethod + def from_name(name): + """Convert a string into a CToken enum value""" + if name in CToken._name_to_val: + return CToken._name_to_val[name] + + return CToken.MISMATCH + + def __init__(self, kind, value, pos, + brace_level, paren_level, bracket_level): + self.kind =3D kind + self.value =3D value + self.pos =3D pos + self.brace_level =3D brace_level + self.paren_level =3D paren_level + self.bracket_level =3D bracket_level + + def __repr__(self): + name =3D self.to_name(self.kind) + if isinstance(self.value, str): + value =3D '"' + self.value + '"' + else: + value =3D self.value + + return f"CToken({name}, {value}, {self.pos}, " \ + f"{self.brace_level}, {self.paren_level}, {self.bracket_lev= el})" + +#: Tokens to parse C code. +TOKEN_LIST =3D [ + (CToken.COMMENT, r"//[^\n]*|/\*[\s\S]*?\*/"), + + (CToken.STRING, r'"(?:\\.|[^"\\])*"'), + (CToken.CHAR, r"'(?:\\.|[^'\\])'"), + + (CToken.NUMBER, r"0[xX][0-9a-fA-F]+[uUlL]*|0[0-7]+[uUlL]*|" + r"[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?[fFlL]*"), + + (CToken.PUNC, r"[;,\.]"), + + (CToken.BEGIN, r"[\[\(\{]"), + + (CToken.END, r"[\]\)\}]"), + + (CToken.CPP, r"#\s*(define|include|ifdef|ifndef|if|else|elif|endif= |undef|pragma)\b"), + + (CToken.HASH, r"#"), + + (CToken.OP, r"\+\+|\-\-|\->|=3D=3D|\!=3D|<=3D|>=3D|&&|\|\||<<|>>|= \+=3D|\-=3D|\*=3D|/=3D|%=3D" + r"|&=3D|\|=3D|\^=3D|=3D|\+|\-|\*|/|%|<|>|&|\||\^|~|!|= \?|\:"), + + (CToken.STRUCT, r"\bstruct\b"), + (CToken.UNION, r"\bunion\b"), + (CToken.ENUM, r"\benum\b"), + (CToken.TYPEDEF, r"\bkinddef\b"), + + (CToken.NAME, r"[A-Za-z_][A-Za-z0-9_]*"), + + (CToken.SPACE, r"[\s]+"), + + (CToken.MISMATCH,r"."), +] + +#: Handle C continuation lines. +RE_CONT =3D KernRe(r"\\\n") + +RE_COMMENT_START =3D KernRe(r'/\*\s*') + +#: tokenizer regex. Will be filled at the first CTokenizer usage. +re_scanner =3D None + +class CTokenizer(): + """ + Scan C statements and definitions and produce tokens. + + When converted to string, it drops comments and handle public/private + values, respecting depth. + """ + + # This class is inspired and follows the basic concepts of: + # https://docs.python.org/3/library/re.html#writing-a-tokenizer + + def _tokenize(self, source): + """ + Interactor that parses ``source``, splitting it into tokens, as de= fined + at ``self.TOKEN_LIST``. + + The interactor returns a CToken class object. + """ + + # Handle continuation lines. Note that kdoc_parser already has a + # logic to do that. Still, let's keep it for completeness, as we m= ight + # end re-using this tokenizer outsize kernel-doc some day - or we = may + # eventually remove from there as a future cleanup. + source =3D RE_CONT.sub("", source) + + brace_level =3D 0 + paren_level =3D 0 + bracket_level =3D 0 + + for match in re_scanner.finditer(source): + kind =3D CToken.from_name(match.lastgroup) + pos =3D match.start() + value =3D match.group() + + if kind =3D=3D CToken.MISMATCH: + raise RuntimeError(f"Unexpected token '{value}' on {pos}:\= n\t{source}") + elif kind =3D=3D CToken.BEGIN: + if value =3D=3D '(': + paren_level +=3D 1 + elif value =3D=3D '[': + bracket_level +=3D 1 + else: # value =3D=3D '{' + brace_level +=3D 1 + + elif kind =3D=3D CToken.END: + if value =3D=3D ')' and paren_level > 0: + paren_level -=3D 1 + elif value =3D=3D ']' and bracket_level > 0: + bracket_level -=3D 1 + elif brace_level > 0: # value =3D=3D '}' + brace_level -=3D 1 + + yield CToken(kind, value, pos, + brace_level, paren_level, bracket_level) + + def __init__(self, source): + """ + Create a regular expression to handle TOKEN_LIST. + + While I generally don't like using regex group naming via: + (?P...) + + in this particular case, it makes sense, as we can pick the name + when matching a code via re_scanner(). + """ + global re_scanner + + if not re_scanner: + re_tokens =3D [] + + for kind, pattern in TOKEN_LIST: + name =3D CToken.to_name(kind) + re_tokens.append(f"(?P<{name}>{pattern})") + + re_scanner =3D KernRe("|".join(re_tokens), re.MULTILINE | re.D= OTALL) + + self.tokens =3D [] + for tok in self._tokenize(source): + self.tokens.append(tok) + + def __str__(self): + out=3D"" + show_stack =3D [True] + + for tok in self.tokens: + if tok.kind =3D=3D CToken.BEGIN: + show_stack.append(show_stack[-1]) + + elif tok.kind =3D=3D CToken.END: + prev =3D show_stack[-1] + if len(show_stack) > 1: + show_stack.pop() + + if not prev and show_stack[-1]: + # + # Try to preserve indent + # + out +=3D "\t" * (len(show_stack) - 1) + + out +=3D str(tok.value) + continue + + elif tok.kind =3D=3D CToken.COMMENT: + comment =3D RE_COMMENT_START.sub("", tok.value) + + if comment.startswith("private:"): + show_stack[-1] =3D False + show =3D False + elif comment.startswith("public:"): + show_stack[-1] =3D True + + continue + + if show_stack[-1]: + out +=3D str(tok.value) + + return out + + #: Nested delimited pairs (brackets and parenthesis) DELIMITER_PAIRS =3D { '{': '}', --=20 2.53.0