From nobody Thu Oct 9 02:52:17 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2080E192590; Sun, 22 Jun 2025 06:02:51 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1750572172; cv=none; b=pAlvis6GajdglvagZKRmxPXGQ0Ocu+UlfXXkFVVDPkrvRQXPAWmkRx2b7iwDtLEfBhbWKh9OtmQu+s/G+y4MX//M0ku2byxAKaDb1zPl/41BkauZ3jtPKWLjr/bhpRNUmx9D04aGHczae2C3ODmYIP716a/ZlXz1SuG2lpJ2Kjg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1750572172; c=relaxed/simple; bh=XT+7KR3Tg4iAhlUU3c6mKrOGy1MIPvbKyEPBCyZpAmw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=tBEO7o75fPS1nIlaIRU9IdLHXvSjK8qDa68GCHRBy00TTNSirR53hhIK+MQA3I4Bpg5BDRAqsjzGA6rJJRRlwXh3ElF6dBHYLdlP7xckFfLLf11E5YRXoKlis/Eu0iH1ccNhS+yDXLXIX67Nq2r8g7XmMRUXEGw5WI7HB5CG2aM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=rEFcxNQE; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="rEFcxNQE" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 317A3C4CEF2; Sun, 22 Jun 2025 06:02:51 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1750572171; bh=XT+7KR3Tg4iAhlUU3c6mKrOGy1MIPvbKyEPBCyZpAmw=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=rEFcxNQEoRDQPdcN+3RTZPO2y1AS7EBTVHtql9cqKTurvXAQODelenF4AygqSHKwN 2NOiIGTy9wgVbIBmDdlIyQVyiWEJur7wAK5R8aq4eTdA+9m5C96wzwnja2kZFqLyeW ++jnZ+Ja5SdxYvWj9nCTQd/Q/J/6XK6fD4dO1s20Os5HpAa/wUA7w+3NeX+vIiX4uJ 1LcTnvULm+xnxuSVQ2hN1S81q/Gzcs1jf+IhF02ly8fAH8lDbYz40tpkfVm0ICTh5p 3oAk0pLaakeTPMbQd4bh8cJqwd0Wf0rgeWRId9yX4kQxzNOj3V69XI1RKGlpctqlB2 e3szlHPdM4uQw== Received: from mchehab by mail.kernel.org with local (Exim 4.98.2) (envelope-from ) id 1uTDmz-00000000o2G-1a2O; Sun, 22 Jun 2025 08:02:49 +0200 From: Mauro Carvalho Chehab To: Linux Doc Mailing List , Jonathan Corbet Cc: Mauro Carvalho Chehab , "Akira Yokosawa" , "Mauro Carvalho Chehab" , linux-kernel@vger.kernel.org Subject: [PATCH v3 04/15] scripts: test_doc_build.py: make capture assynchronous Date: Sun, 22 Jun 2025 08:02:33 +0200 Message-ID: <9b0a60b5047137b5ba764701268da992767b128c.1750571906.git.mchehab+huawei@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: Mauro Carvalho Chehab Content-Type: text/plain; charset="utf-8" Prepare the tool to allow writing the output into log files. For such purpose, receive stdin/stdout messages asynchronously. Signed-off-by: Mauro Carvalho Chehab --- scripts/test_doc_build.py | 393 +++++++++++++++++++++++++------------- 1 file changed, 255 insertions(+), 138 deletions(-) diff --git a/scripts/test_doc_build.py b/scripts/test_doc_build.py index 482716fbe91d..94f2f2d8c3b7 100755 --- a/scripts/test_doc_build.py +++ b/scripts/test_doc_build.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright(c) 2025: Mauro Carvalho Chehab # -# pylint: disable=3DC0103,R1715 +# pylint: disable=3DR0903,R0913,R0914,R0917 =20 """ Install minimal supported requirements for different Sphinx versions @@ -10,20 +10,20 @@ and optionally test the build. """ =20 import argparse +import asyncio import os.path import sys import time - -from subprocess import run +import subprocess =20 # Minimal python version supported by the building system -python_bin =3D "python3.9" +MINIMAL_PYTHON_VERSION =3D "python3.9" =20 # Starting from 8.0.2, Python 3.9 becomes too old -python_changes =3D {(8, 0, 2): "python3"} +PYTHON_VER_CHANGES =3D {(8, 0, 2): "python3"} =20 # Sphinx versions to be installed and their incremental requirements -sphinx_requirements =3D { +SPHINX_REQUIREMENTS =3D { (3, 4, 3): { "alabaster": "0.7.13", "babel": "2.17.0", @@ -101,141 +101,258 @@ sphinx_requirements =3D { } =20 =20 +class AsyncCommands: + """Excecute command synchronously""" + + stdout =3D None + stderr =3D None + output =3D None + + async def _read(self, stream, verbose, is_info): + """Ancillary routine to capture while displaying""" + + while stream is not None: + line =3D await stream.readline() + if line: + out =3D line.decode("utf-8", errors=3D"backslashreplace") + self.output +=3D out + if is_info: + if verbose: + print(out.rstrip("\n")) + + self.stdout +=3D out + else: + if verbose: + print(out.rstrip("\n"), file=3Dsys.stderr) + + self.stderr +=3D out + else: + break + + async def run(self, cmd, capture_output=3DFalse, check=3DFalse, + env=3DNone, verbose=3DTrue): + + """ + Execute an arbitrary command, handling errors. + + Please notice that this class is not thread safe + """ + + self.stdout =3D "" + self.stderr =3D "" + self.output =3D "" + + if verbose: + print("$ ", " ".join(cmd)) + + proc =3D await asyncio.create_subprocess_exec(cmd[0], + *cmd[1:], + env=3Denv, + stdout=3Dasyncio.subpr= ocess.PIPE, + stderr=3Dasyncio.subpr= ocess.PIPE) + + # Handle input and output in realtime + await asyncio.gather( + self._read(proc.stdout, verbose, True), + self._read(proc.stderr, verbose, False), + ) + + await proc.wait() + + if check and proc.returncode > 0: + raise subprocess.CalledProcessError(returncode=3Dproc.returnco= de, + cmd=3D" ".join(cmd), + output=3Dself.stdout, + stderr=3Dself.stderr) + + if capture_output: + if proc.returncode > 0: + print("Error {proc.returncode}", file=3Dsys.stderr) + return "" + + return self.output + + ret =3D subprocess.CompletedProcess(args=3Dcmd, + returncode=3Dproc.returncode, + stdout=3Dself.stdout, + stderr=3Dself.stderr) + + return ret + + +class SphinxVenv: + """ + Installs Sphinx on one virtual env per Sphinx version with a minimal + set of dependencies, adjusting them to each specific version. + """ + + def __init__(self): + """Initialize instance variables""" + + self.built_time =3D {} + self.first_run =3D True + + async def _handle_version(self, args, cur_ver, cur_requirements, pytho= n_bin): + """Handle a single Sphinx version""" + + cmd =3D AsyncCommands() + + ver =3D ".".join(map(str, cur_ver)) + + if not self.first_run and args.wait_input and args.make: + ret =3D input("Press Enter to continue or 'a' to abort: ").str= ip().lower() + if ret =3D=3D "a": + print("Aborted.") + sys.exit() + else: + self.first_run =3D False + + venv_dir =3D f"Sphinx_{ver}" + req_file =3D f"requirements_{ver}.txt" + + print(f"\nSphinx {ver} with {python_bin}") + + # Create venv + await cmd.run([python_bin, "-m", "venv", venv_dir], check=3DTrue) + pip =3D os.path.join(venv_dir, "bin/pip") + + # Create install list + reqs =3D [] + for pkg, verstr in cur_requirements.items(): + reqs.append(f"{pkg}=3D=3D{verstr}") + + reqs.append(f"Sphinx=3D=3D{ver}") + + await cmd.run([pip, "install"] + reqs, check=3DTrue, verbose=3DTru= e) + + # Freeze environment + result =3D await cmd.run([pip, "freeze"], verbose=3DFalse, check= =3DTrue) + + # Pip install succeeded. Write requirements file + if args.write: + with open(req_file, "w", encoding=3D"utf-8") as fp: + fp.write(result.stdout) + + if args.make: + start_time =3D time.time() + + # Prepare a venv environment + env =3D os.environ.copy() + bin_dir =3D os.path.join(venv_dir, "bin") + env["PATH"] =3D bin_dir + ":" + env["PATH"] + env["VIRTUAL_ENV"] =3D venv_dir + if "PYTHONHOME" in env: + del env["PYTHONHOME"] + + # Test doc build + await cmd.run(["make", "cleandocs"], env=3Denv, check=3DTrue) + make =3D ["make"] + args.make_args + ["htmldocs"] + + print(f". {bin_dir}/activate") + print(" ".join(make)) + print("deactivate") + await cmd.run(make, env=3Denv, check=3DTrue) + + end_time =3D time.time() + elapsed_time =3D end_time - start_time + hours, minutes =3D divmod(elapsed_time, 3600) + minutes, seconds =3D divmod(minutes, 60) + + hours =3D int(hours) + minutes =3D int(minutes) + seconds =3D int(seconds) + + self.built_time[ver] =3D f"{hours:02d}:{minutes:02d}:{seconds:= 02d}" + + print(f"Finished doc build for Sphinx {ver}. Elapsed time: {se= lf.built_time[ver]}") + + async def run(self, args): + """ + Navigate though multiple Sphinx versions, handling each of them + on a loop. + """ + + cur_requirements =3D {} + python_bin =3D MINIMAL_PYTHON_VERSION + + for cur_ver, new_reqs in SPHINX_REQUIREMENTS.items(): + cur_requirements.update(new_reqs) + + if cur_ver in PYTHON_VER_CHANGES: # pylint: disable= =3DR1715 + + python_bin =3D PYTHON_VER_CHANGES[cur_ver] + + if args.min_version: + if cur_ver < args.min_version: + continue + + if args.max_version: + if cur_ver > args.max_version: + break + + await self._handle_version(args, cur_ver, cur_requirements, + python_bin) + + if args.make: + print() + print("Summary:") + for ver, elapsed_time in sorted(self.built_time.items()): + print(f"\tSphinx {ver} elapsed time: {elapsed_time}") + + def parse_version(ver_str): """Convert a version string into a tuple.""" =20 return tuple(map(int, ver_str.split("."))) =20 =20 -parser =3D argparse.ArgumentParser(description=3D"Build docs for different= sphinx_versions.") - -parser.add_argument('-v', '--version', help=3D'Sphinx single version', - type=3Dparse_version) -parser.add_argument('--min-version', "--min", help=3D'Sphinx minimal versi= on', - type=3Dparse_version) -parser.add_argument('--max-version', "--max", help=3D'Sphinx maximum versi= on', - type=3Dparse_version) -parser.add_argument('-a', '--make_args', - help=3D'extra arguments for make htmldocs, like SPHINX= DIRS=3Dnetlink/specs', - nargs=3D"*") -parser.add_argument('-w', '--write', help=3D'write a requirements.txt file= ', - action=3D'store_true') -parser.add_argument('-m', '--make', - help=3D'Make documentation', - action=3D'store_true') -parser.add_argument('-i', '--wait-input', - help=3D'Wait for an enter before going to the next ver= sion', - action=3D'store_true') - -args =3D parser.parse_args() - -if not args.make_args: - args.make_args =3D [] - -if args.version: - if args.min_version or args.max_version: - sys.exit("Use either --version or --min-version/--max-version") - else: - args.min_version =3D args.version - args.max_version =3D args.version - -sphinx_versions =3D sorted(list(sphinx_requirements.keys())) - -if not args.min_version: - args.min_version =3D sphinx_versions[0] - -if not args.max_version: - args.max_version =3D sphinx_versions[-1] - -first_run =3D True -cur_requirements =3D {} -built_time =3D {} - -for cur_ver, new_reqs in sphinx_requirements.items(): - cur_requirements.update(new_reqs) - - if cur_ver in python_changes: - python_bin =3D python_changes[cur_ver] - - ver =3D ".".join(map(str, cur_ver)) - - if args.min_version: - if cur_ver < args.min_version: - continue - - if args.max_version: - if cur_ver > args.max_version: - break - - if not first_run and args.wait_input and args.make: - ret =3D input("Press Enter to continue or 'a' to abort: ").strip()= .lower() - if ret =3D=3D "a": - print("Aborted.") - sys.exit() - else: - first_run =3D False - - venv_dir =3D f"Sphinx_{ver}" - req_file =3D f"requirements_{ver}.txt" - - print(f"\nSphinx {ver} with {python_bin}") - - # Create venv - run([python_bin, "-m", "venv", venv_dir], check=3DTrue) - pip =3D os.path.join(venv_dir, "bin/pip") - - # Create install list - reqs =3D [] - for pkg, verstr in cur_requirements.items(): - reqs.append(f"{pkg}=3D=3D{verstr}") - - reqs.append(f"Sphinx=3D=3D{ver}") - - run([pip, "install"] + reqs, check=3DTrue) - - # Freeze environment - result =3D run([pip, "freeze"], capture_output=3DTrue, text=3DTrue, ch= eck=3DTrue) - - # Pip install succeeded. Write requirements file - if args.write: - with open(req_file, "w", encoding=3D"utf-8") as fp: - fp.write(result.stdout) - - if args.make: - start_time =3D time.time() - - # Prepare a venv environment - env =3D os.environ.copy() - bin_dir =3D os.path.join(venv_dir, "bin") - env["PATH"] =3D bin_dir + ":" + env["PATH"] - env["VIRTUAL_ENV"] =3D venv_dir - if "PYTHONHOME" in env: - del env["PYTHONHOME"] - - # Test doc build - run(["make", "cleandocs"], env=3Denv, check=3DTrue) - make =3D ["make"] + args.make_args + ["htmldocs"] - - print(f". {bin_dir}/activate") - print(" ".join(make)) - print("deactivate") - run(make, env=3Denv, check=3DTrue) - - end_time =3D time.time() - elapsed_time =3D end_time - start_time - hours, minutes =3D divmod(elapsed_time, 3600) - minutes, seconds =3D divmod(minutes, 60) - - hours =3D int(hours) - minutes =3D int(minutes) - seconds =3D int(seconds) - - built_time[ver] =3D f"{hours:02d}:{minutes:02d}:{seconds:02d}" - - print(f"Finished doc build for Sphinx {ver}. Elapsed time: {built_= time[ver]}") - -if args.make: - print() - print("Summary:") - for ver, elapsed_time in sorted(built_time.items()): - print(f"\tSphinx {ver} elapsed time: {elapsed_time}") +async def main(): + """Main program""" + + parser =3D argparse.ArgumentParser(description=3D"Build docs for diffe= rent sphinx_versions.") + + parser.add_argument('-v', '--version', help=3D'Sphinx single version', + type=3Dparse_version) + parser.add_argument('--min-version', "--min", help=3D'Sphinx minimal v= ersion', + type=3Dparse_version) + parser.add_argument('--max-version', "--max", help=3D'Sphinx maximum v= ersion', + type=3Dparse_version) + parser.add_argument('-a', '--make_args', + help=3D'extra arguments for make htmldocs, like SP= HINXDIRS=3Dnetlink/specs', + nargs=3D"*") + parser.add_argument('-w', '--write', help=3D'write a requirements.txt = file', + action=3D'store_true') + parser.add_argument('-m', '--make', + help=3D'Make documentation', + action=3D'store_true') + parser.add_argument('-i', '--wait-input', + help=3D'Wait for an enter before going to the next= version', + action=3D'store_true') + + args =3D parser.parse_args() + + if not args.make_args: + args.make_args =3D [] + + if args.version: + if args.min_version or args.max_version: + sys.exit("Use either --version or --min-version/--max-version") + else: + args.min_version =3D args.version + args.max_version =3D args.version + + sphinx_versions =3D sorted(list(SPHINX_REQUIREMENTS.keys())) + + if not args.min_version: + args.min_version =3D sphinx_versions[0] + + if not args.max_version: + args.max_version =3D sphinx_versions[-1] + + venv =3D SphinxVenv() + await venv.run(args) + + +# Call main method +if __name__ =3D=3D "__main__": + asyncio.run(main()) --=20 2.49.0