From nobody Mon Feb 9 13:59:23 2026 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of groups.io designates 66.175.222.108 as permitted sender) client-ip=66.175.222.108; envelope-from=bounce+27952+67645+1787277+3901457@groups.io; helo=mail02.groups.io; Authentication-Results: mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of groups.io designates 66.175.222.108 as permitted sender) smtp.mailfrom=bounce+27952+67645+1787277+3901457@groups.io; dmarc=fail(p=none dis=none) header.from=intel.com ARC-Seal: i=1; a=rsa-sha256; t=1605588609; cv=none; d=zohomail.com; s=zohoarc; b=bcsRt7CQUv4sp8EqD13FXgGIIDj9AiPiEpKbdCimpTP1Xr2bRw/prdUwDTg9TVrDy1QrcawWnPdrHhmxPn82L7SD/0qHTMH+H6r2oDNFFAYd10ymeerqhBNpq6XrGVhLLRqO8FbfNB/KfkaoYWckWSRHk1mueEV0vv228klKJeg= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1605588609; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Id:List-Unsubscribe:MIME-Version:Message-ID:Reply-To:References:Sender:Subject:To; bh=G+sz/dJ+3+zIPyuenqu0Vp3Mbh/thhd2WCZEnG0+MSM=; b=g6OnGxtOugSDz8WPjUSuC2PpnCLJTEICSUDfBsWhQdiBWWSxaVlm3J8E+yOFWz5LJ0Htny/PowxbIVBuZ9JNqOvITbPip5C6VaSmhZfW0vHEJy1paUxykqGwCfsWvz03Ppt2MmbHUADxERCMUIehvmNFsNwnOq/Tn6covLu46Xk= ARC-Authentication-Results: i=1; mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of groups.io designates 66.175.222.108 as permitted sender) smtp.mailfrom=bounce+27952+67645+1787277+3901457@groups.io; dmarc=fail header.from= (p=none dis=none) header.from= Received: from mail02.groups.io (mail02.groups.io [66.175.222.108]) by mx.zohomail.com with SMTPS id 1605588609384365.12636200477993; Mon, 16 Nov 2020 20:50:09 -0800 (PST) Return-Path: X-Received: by 127.0.0.2 with SMTP id H5BqYY1788612xdfLj2mhb2z; Mon, 16 Nov 2020 20:50:09 -0800 X-Received: from mga05.intel.com (mga05.intel.com []) by mx.groups.io with SMTP id smtpd.web12.3770.1605588604161105067 for ; Mon, 16 Nov 2020 20:50:08 -0800 IronPort-SDR: P1FDKe8LfWaupNy6l6jga9v8Lkm/I32BtkarasSVJ+AZq3lxUnxShsrOt6+JmKT0JlT0arv8XC /+CnqJvTvpSA== X-IronPort-AV: E=McAfee;i="6000,8403,9807"; a="255574605" X-IronPort-AV: E=Sophos;i="5.77,484,1596524400"; d="scan'208";a="255574605" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-Received: from orsmga004.jf.intel.com ([10.7.209.38]) by fmsmga105.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 16 Nov 2020 20:50:08 -0800 IronPort-SDR: GB4kzwTtJbKTAQh6hLtRo+GgT/Ogeq2SjbyfqQPVvZ8S4VurwG/661JYrZKS/FOvfkJ6OuGfE2 Lbbf7Lunn+MA== X-IronPort-AV: E=Sophos;i="5.77,484,1596524400"; d="scan'208";a="475783070" X-Received: from ecbjorge-mobl1.amr.corp.intel.com ([10.251.18.231]) by orsmga004-auth.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 16 Nov 2020 20:50:08 -0800 From: "Bjorge, Erik C" To: devel@edk2.groups.io Cc: Ashley E Desimone , Nate DeSimone , Puja Pandya , Bret Barkelew , Prince Agyeman Subject: [edk2-devel] [edk2-staging/EdkRepo] [PATCH v2 2/2] EdkRepo: Enable use of repo cache support. Date: Mon, 16 Nov 2020 20:49:45 -0800 Message-Id: In-Reply-To: References: MIME-Version: 1.0 Precedence: Bulk List-Unsubscribe: Sender: devel@edk2.groups.io List-Id: Mailing-List: list devel@edk2.groups.io; contact devel+owner@edk2.groups.io Reply-To: devel@edk2.groups.io,erik.c.bjorge@intel.com X-Gm-Message-State: jUqCLwEw3pzWjXgAeO6tzvdVx1787277AA= Content-Transfer-Encoding: quoted-printable DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=groups.io; q=dns/txt; s=20140610; t=1605588609; bh=gFK5FKpAfXrRM0cypy5/ai1OJQTIMe/CJMzHbeSvM+s=; h=Cc:Date:From:Reply-To:Subject:To; b=PA911W21tn9jHqifHg7xHy/SdT0rgPRKxMH9uJpK/RfwukqinAvvgQ/nPHnEuXciLsu TjwM72i7H3wE5I27KVKUDBwKyP7gywrls3bhKXI7Jmqy4Pq/PBtg/wgs9DSY7qsEzage1 qEovw1R5C1uGdToTswEko34AH7VKk5sxS1k= X-ZohoMail-DKIM: pass (identity @groups.io) Content-Type: text/plain; charset="utf-8" This changes enables the local repo cache to be used when cloning and syncing changes. The repo cache applies to submodules as well. Cc: Ashley E Desimone Cc: Nate DeSimone Cc: Puja Pandya Cc: Bret Barkelew Cc: Prince Agyeman Cc: Erik Bjorge Signed-off-by: Erik Bjorge Reviewed-by: Ashley DeSimone --- edkrepo/commands/checkout_command.py | 3 ++- edkrepo/commands/checkout_pin_command.py | 8 +++++++- edkrepo/commands/clone_command.py | 15 ++++++++++++-- edkrepo/commands/sync_command.py | 12 +++++++++--- edkrepo/common/common_repo_functions.py | 25 ++++++++++++++++++++---- project_utils/submodule.py | 13 ++++++++---- 6 files changed, 61 insertions(+), 15 deletions(-) diff --git a/edkrepo/commands/checkout_command.py b/edkrepo/commands/checko= ut_command.py index 0169f30..2ce26c0 100644 --- a/edkrepo/commands/checkout_command.py +++ b/edkrepo/commands/checkout_command.py @@ -16,6 +16,7 @@ import os from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgum= ent import edkrepo.commands.arguments.checkout_args as arguments import edkrepo.commands.humble.checkout_humble as humble +from edkrepo.common.common_cache_functions import get_repo_cache_obj from edkrepo.common.common_repo_functions import checkout, combination_is_= in_manifest from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersExcep= tion from edkrepo.config.config_factory import get_workspace_manifest @@ -42,6 +43,6 @@ class CheckoutCommand(EdkrepoCommand): =20 def run_command(self, args, config): if combination_is_in_manifest(args.Combination, get_workspace_mani= fest()): - checkout(args.Combination, args.verbose, args.override) + checkout(args.Combination, args.verbose, args.override, get_re= po_cache_obj(config)) else: raise EdkrepoInvalidParametersException(humble.NO_COMBO.format= (args.Combination)) diff --git a/edkrepo/commands/checkout_pin_command.py b/edkrepo/commands/ch= eckout_pin_command.py index 1c58113..0ad1b48 100644 --- a/edkrepo/commands/checkout_pin_command.py +++ b/edkrepo/commands/checkout_pin_command.py @@ -14,6 +14,7 @@ from git import Repo from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgum= ent, SourceManifestRepoArgument import edkrepo.commands.arguments.checkout_pin_args as arguments import edkrepo.commands.humble.checkout_pin_humble as humble +from edkrepo.common.common_cache_functions import get_repo_cache_obj from edkrepo.common.common_repo_functions import sparse_checkout_enabled, = reset_sparse_checkout, sparse_checkout from edkrepo.common.common_repo_functions import check_dirty_repos, checko= ut_repos, combinations_in_manifest from edkrepo.common.humble import SPARSE_CHECKOUT, SPARSE_RESET, SUBMODULE= _DEINIT_FAILED @@ -21,6 +22,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoInval= idParametersException, from edkrepo.common.workspace_maintenance.manifest_repos_maintenance impor= t list_available_manifest_repos from edkrepo.common.workspace_maintenance.manifest_repos_maintenance impor= t find_source_manifest_repo from edkrepo.config.config_factory import get_workspace_path, get_workspac= e_manifest +from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME from edkrepo_manifest_parser.edk_manifest import ManifestXml from project_utils.submodule import deinit_full, maintain_submodules =20 @@ -82,7 +84,11 @@ class CheckoutPinCommand(EdkrepoCommand): checkout_repos(args.verbose, args.override, pin_repo_sources, = workspace_path, manifest) manifest.write_current_combo(humble.PIN_COMBO.format(args.pinf= ile)) finally: - maintain_submodules(workspace_path, pin, submodule_combo, args= .verbose) + cache_path =3D None + cache_obj =3D get_repo_cache_obj(config) + if cache_obj is not None: + cache_path =3D cache_obj.get_cache_path(SUBMODULE_CACHE_RE= PO_NAME) + maintain_submodules(workspace_path, pin, submodule_combo, args= .verbose, cache_path) if sparse_enabled: print(SPARSE_CHECKOUT) sparse_checkout(workspace_path, pin_repo_sources, manifest) diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_com= mand.py index 8769102..56c15c9 100644 --- a/edkrepo/commands/clone_command.py +++ b/edkrepo/commands/clone_command.py @@ -14,6 +14,8 @@ import sys from edkrepo.commands.edkrepo_command import EdkrepoCommand from edkrepo.commands.edkrepo_command import SubmoduleSkipArgument, Source= ManifestRepoArgument import edkrepo.commands.arguments.clone_args as arguments +from edkrepo.common.common_cache_functions import get_repo_cache_obj +from edkrepo.common.common_cache_functions import add_missing_cache_repos from edkrepo.common.common_repo_functions import clone_repos, sparse_check= out, verify_single_manifest from edkrepo.common.common_repo_functions import update_editor_config, com= binations_in_manifest from edkrepo.common.common_repo_functions import write_included_config, wr= ite_conditional_include @@ -28,6 +30,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_= maintenance import list from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenanc= e_humble import PROJ_NOT_IN_REPO, SOURCE_MANIFEST_REPO_NOT_FOUND from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml from project_utils.submodule import maintain_submodules +from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME =20 =20 class CloneCommand(EdkrepoCommand): @@ -151,11 +154,19 @@ class CloneCommand(EdkrepoCommand): # Set up submodule alt url config settings prior to cloning any re= pos submodule_included_configs =3D write_included_config(manifest.remo= tes, manifest.submodule_alternate_remotes, local_manifest_dir) write_conditional_include(workspace_dir, repo_sources_to_clone, su= bmodule_included_configs) - clone_repos(args, workspace_dir, repo_sources_to_clone, project_cl= ient_side_hooks, config, manifest) + + # Determine if caching is going to be used and then clone + cache_obj =3D get_repo_cache_obj(config) + if cache_obj is not None: + add_missing_cache_repos(cache_obj, manifest, args.verbose) + clone_repos(args, workspace_dir, repo_sources_to_clone, project_cl= ient_side_hooks, config, manifest, cache_obj) =20 # Init submodules if not args.skip_submodule: - maintain_submodules(workspace_dir, manifest, combo_name, args.= verbose) + cache_path =3D None + if cache_obj is not None: + cache_path =3D cache_obj.get_cache_path(SUBMODULE_CACHE_RE= PO_NAME) + maintain_submodules(workspace_dir, manifest, combo_name, args.= verbose, cache_path) =20 # Perform a sparse checkout if requested. use_sparse =3D args.sparse diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_comma= nd.py index c4ee330..ff48f50 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -32,6 +32,7 @@ from edkrepo.common.humble import MIRROR_BEHIND_PRIMARY_R= EPO, SYNC_NEEDS_REBASE, from edkrepo.common.humble import SYNC_BRANCH_CHANGE_ON_LOCAL, SYNC_INCOMP= ATIBLE_COMBO from edkrepo.common.humble import SYNC_REBASE_CALC_FAIL from edkrepo.common.pathfix import get_actual_path, expanduser +from edkrepo.common.common_cache_functions import get_repo_cache_obj from edkrepo.common.common_repo_functions import clone_repos, sparse_check= out_enabled from edkrepo.common.common_repo_functions import reset_sparse_checkout, sp= arse_checkout, verify_single_manifest from edkrepo.common.common_repo_functions import checkout_repos, check_dir= ty_repos @@ -47,6 +48,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_= maintenance import list from edkrepo.common.ui_functions import init_color_console from edkrepo.config.config_factory import get_workspace_path, get_workspac= e_manifest, get_edkrepo_global_data_directory from edkrepo.config.config_factory import get_workspace_manifest_file +from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml from project_utils.submodule import deinit_submodules, maintain_submodules =20 @@ -102,7 +104,7 @@ class SyncCommand(EdkrepoCommand): if not args.update_local_manifest: self.__check_for_new_manifest(args, config, initial_manifest, = workspace_path, global_manifest_directory) check_dirty_repos(initial_manifest, workspace_path) - + # Determine if sparse checkout needs to be disabled for this opera= tion sparse_settings =3D initial_manifest.sparse_settings sparse_enabled =3D sparse_checkout_enabled(workspace_path, initial= _sources) @@ -116,7 +118,7 @@ class SyncCommand(EdkrepoCommand): reset_sparse_checkout(workspace_path, initial_sources) =20 # Get the latest manifest if requested - if args.update_local_manifest: # NOTE: hyphens in arg name replac= ed with underscores due to argparse + if args.update_local_manifest: # NOTE: hyphens in arg name replac= ed with underscores due to argparse self.__update_local_manifest(args, config, initial_manifest, w= orkspace_path, global_manifest_directory) manifest =3D get_workspace_manifest() if args.update_local_manifest: @@ -212,7 +214,11 @@ class SyncCommand(EdkrepoCommand): =20 # Initialize submodules if not args.skip_submodule: - maintain_submodules(workspace_path, manifest, current_combo, a= rgs.verbose) + cache_path =3D None + cache_obj =3D get_repo_cache_obj(config) + if cache_obj is not None: + cache_path =3D cache_obj.get_cache_path(SUBMODULE_CACHE_RE= PO_NAME) + maintain_submodules(workspace_path, manifest, current_combo, a= rgs.verbose, cache_path) =20 # Restore sparse checkout state if sparse_enabled: diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/commo= n_repo_functions.py index 2277c1e..336661a 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -56,6 +56,7 @@ from project_utils.sparse import BuildInfo, process_spars= e_checkout from edkrepo.config.config_factory import get_workspace_path from edkrepo.config.config_factory import get_workspace_manifest from edkrepo.config.tool_config import CI_INDEX_FILE_NAME +from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersExcep= tion from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml from edkrepo.common.edkrepo_exception import EdkrepoNotFoundException, Edk= repoGitException, EdkrepoWarningException @@ -75,12 +76,25 @@ CLEAR_LINE =3D '\x1b[K' DEFAULT_REMOTE_NAME =3D 'origin' PRIMARY_REMOTE_NAME =3D 'primary' =20 -def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_h= ooks, config, manifest): + +def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_h= ooks, config, manifest, cache_obj=3DNone): for repo_to_clone in repos_to_clone: local_repo_path =3D os.path.join(workspace_dir, repo_to_clone.root) local_repo_url =3D repo_to_clone.remote_url + cache_path =3D None + if cache_obj is not None: + cache_path =3D cache_obj.get_cache_path(local_repo_url) print("Cloning from: " + str(local_repo_url)) - repo =3D Repo.clone_from(local_repo_url, local_repo_path, progress= =3DGitProgressHandler(), no_checkout=3DTrue) + if cache_path is not None: + print('+ Using cache at {}'.format(cache_path)) + repo =3D Repo.clone_from(local_repo_url, local_repo_path, + progress=3DGitProgressHandler(), + reference_if_able=3Dcache_path, + no_checkout=3DTrue) + else: + repo =3D Repo.clone_from(local_repo_url, local_repo_path, + progress=3DGitProgressHandler(), + no_checkout=3DTrue) # Fetch notes repo.remotes.origin.fetch("refs/notes/*:refs/notes/*") =20 @@ -427,7 +441,7 @@ def combination_is_in_manifest(combination, manifest): return combination in combination_names =20 =20 -def checkout(combination, verbose=3DFalse, override=3DFalse, log=3DNone): +def checkout(combination, verbose=3DFalse, override=3DFalse, log=3DNone, c= ache_obj=3DNone): workspace_path =3D get_workspace_path() manifest =3D get_workspace_manifest() =20 @@ -493,7 +507,10 @@ def checkout(combination, verbose=3DFalse, override=3D= False, log=3DNone): # Return to the initial combo, since there was an issue with cheki= ng out the selected combo checkout_repos(verbose, override, initial_repo_sources, workspace_= path, manifest) finally: - maintain_submodules(workspace_path, manifest, submodule_combo, ver= bose) + cache_path =3D None + if cache_obj is not None: + cache_path =3D cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_N= AME) + maintain_submodules(workspace_path, manifest, submodule_combo, ver= bose, cache_path) if sparse_enabled or sparse_diff: print(SPARSE_CHECKOUT) sparse_checkout(workspace_path, current_repos, manifest) diff --git a/project_utils/submodule.py b/project_utils/submodule.py index 3d1b620..f735125 100644 --- a/project_utils/submodule.py +++ b/project_utils/submodule.py @@ -61,7 +61,7 @@ def _deinit(repo, submodules=3DNone, verbose=3DFalse): return =20 =20 -def _update(repo, submodules=3DNone, verbose=3DFalse, recursive=3DFalse): +def _update(repo, submodules=3DNone, verbose=3DFalse, recursive=3DFalse, c= ache_path=3DNone): """ Performs the update of submodules. This includes the sync and update = operations. =20 @@ -82,6 +82,8 @@ def _update(repo, submodules=3DNone, verbose=3DFalse, rec= ursive=3DFalse): cmd =3D ['git', 'submodule', 'update', '--init'] if recursive: cmd.append('--recursive') + if cache_path is not None: + cmd.extend(['--reference', cache_path]) output_data =3D repo.git.execute(cmd, with_extended_output=3DTrue,= with_stdout=3DTrue) display_git_output(output_data, verbose) else: @@ -99,6 +101,8 @@ def _update(repo, submodules=3DNone, verbose=3DFalse, re= cursive=3DFalse): cmd =3D ['git', 'submodule', 'update', '--init'] if sub.recursive: cmd.append('--recursive') + if cache_path is not None: + cmd.extend(['--reference', cache_path]) cmd.extend(['--', sub.path]) output_data =3D repo.git.execute(cmd, with_extended_output=3DT= rue, with_stdout=3DTrue) display_git_output(output_data, verbose) @@ -269,7 +273,7 @@ def deinit_submodules(workspace, start_manifest, start_= combo, _deinit(repo, deinit_list, verbose) =20 =20 -def maintain_submodules(workspace, manifest, combo_name, verbose=3DFalse): +def maintain_submodules(workspace, manifest, combo_name, verbose=3DFalse, = cache_path=3DNone): """ Updates the submodules for a specific repo. =20 @@ -277,6 +281,7 @@ def maintain_submodules(workspace, manifest, combo_name= , verbose=3DFalse): manifest - The manifest parser object for the project. combo_name - The combination name to use for submodule maintenance. verbose - Enable verbose messages. + cache_path - Path to the submodule cache repo. A value of None in= dicates that no cache repo exists. """ # Process each repo that may have submodules enabled print(strings.SUBMOD_INIT_UPDATE) @@ -303,9 +308,9 @@ def maintain_submodules(workspace, manifest, combo_name= , verbose=3DFalse): =20 # Perform sync/update if len(repo_subs) =3D=3D 0: - _update(repo, None, verbose) + _update(repo, None, verbose, cache_path=3Dcache_path) else: - _update(repo, repo_subs, verbose) + _update(repo, repo_subs, verbose, cache_path=3Dcache_path) =20 =20 if __name__ =3D=3D '__main__': --=20 2.21.0.windows.1 -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#67645): https://edk2.groups.io/g/devel/message/67645 Mute This Topic: https://groups.io/mt/78310536/1787277 Group Owner: devel+owner@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [importer@patchew.org] -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-