From nobody Sat Feb 7 05:14:55 2026 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of groups.io designates 66.175.222.12 as permitted sender) client-ip=66.175.222.12; envelope-from=bounce+27952+59855+1787277+3901457@groups.io; helo=web01.groups.io; Authentication-Results: mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of groups.io designates 66.175.222.12 as permitted sender) smtp.mailfrom=bounce+27952+59855+1787277+3901457@groups.io; dmarc=fail(p=none dis=none) header.from=intel.com ARC-Seal: i=1; a=rsa-sha256; t=1589925086; cv=none; d=zohomail.com; s=zohoarc; b=GMivejA7NeCXnFF4gVOjAcwer52R9YJBRuvdV6Wn1b2sxK9vzOPL10gb/S6ATS8HDFh9ahAww/oodsV4YVJXy/XrlHS1wz5cx5swt9KAAD6laOmeEzwLezFMMjSMhY455xXVB18UYhgnUC4Ri/ODHhvSIgQcxpWJWuEl3xaTvi8= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1589925086; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Id:List-Unsubscribe:MIME-Version:Message-ID:Reply-To:References:Sender:Subject:To; bh=Et+f8/XQQjNv2J9aXZdnKS9M4v6RvTqny4lcqIB/jok=; b=cN0Mr0PatQ8olXEDy2qne41oo+dnENRhqbVhF8iSheulaOizcUdSg0eCjUvMdi95GS7WyBvF7f2AQCCdcXietKi70HXoX0LJtjoxHnrlP17vOQ4mE2KH8E7gknluCFQ8KwuCj2tbubh1PJ01JB7XL8QDGYq0bu7lnWSwgLY2W7o= ARC-Authentication-Results: i=1; mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of groups.io designates 66.175.222.12 as permitted sender) smtp.mailfrom=bounce+27952+59855+1787277+3901457@groups.io; dmarc=fail header.from= (p=none dis=none) header.from= Received: from web01.groups.io (web01.groups.io [66.175.222.12]) by mx.zohomail.com with SMTPS id 1589925086350662.415882203843; Tue, 19 May 2020 14:51:26 -0700 (PDT) Return-Path: X-Received: by 127.0.0.2 with SMTP id x50LYY1788612xEv7YcPXZAN; Tue, 19 May 2020 14:51:25 -0700 X-Received: from mga06.intel.com (mga06.intel.com []) by mx.groups.io with SMTP id smtpd.web12.951.1589925081386553563 for ; Tue, 19 May 2020 14:51:25 -0700 IronPort-SDR: hzTpvNCYT4mQp90/X8QsNM1bhAxBWcGqLT5nLnJ7ATCk4Zw8gQGFnED/1UfIZvcu2H47Poee3v oOuR5bIF7aAA== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by orsmga104.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 19 May 2020 14:51:25 -0700 IronPort-SDR: iIlNPKFKDfhhcCEI/9Ud46OQ8agT3I/0N1aFqDF20ueGZ9yNDpaIUKP9eGvC3BxU2RGOy2fQNN 4TdGw8go9r5Q== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.73,411,1583222400"; d="scan'208";a="289122133" X-Received: from ecbjorge-mobl1.amr.corp.intel.com ([10.209.104.72]) by fmsmga004.fm.intel.com with ESMTP; 19 May 2020 14:51:24 -0700 From: "Bjorge, Erik C" To: devel@edk2.groups.io Cc: Ashley E Desimone , Nate DeSimone , Puja Pandya , Bret Barkelew , Prince Agyeman Subject: [edk2-devel] [edk2-staging/EdkRepo] [PATCH v2 2/2] EdkRepo: Update commands to use new submodule code Date: Tue, 19 May 2020 14:51:06 -0700 Message-Id: In-Reply-To: References: MIME-Version: 1.0 Precedence: Bulk List-Unsubscribe: Sender: devel@edk2.groups.io List-Id: Mailing-List: list devel@edk2.groups.io; contact devel+owner@edk2.groups.io Reply-To: devel@edk2.groups.io,erik.c.bjorge@intel.com X-Gm-Message-State: HW0voZnY0AnLVUocwsNrr6j0x1787277AA= Content-Transfer-Encoding: quoted-printable DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=groups.io; q=dns/txt; s=20140610; t=1589925085; bh=SNS34OOFU1mLCxoQe+HGSFGWv39GiaCXQuqyoJlfJ6c=; h=Cc:Date:From:Reply-To:Subject:To; b=ePbYzOr90rw8y7Jpp0LhcLNZ9kQdrdaFNDEu/lW+UIlg7LOXUlmfc2ZUMMM0qpdIWCa 70JGiSfWPguV1D/VJT09tmh1ez0i025UMkr+bQcpp6ZAmdvdz/OLJPM8KRt47VlLcvhgq Hf622M6fh2qoHhCC1fjcNOxxAX59jENbjws= X-ZohoMail-DKIM: pass (identity @groups.io) Content-Type: text/plain; charset="utf-8" Replaced use of maintain_submodules to use new common code. This allows for selective submodule initialization via the manifest file. Cc: Ashley E Desimone Cc: Nate DeSimone Cc: Puja Pandya Cc: Bret Barkelew Cc: Prince Agyeman Cc: Erik Bjorge Signed-off-by: Erik Bjorge --- edkrepo/commands/checkout_pin_command.py | 6 ++++ edkrepo/commands/clone_command.py | 20 +++++++---- edkrepo/commands/sync_command.py | 25 +++++++++----- edkrepo/common/common_repo_functions.py | 43 +++++------------------- 4 files changed, 45 insertions(+), 49 deletions(-) diff --git a/edkrepo/commands/checkout_pin_command.py b/edkrepo/commands/ch= eckout_pin_command.py index 4aaf2b5..9d7346a 100644 --- a/edkrepo/commands/checkout_pin_command.py +++ b/edkrepo/commands/checkout_pin_command.py @@ -22,6 +22,8 @@ from edkrepo.common.workspace_maintenance.manifest_repos_= maintenance import list from edkrepo.common.workspace_maintenance.manifest_repos_maintenance impor= t find_source_manifest_repo from edkrepo.config.config_factory import get_workspace_path, get_workspac= e_manifest from edkrepo_manifest_parser.edk_manifest import ManifestXml +from project_utils.submodule import deinit_submodules, maintain_submodules + =20 class CheckoutPinCommand(EdkrepoCommand): def __init__(self): @@ -68,11 +70,15 @@ class CheckoutPinCommand(EdkrepoCommand): if sparse_enabled: print(SPARSE_RESET) reset_sparse_checkout(workspace_path, manifest_sources) + submodule_combo =3D pin.general_config.current_combo + deinit_submodules(workspace_path, manifest, manifest.general_confi= g.current_combo, + pin, submodule_combo, args.verbose) pin_repo_sources =3D pin.get_repo_sources(pin.general_config.curre= nt_combo) try: checkout_repos(args.verbose, args.override, pin_repo_sources, = workspace_path, manifest) manifest.write_current_combo(humble.PIN_COMBO.format(args.pinf= ile)) finally: + maintain_submodules(workspace_path, pin, submodule_combo, args= .verbose) if sparse_enabled: print(SPARSE_CHECKOUT) sparse_checkout(workspace_path, pin_repo_sources, manifest) diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_com= mand.py index 3f2e6e1..f638090 100644 --- a/edkrepo/commands/clone_command.py +++ b/edkrepo/commands/clone_command.py @@ -25,6 +25,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_= maintenance import pull from edkrepo.common.workspace_maintenance.manifest_repos_maintenance impor= t list_available_manifest_repos from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenanc= e_humble import PROJ_NOT_IN_REPO, SOURCE_MANIFEST_REPO_NOT_FOUND from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml +from project_utils.submodule import maintain_submodules =20 =20 class CloneCommand(EdkrepoCommand): @@ -108,7 +109,6 @@ class CloneCommand(EdkrepoCommand): manifest =3D ManifestXml(local_manifest_path) =20 # Process the combination name and make sure it can be found in th= e manifest - combo_name =3D None if args.Combination is not None: try: combo_name =3D case_insensitive_single_match(args.Combinat= ion, combinations_in_manifest(manifest)) @@ -122,13 +122,15 @@ class CloneCommand(EdkrepoCommand): # Since pin files are subset of manifest files they do not hav= e a "default combo" it is set to None. In this # case use the current_combo instead. combo_name =3D manifest.general_config.current_combo + else: + # If a combo was not specified or a pin file used the default = combo should be cloned. Also ensure that the + # current combo is updated to match. + combo_name =3D manifest.general_config.default_combo + manifest.write_current_combo(combo_name) =20 # Get the list of repos to clone and clone them - manifest_config =3D manifest.general_config - if combo_name: - repo_sources_to_clone =3D manifest.get_repo_sources(combo_name) - else: - repo_sources_to_clone =3D manifest.get_repo_sources(manifest_c= onfig.default_combo) + repo_sources_to_clone =3D manifest.get_repo_sources(combo_name) + #check that the repo sources do not contain duplicated local roots local_roots =3D [r.root for r in repo_sources_to_clone] for root in local_roots: @@ -141,7 +143,11 @@ class CloneCommand(EdkrepoCommand): # Set up submodule alt url config settings prior to cloning any re= pos submodule_included_configs =3D write_included_config(manifest.remo= tes, manifest.submodule_alternate_remotes, local_manifest_dir) write_conditional_include(workspace_dir, repo_sources_to_clone, su= bmodule_included_configs) - clone_repos(args, workspace_dir, repo_sources_to_clone, project_cl= ient_side_hooks, config, args.skip_submodule, manifest) + clone_repos(args, workspace_dir, repo_sources_to_clone, project_cl= ient_side_hooks, config, manifest) + + # Init submodules + if not args.skip_submodule: + maintain_submodules(workspace_dir, manifest, combo_name, args.= verbose) =20 # Perform a sparse checkout if requested. use_sparse =3D args.sparse diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_comma= nd.py index b662c62..13b46e8 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -38,7 +38,7 @@ from edkrepo.common.common_repo_functions import checkout= _repos, check_dirty_rep from edkrepo.common.common_repo_functions import update_editor_config from edkrepo.common.common_repo_functions import update_repo_commit_templa= te, get_latest_sha from edkrepo.common.common_repo_functions import has_primary_repo_remote, = fetch_from_primary_repo, in_sync_with_primary -from edkrepo.common.common_repo_functions import update_hooks, maintain_su= bmodules, combinations_in_manifest +from edkrepo.common.common_repo_functions import update_hooks, combination= s_in_manifest from edkrepo.common.common_repo_functions import write_included_config, re= move_included_config from edkrepo.common.workspace_maintenance.workspace_maintenance import gen= erate_name_for_obsolete_backup from edkrepo.common.workspace_maintenance.manifest_repos_maintenance impor= t pull_workspace_manifest_repo @@ -48,6 +48,7 @@ from edkrepo.common.ui_functions import init_color_console from edkrepo.config.config_factory import get_workspace_path, get_workspac= e_manifest, get_edkrepo_global_data_directory from edkrepo.config.config_factory import get_workspace_manifest_file from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml +from project_utils.submodule import deinit_submodules, maintain_submodules =20 =20 class SyncCommand(EdkrepoCommand): @@ -84,6 +85,7 @@ class SyncCommand(EdkrepoCommand): current_combo =3D initial_manifest.general_config.current_combo initial_sources =3D initial_manifest.get_repo_sources(current_comb= o) initial_hooks =3D initial_manifest.repo_hooks + initial_combo =3D current_combo =20 source_global_manifest_repo =3D find_source_manifest_repo(initial_= manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest= _repo) pull_workspace_manifest_repo(initial_manifest, config['cfg_file'],= config['user_cfg_file'], args.source_manifest_repo, False) @@ -100,6 +102,7 @@ class SyncCommand(EdkrepoCommand): if not args.update_local_manifest: self.__check_for_new_manifest(args, config, initial_manifest, = workspace_path, global_manifest_directory) check_dirty_repos(initial_manifest, workspace_path) + # Determine if sparse checkout needs to be disabled for this opera= tion sparse_settings =3D initial_manifest.sparse_settings sparse_enabled =3D sparse_checkout_enabled(workspace_path, initial= _sources) @@ -113,7 +116,7 @@ class SyncCommand(EdkrepoCommand): reset_sparse_checkout(workspace_path, initial_sources) =20 # Get the latest manifest if requested - if args.update_local_manifest: #NOTE: hyphens in arg name replaced= with underscores due to argparse + if args.update_local_manifest: # NOTE: hyphens in arg name replac= ed with underscores due to argparse self.__update_local_manifest(args, config, initial_manifest, w= orkspace_path, global_manifest_directory) manifest =3D get_workspace_manifest() if args.update_local_manifest: @@ -127,6 +130,12 @@ class SyncCommand(EdkrepoCommand): repo_sources_to_sync =3D manifest.get_repo_sources(current_com= bo) manifest.write_current_combo(current_combo) =20 + # At this point both new and old manifest files are ready so we ca= n deinit any + # submodules that are removed due to a manifest update. + if not args.skip_submodule: + deinit_submodules(workspace_path, initial_manifest, initial_co= mbo, + manifest, current_combo, args.verbose) + sync_error =3D False # Calculate the hooks which need to be updated, added or removed f= or the sync if args.update_local_manifest: @@ -195,16 +204,16 @@ class SyncCommand(EdkrepoCommand): elif args.verbose: print(NO_SYNC_DETACHED_HEAD.format(repo_to_sync.root)) =20 - if not args.skip_submodule: - if repo_to_sync.enable_submodule: - # Perform submodule updates and url redirection - maintain_submodules(repo_to_sync, repo) # Update commit message templates - update_repo_commit_template(workspace_path, repo, repo_to_sync= , config, global_manifest_directory) + update_repo_commit_template(workspace_path, repo, repo_to_sync= , config, global_manifest_directory) =20 if sync_error: print(SYNC_ERROR) =20 + # Initialize submodules + if not args.skip_submodule: + maintain_submodules(workspace_path, manifest, current_combo, a= rgs.verbose) + # Restore sparse checkout state if sparse_enabled: print(SPARSE_CHECKOUT) @@ -326,7 +335,7 @@ class SyncCommand(EdkrepoCommand): print(path_to_source) if len(sources_to_remove) > 0: print(SYNC_REMOVE_LIST_END_FORMATTING) - clone_repos(args, workspace_path, sources_to_clone, new_manife= st_to_check.repo_hooks, config, args.skip_submodule, new_manifest_to_check) + clone_repos(args, workspace_path, sources_to_clone, new_manife= st_to_check.repo_hooks, config, new_manifest_to_check) # Make a list of and only checkout repos that were newly clone= d. Sync keeps repos on their initial active branches # cloning the entire combo can prevent existing repos from cor= rectly being returned to their proper branch repos_to_checkout =3D [] diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/commo= n_repo_functions.py index 61133f1..8c79f3d 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -68,12 +68,13 @@ from edkrepo_manifest_parser.edk_manifest_validation im= port validate_manifestrep from edkrepo_manifest_parser.edk_manifest_validation import get_manifest_v= alidation_status from edkrepo_manifest_parser.edk_manifest_validation import print_manifest= _errors from edkrepo_manifest_parser.edk_manifest_validation import validate_manif= estfiles +from project_utils.submodule import deinit_submodules, maintain_submodules =20 CLEAR_LINE =3D '\x1b[K' DEFAULT_REMOTE_NAME =3D 'origin' PRIMARY_REMOTE_NAME =3D 'primary' =20 -def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_h= ooks, config, skip_submodule, manifest): +def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_h= ooks, config, manifest): for repo_to_clone in repos_to_clone: local_repo_path =3D os.path.join(workspace_dir, repo_to_clone.root) local_repo_url =3D repo_to_clone.remote_url @@ -108,10 +109,6 @@ def clone_repos(args, workspace_dir, repos_to_clone, p= roject_client_side_hooks, else: raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT) =20 - if not skip_submodule: - if repo_to_clone.enable_submodule: - maintain_submodules(repo_to_clone, repo, args.verbose) - try: if 'source_manifest_repo' in vars(args).keys(): src_manifest_repo =3D find_source_manifest_repo(manifest, = config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) @@ -184,32 +181,6 @@ def write_conditional_include(workspace_path, repo_sou= rces, included_configs): gitglobalconfig.add_section(section) gitglobalconfig.set(section, 'path', path) =20 -def maintain_submodules(repo_sources, repo, verbose =3D False): - try: - output_data =3D repo.git.execute(['git', 'submodule', 'init'], wit= h_extended_output=3DTrue, with_stdout=3DTrue) - if verbose and output_data[0]: - print(output_data[0]) - if output_data[1]: - print(output_data[1]) - if verbose and output_data[2]: - print(output_data[2]) - output_data =3D repo.git.execute(['git', 'submodule', 'sync', '--r= ecursive'], with_extended_output=3DTrue, with_stdout=3DTrue) - if verbose and output_data[0]: - print(output_data[0]) - if output_data[1]: - print(output_data[1]) - if verbose and output_data[2]: - print(output_data[2]) - output_data =3D repo.git.execute(['git', 'submodule', 'update', '-= -recursive'], with_extended_output=3DTrue, with_stdout=3DTrue) - if verbose and output_data[0]: - print(output_data[0]) - if output_data[1]: - print(output_data[1]) - if verbose and output_data[2]: - print(output_data[2]) - except: - raise EdkrepoGitException(SUBMODULE_FAILURE.format(repo_sources.re= mote_name)) - def install_hooks(hooks, local_repo_path, repo_for_install, config, global= _manifest_directory): # Determine the which hooks are for the repo in question and which are= from a URL based source or are in a global # manifest repo relative path @@ -399,9 +370,6 @@ def checkout_repos(verbose, override, repos_to_checkout= , workspace_path, manifes else: raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT) =20 - if repo_to_checkout.enable_submodule: - maintain_submodules(repo_to_checkout, repo, verbose) - def validate_manifest_repo(manifest_repo, verbose=3DFalse, archived=3DFals= e): print(VERIFY_GLOBAL) if archived: @@ -514,9 +482,11 @@ def checkout(combination_or_sha, verbose=3DFalse, over= ride=3DFalse, log=3DNone): # Create combo_or_sha so we have original input and do not introduce a= ny # unintended behavior by messing with parameters. combo_or_sha =3D combination_or_sha + submodule_combo =3D manifest.general_config.current_combo try: # Try to handle normalize combo name to match the manifest file. combo_or_sha =3D case_insensitive_single_match(combo_or_sha, combi= nations_in_manifest(manifest)) + submodule_combo =3D combo_or_sha except: # No match so leave it alone. It must be a SHA1 or a bad combo na= me. pass @@ -528,6 +498,10 @@ def checkout(combination_or_sha, verbose=3DFalse, over= ride=3DFalse, log=3DNone): log=3Dlog) initial_repo_sources =3D manifest.get_repo_sources(manifest.general_co= nfig.current_combo) =20 + # Deinit any submodules that have been removed. + deinit_submodules(workspace_path, manifest, manifest.general_config.cu= rrent_combo, + manifest, submodule_combo, verbose) + # Disable sparse checkout current_repos =3D initial_repo_sources sparse_enabled =3D sparse_checkout_enabled(workspace_path, initial_rep= o_sources) @@ -566,6 +540,7 @@ def checkout(combination_or_sha, verbose=3DFalse, overr= ide=3DFalse, log=3DNone): # Return to the initial combo, since there was an issue with cheki= ng out the selected combo checkout_repos(verbose, override, initial_repo_sources, workspace_= path, manifest) finally: + maintain_submodules(workspace_path, manifest, submodule_combo, ver= bose) if sparse_enabled or sparse_diff: print(SPARSE_CHECKOUT) sparse_checkout(workspace_path, current_repos, manifest) --=20 2.21.0.windows.1 -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#59855): https://edk2.groups.io/g/devel/message/59855 Mute This Topic: https://groups.io/mt/74336547/1787277 Group Owner: devel+owner@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [importer@patchew.org] -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-