From nobody Thu Apr 25 19:34:19 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of groups.io designates 66.175.222.12 as permitted sender) client-ip=66.175.222.12; envelope-from=bounce+27952+63462+1787277+3901457@groups.io; helo=web01.groups.io; Authentication-Results: mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of groups.io designates 66.175.222.12 as permitted sender) smtp.mailfrom=bounce+27952+63462+1787277+3901457@groups.io; dmarc=fail(p=none dis=none) header.from=intel.com ARC-Seal: i=1; a=rsa-sha256; t=1596071447; cv=none; d=zohomail.com; s=zohoarc; b=CmQDTBiX2wR8bGMdoCrN1sUqp4veO/z5fOknWCqDKnHftHB51m3leepsT876rXIQHM/cpmMjT8eHlTN2l5avQIAMISlahhUVTfygyhNHrOYTm8rv2GjE05tYxLBxNaON00YBNJvWlfhHWBIrLnWrmH2tFxIW0WYpTeagJdsQFQg= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1596071447; h=Content-Transfer-Encoding:Cc:Date:From:List-Id:List-Unsubscribe:MIME-Version:Message-ID:Reply-To:Sender:Subject:To; bh=Ar2vCNPmN6uL1rv/NLOSlo8N5tWL+QrmJ6Cd9NPfiVo=; b=Uj2jCSzw5vyN4291uXECqkSgmpO9TeJ5kMuIRMo0LQfUEDTC4GCJH07l0xaAl63ePtr3MEOOmAYF4MpLiawXFsV9d/Agvse1vxZIfTJ9c3vSR9Gm9O5O0pN5P7x9YagWcp9dkT25oxLJMV5lNdXFON7FFh+B+SuM8S4hDDB5pZw= ARC-Authentication-Results: i=1; mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of groups.io designates 66.175.222.12 as permitted sender) smtp.mailfrom=bounce+27952+63462+1787277+3901457@groups.io; dmarc=fail header.from= (p=none dis=none) header.from= Received: from web01.groups.io (web01.groups.io [66.175.222.12]) by mx.zohomail.com with SMTPS id 1596071447847725.0959635849273; Wed, 29 Jul 2020 18:10:47 -0700 (PDT) Return-Path: X-Received: by 127.0.0.2 with SMTP id RPnRYY1788612x3bkNTON3TC; Wed, 29 Jul 2020 18:10:47 -0700 X-Received: from mga18.intel.com (mga18.intel.com [134.134.136.126]) by mx.groups.io with SMTP id smtpd.web10.4845.1596071446718291790 for ; Wed, 29 Jul 2020 18:10:46 -0700 IronPort-SDR: GV5eaV+NyAlclBaqOtT8471ElwTb6v0TnD4PUUNdImHRK/t15ZDDRNbo3tViyMBKFNQlmqhv8y JcVaqFyl1v+Q== X-IronPort-AV: E=McAfee;i="6000,8403,9697"; a="139055591" X-IronPort-AV: E=Sophos;i="5.75,412,1589266800"; d="scan'208";a="139055591" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by orsmga106.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 29 Jul 2020 18:10:45 -0700 IronPort-SDR: B8EcZK7l39MOoco79QeqYrcXCw4rIXVHdiqQDYyTbvN2UPJgODcdtP2tq1w8pXXogHX2ErU21c fKZEeb8K1iOw== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.75,412,1589266800"; d="scan'208";a="328814515" X-Received: from ecbjorge-mobl1.amr.corp.intel.com ([10.212.23.242]) by FMSMGA003.fm.intel.com with ESMTP; 29 Jul 2020 18:10:45 -0700 From: "Bjorge, Erik C" To: devel@edk2.groups.io Cc: Ashley E Desimone , Nate DeSimone , Puja Pandya , Bret Barkelew , Prince Agyeman Subject: [edk2-devel] [edk2-staging/EdkRepo] [PATCH v1] EdkRepo: Fixing inconsistent win32 HOME path mapping Date: Wed, 29 Jul 2020 18:10:25 -0700 Message-Id: <317733dbaa367ac5df0bf72f8708c66ac5ea9d40.1596070906.git.erik.c.bjorge@intel.com> MIME-Version: 1.0 Precedence: Bulk List-Unsubscribe: Sender: devel@edk2.groups.io List-Id: Mailing-List: list devel@edk2.groups.io; contact devel+owner@edk2.groups.io Reply-To: devel@edk2.groups.io,erik.c.bjorge@intel.com X-Gm-Message-State: NPxkZYIefJSPWvykMF76y2gpx1787277AA= Content-Transfer-Encoding: quoted-printable DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=groups.io; q=dns/txt; s=20140610; t=1596071447; bh=xYBMZNsPhxwwSm2tuFQ0pQi4InBaHdfRdKTKR5Sepq0=; h=Cc:Date:From:Reply-To:Subject:To; b=DgFs5nZs+T0tN+BVDeyN9KW0lPuUv6bswIczQNCt1DOh35VRbae8wGcg4SxYmuYklFl xNRS04fggZnncAqmWsuMA9LQgw9qTrIH4rqPStj7KzEz2E/SMiynmj2OJv7Alww+gVt+k odKjkJHZO6XpiziRmRM+YEd2guyTkJL77jU= X-ZohoMail-DKIM: pass (identity @groups.io) Content-Type: text/plain; charset="utf-8" In Python 3.8.x the method of mapping the users home path was changed to look at USERPROFILE and then HOMEDRIVE\HOMEPATH. This change removed the check of HOME generated by git BASH based on HOMEDRIVE/HOMEPATH. For users with a USERPROFILE environment variable that does not match the HOMEDRIVE\HOMEPATH environment variables this can lead to inconsistent decoding of files such as .gitconfig between BASH and EdkRepo. This change attempts to enforce a consistent decode order. HOME, HOMEDRIVE\HOMEPATH and USERPROFILE. Cc: Ashley E Desimone Cc: Nate DeSimone Cc: Puja Pandya Cc: Bret Barkelew Cc: Prince Agyeman Cc: Erik Bjorge Signed-off-by: Erik Bjorge Reviewed-by: Ashley DeSimone Reviewed-by: Nate DeSimone --- edkrepo/commands/sync_command.py | 46 ++++++++++----------- edkrepo/common/common_repo_functions.py | 6 +-- edkrepo/common/pathfix.py | 54 ++++++++++++++++++++++++- edkrepo/common/ui_functions.py | 6 ++- edkrepo/config/config_factory.py | 3 +- 5 files changed, 85 insertions(+), 30 deletions(-) diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_comma= nd.py index 2cb4630..ea3eed3 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -31,14 +31,14 @@ from edkrepo.common.humble import NO_SYNC_DETACHED_HEAD= , SYNC_COMMITS_ON_MASTER, from edkrepo.common.humble import MIRROR_BEHIND_PRIMARY_REPO, SYNC_NEEDS_R= EBASE, INCLUDED_FILE_NAME from edkrepo.common.humble import SYNC_BRANCH_CHANGE_ON_LOCAL, SYNC_INCOMP= ATIBLE_COMBO from edkrepo.common.humble import SYNC_REBASE_CALC_FAIL -from edkrepo.common.pathfix import get_actual_path +from edkrepo.common.pathfix import get_actual_path, expanduser from edkrepo.common.common_repo_functions import clone_repos, sparse_check= out_enabled from edkrepo.common.common_repo_functions import reset_sparse_checkout, sp= arse_checkout, verify_single_manifest from edkrepo.common.common_repo_functions import checkout_repos, check_dir= ty_repos from edkrepo.common.common_repo_functions import update_editor_config from edkrepo.common.common_repo_functions import update_repo_commit_templa= te, get_latest_sha from edkrepo.common.common_repo_functions import has_primary_repo_remote, = fetch_from_primary_repo, in_sync_with_primary -from edkrepo.common.common_repo_functions import update_hooks, combination= s_in_manifest +from edkrepo.common.common_repo_functions import update_hooks, combination= s_in_manifest from edkrepo.common.common_repo_functions import write_included_config, re= move_included_config from edkrepo.common.workspace_maintenance.workspace_maintenance import gen= erate_name_for_obsolete_backup from edkrepo.common.workspace_maintenance.manifest_repos_maintenance impor= t pull_workspace_manifest_repo @@ -48,7 +48,7 @@ from edkrepo.common.ui_functions import init_color_console from edkrepo.config.config_factory import get_workspace_path, get_workspac= e_manifest, get_edkrepo_global_data_directory from edkrepo.config.config_factory import get_workspace_manifest_file from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml -from project_utils.submodule import deinit_submodules, maintain_submodules +from project_utils.submodule import deinit_submodules, maintain_submodules =20 =20 class SyncCommand(EdkrepoCommand): @@ -85,7 +85,7 @@ class SyncCommand(EdkrepoCommand): current_combo =3D initial_manifest.general_config.current_combo initial_sources =3D initial_manifest.get_repo_sources(current_comb= o) initial_hooks =3D initial_manifest.repo_hooks - initial_combo =3D current_combo + initial_combo =3D current_combo =20 source_global_manifest_repo =3D find_source_manifest_repo(initial_= manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest= _repo) pull_workspace_manifest_repo(initial_manifest, config['cfg_file'],= config['user_cfg_file'], args.source_manifest_repo, False) @@ -102,7 +102,7 @@ class SyncCommand(EdkrepoCommand): if not args.update_local_manifest: self.__check_for_new_manifest(args, config, initial_manifest, = workspace_path, global_manifest_directory) check_dirty_repos(initial_manifest, workspace_path) - + # Determine if sparse checkout needs to be disabled for this opera= tion sparse_settings =3D initial_manifest.sparse_settings sparse_enabled =3D sparse_checkout_enabled(workspace_path, initial= _sources) @@ -116,7 +116,7 @@ class SyncCommand(EdkrepoCommand): reset_sparse_checkout(workspace_path, initial_sources) =20 # Get the latest manifest if requested - if args.update_local_manifest: # NOTE: hyphens in arg name replac= ed with underscores due to argparse + if args.update_local_manifest: # NOTE: hyphens in arg name replac= ed with underscores due to argparse self.__update_local_manifest(args, config, initial_manifest, w= orkspace_path, global_manifest_directory) manifest =3D get_workspace_manifest() if args.update_local_manifest: @@ -130,12 +130,12 @@ class SyncCommand(EdkrepoCommand): repo_sources_to_sync =3D manifest.get_repo_sources(current_com= bo) manifest.write_current_combo(current_combo) =20 - # At this point both new and old manifest files are ready so we ca= n deinit any - # submodules that are removed due to a manifest update. - if not args.skip_submodule: - deinit_submodules(workspace_path, initial_manifest, initial_co= mbo, - manifest, current_combo, args.verbose) - + # At this point both new and old manifest files are ready so we ca= n deinit any + # submodules that are removed due to a manifest update. + if not args.skip_submodule: + deinit_submodules(workspace_path, initial_manifest, initial_co= mbo, + manifest, current_combo, args.verbose) + sync_error =3D False # Calculate the hooks which need to be updated, added or removed f= or the sync if args.update_local_manifest: @@ -205,15 +205,15 @@ class SyncCommand(EdkrepoCommand): print(NO_SYNC_DETACHED_HEAD.format(repo_to_sync.root)) =20 # Update commit message templates - update_repo_commit_template(workspace_path, repo, repo_to_sync= , config, global_manifest_directory) + update_repo_commit_template(workspace_path, repo, repo_to_sync= , config, global_manifest_directory) =20 if sync_error: print(SYNC_ERROR) =20 - # Initialize submodules - if not args.skip_submodule: - maintain_submodules(workspace_path, manifest, current_combo, a= rgs.verbose) - + # Initialize submodules + if not args.skip_submodule: + maintain_submodules(workspace_path, manifest, current_combo, a= rgs.verbose) + # Restore sparse checkout state if sparse_enabled: print(SPARSE_CHECKOUT) @@ -244,9 +244,9 @@ class SyncCommand(EdkrepoCommand): # the default combo initial_combos =3D combinations_in_manifest(initial_manifest) new_combos =3D combinations_in_manifest(new_manifest_to_check) - if current_combo not in new_combos: - new_sources_for_current_combo =3D new_manifest_to_check.get_re= po_sources(new_manifest_to_check.general_config.default_combo) - new_sources =3D new_sources_for_current_combo + if current_combo not in new_combos: + new_sources_for_current_combo =3D new_manifest_to_check.get_re= po_sources(new_manifest_to_check.general_config.default_combo) + new_sources =3D new_sources_for_current_combo else: new_sources_for_current_combo =3D new_manifest_to_check.get_re= po_sources(current_combo) new_sources =3D new_manifest_to_check.get_repo_sources(current= _combo) @@ -330,7 +330,7 @@ class SyncCommand(EdkrepoCommand): print(path_to_source) if len(sources_to_remove) > 0: print(SYNC_REMOVE_LIST_END_FORMATTING) - clone_repos(args, workspace_path, sources_to_clone, new_manife= st_to_check.repo_hooks, config, new_manifest_to_check) + clone_repos(args, workspace_path, sources_to_clone, new_manife= st_to_check.repo_hooks, config, new_manifest_to_check) # Make a list of and only checkout repos that were newly clone= d. Sync keeps repos on their initial active branches # cloning the entire combo can prevent existing repos from cor= rectly being returned to their proper branch repos_to_checkout =3D [] @@ -395,7 +395,7 @@ class SyncCommand(EdkrepoCommand): print(SYNC_MANIFEST_UPDATE) =20 def __check_submodule_config(self, workspace_path, manifest, repo_sour= ces): - gitconfigpath =3D os.path.normpath(os.path.expanduser("~/.gitconfi= g")) + gitconfigpath =3D os.path.normpath(expanduser("~/.gitconfig")) gitglobalconfig =3D git.GitConfigParser(gitconfigpath, read_only= =3DFalse) try: local_manifest_dir =3D os.path.join(workspace_path, "repo") @@ -448,7 +448,7 @@ class SyncCommand(EdkrepoCommand): gitglobalconfig.release() =20 def __clean_git_globalconfig(self): - global_gitconfig_path =3D os.path.normpath(os.path.expanduser("~/.= gitconfig")) + global_gitconfig_path =3D os.path.normpath(expanduser("~/.gitconfi= g")) with git.GitConfigParser(global_gitconfig_path, read_only=3DFalse)= as git_globalconfig: includeif_regex =3D re.compile('^includeIf "gitdir:(/.+)/"$') for section in git_globalconfig.sections(): diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/commo= n_repo_functions.py index a6a38cf..d14f89a 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -51,7 +51,7 @@ from edkrepo.common.humble import ERROR_WRITING_INCLUDE, = MULTIPLE_SOURCE_ATTRIBU from edkrepo.common.humble import VERIFY_GLOBAL, VERIFY_ARCHIVED, VERIFY_P= ROJ, VERIFY_PROJ_FAIL from edkrepo.common.humble import VERIFY_PROJ_NOT_IN_INDEX, VERIFY_GLOBAL_= FAIL from edkrepo.common.humble import SUBMODULE_DEINIT_FAILED -from edkrepo.common.pathfix import get_actual_path +from edkrepo.common.pathfix import get_actual_path, expanduser from project_utils.sparse import BuildInfo, process_sparse_checkout from edkrepo.config.config_factory import get_workspace_path from edkrepo.config.config_factory import get_workspace_manifest @@ -164,7 +164,7 @@ def remove_included_config(remotes, submodule_alt_remot= es, repo_directory): os.remove(include_to_remove) =20 def write_conditional_include(workspace_path, repo_sources, included_confi= gs): - gitconfigpath =3D os.path.normpath(os.path.expanduser("~/.gitconfig")) + gitconfigpath =3D os.path.normpath(expanduser("~/.gitconfig")) for source in repo_sources: for included_config in included_configs: if included_config[0] =3D=3D source.remote_name: @@ -566,7 +566,7 @@ def update_repo_commit_template(workspace_dir, repo, re= po_info, config, global_m =20 #Check for the presence of a gloablly defined commit template global_template_in_use =3D False - global_gitconfig_path =3D os.path.normpath(os.path.expanduser("~/.gitc= onfig")) + global_gitconfig_path =3D os.path.normpath(expanduser("~/.gitconfig")) with git.GitConfigParser(global_gitconfig_path, read_only=3DFalse) as = gitglobalconfig: if gitglobalconfig.has_option(section=3D'commit', option=3D'templa= te'): global_template =3D gitglobalconfig.get_value(section=3D'commi= t', option=3D'template') diff --git a/edkrepo/common/pathfix.py b/edkrepo/common/pathfix.py index 6f3b151..1a9c20f 100644 --- a/edkrepo/common/pathfix.py +++ b/edkrepo/common/pathfix.py @@ -3,7 +3,7 @@ ## @file # checkout_command.py # -# Copyright (c) 2018- 2019, Intel Corporation. All rights reserved.
+# Copyright (c) 2018- 2020, Intel Corporation. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # import os @@ -159,3 +159,55 @@ def get_actual_path(path): return ''.join(actual_path) else: return path + + +def _get_bothseps(path): + if isinstance(path, bytes): + return b'\\/' + else: + return '\\/' + + +def expanduser(path): + """ + Wrapper to consistently map the users HOME directory location. Curren= tly + three separate environment variable sets exist to do this mapping. Th= e default + python mapping in ntpath.py may not work because of different priority= of decode. + This function is designed to remove any variation. + + Note: This is a copy of the ntpath.py function with minor modification= s. + """ + if sys.platform !=3D 'win32': + return os.path.expanduser(path) + + path =3D os.fspath(path) + if isinstance(path, bytes): + tilde =3D b'~' + else: + tilde =3D '~' + if not path.startswith(tilde): + return path + i, n =3D 1, len(path) + while i < n and path[i] not in _get_bothseps(path): + i +=3D 1 + + if 'HOME' in os.environ: + userhome =3D os.environ['HOME'] + elif 'HOMEPATH' in os.environ: + try: + drive =3D os.environ['HOMEDRIVE'] + except KeyError: + drive =3D '' + userhome =3D os.path.join(drive, os.environ['HOMEPATH']) + elif 'USERPROFILE' in os.environ: + userhome =3D os.environ['USERPROFILE'] + else: + return path + + if isinstance(path, bytes): + userhome =3D os.fsencode(userhome) + + if i !=3D 1: # ~user + userhome =3D os.path.join(os.path.dirname(userhome), path[1:i]) + + return userhome + path[i:] diff --git a/edkrepo/common/ui_functions.py b/edkrepo/common/ui_functions.py index d42c2de..a23ca91 100644 --- a/edkrepo/common/ui_functions.py +++ b/edkrepo/common/ui_functions.py @@ -3,7 +3,7 @@ ## @file # ui_functions.py # -# Copyright (c) 2017- 2019, Intel Corporation. All rights reserved.
+# Copyright (c) 2017- 2020, Intel Corporation. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # =20 @@ -15,8 +15,10 @@ import git =20 import colorama =20 +from edkrepo.common.pathfix import expanduser + def init_color_console(force_color_output): - config =3D git.GitConfigParser(os.path.normpath(os.path.expanduser("~/= .gitconfig"))) + config =3D git.GitConfigParser(os.path.normpath(expanduser("~/.gitconf= ig"))) config_color =3D config.get("color", "ui", fallback=3D"auto") strip =3D not sys.stdout.isatty() convert =3D sys.stdout.isatty() diff --git a/edkrepo/config/config_factory.py b/edkrepo/config/config_facto= ry.py index 116cb24..a82a438 100644 --- a/edkrepo/config/config_factory.py +++ b/edkrepo/config/config_factory.py @@ -19,6 +19,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoWorks= paceInvalidException, E from edkrepo.common.edkrepo_exception import EdkrepoConfigFileReadOnlyExce= ption from edkrepo.common.humble import MIRROR_PRIMARY_REPOS_MISSING, MIRROR_DEC= ODE_WARNING, MAX_PATCH_SET_INVALID from edkrepo_manifest_parser import edk_manifest +from edkrepo.common.pathfix import expanduser =20 def get_edkrepo_global_data_directory(): global_data_dir =3D None @@ -33,7 +34,7 @@ def get_edkrepo_global_data_directory(): SHGetFolderPath(None, CSIDL_COMMON_APPDATA, None, SHGFP_TYPE_CURRE= NT, common_appdata) global_data_dir =3D os.path.join(common_appdata.value, "edkrepo") elif sys.platform =3D=3D "darwin" or sys.platform.startswith("linux") = or os.name =3D=3D "posix": - global_data_dir =3D os.path.expanduser("~/.edkrepo") + global_data_dir =3D expanduser("~/.edkrepo") if not os.path.isdir(global_data_dir): if not os.path.exists(os.path.dirname(global_data_dir)): raise EdkrepoGlobalDataDirectoryNotFoundException(humble.GLOBA= L_DATA_DIR_NOT_FOUND.format(os.path.dirname(global_data_dir))) --=20 2.21.0.windows.1 -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#63462): https://edk2.groups.io/g/devel/message/63462 Mute This Topic: https://groups.io/mt/75878234/1787277 Group Owner: devel+owner@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [importer@patchew.org] -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-