As was basically decided already a while ago, remove - in the simplest
possible way - the archiving of both qemu-s and mini-os from tarball
generation.

With this the subtree-force-update-all prereq isn't needed anymore in
the top level Makefile. That goal, including the respective ones
underneath tools/, then also are unreferenced and hence are being
dropped, too.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
---
This is the simplistic approach; I'm sure this could now be done quite a
bit more efficiently. I also expect there's no longer a need to run
./configure ahead of the invocation of this script, but since I have no
idea why it was needed earlier on, I'm not removing that here from the
doc. IOW all cleanup that's probably better done separately, by someone
actually using all of that machinery.
---
v3: Drop subtree-force-update-all rules from underneath tools/.
v2: Drop subtree-force-update-all prereqs from ./Makefile.

--- a/Makefile
+++ b/Makefile
@@ -200,10 +200,6 @@ rpmball: dist
 subtree-force-update: mini-os-dir-force-update
 	$(MAKE) -C tools subtree-force-update
 
-.PHONY: subtree-force-update-all
-subtree-force-update-all: mini-os-dir-force-update
-	$(MAKE) -C tools subtree-force-update-all
-
 # Make a source tarball, including qemu sub-trees.
 #
 # src-tarball will use "git describe" for the version number.  This
@@ -214,11 +210,11 @@ subtree-force-update-all: mini-os-dir-fo
 # src-tarball-release will use "make xenversion" as the version
 # number.  This is suitable for release tarballs.
 .PHONY: src-tarball-release
-src-tarball-release: subtree-force-update-all
+src-tarball-release:
 	bash ./tools/misc/mktarball $(XEN_ROOT) $$($(MAKE) -C xen xenversion --no-print-directory)
 
 .PHONY: src-tarball
-src-tarball: subtree-force-update-all
+src-tarball:
 	bash ./tools/misc/mktarball $(XEN_ROOT) $$(git describe)
 
 .PHONY: clean
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -282,8 +282,3 @@ endif
 ifeq ($(CONFIG_X86),y)
 	$(MAKE) -C firmware subtree-force-update
 endif
-
-subtree-force-update-all:
-	$(MAKE) qemu-xen-dir-force-update
-	$(MAKE) qemu-xen-traditional-dir-force-update
-	$(MAKE) -C firmware subtree-force-update-all
--- a/tools/Rules.mk
+++ b/tools/Rules.mk
@@ -206,7 +206,7 @@ subdir-all-% subdir-clean-% subdir-insta
 subdir-distclean-%: .phony
 	$(MAKE) -C $* distclean
 
-no-configure-targets := distclean subdir-distclean% clean subdir-clean% subtree-force-update-all %-dir-force-update
+no-configure-targets := distclean subdir-distclean% clean subdir-clean% %-dir-force-update
 ifeq (,$(filter $(no-configure-targets),$(MAKECMDGOALS)))
 $(XEN_ROOT)/config/Tools.mk:
 	$(error You have to run ./configure before building or installing the tools)
--- a/tools/firmware/Makefile
+++ b/tools/firmware/Makefile
@@ -127,9 +127,5 @@ ifeq ($(CONFIG_OVMF),y)
 	$(MAKE) ovmf-dir-force-update
 endif
 
-subtree-force-update-all:
-	$(MAKE) seabios-dir-force-update
-	$(MAKE) ovmf-dir-force-update
-
 subdir-all-seabios-dir: seabios-dir
 	$(MAKE) -C $< CC=$(SEABIOSCC) LD=$(SEABIOSLD) PYTHON=$(PYTHON) EXTRAVERSION=$(SEABIOS_EXTRAVERSION) all;
--- a/tools/misc/mktarball
+++ b/tools/misc/mktarball
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# mktarball: Make a release tarball (including xen, qemu, and qemu-traditional)
+# mktarball: Make a release tarball (including just xen)
 #
 # Takes 2 arguments, the path to the dist directory and the version
 set -ex
@@ -29,26 +29,6 @@ mkdir -p $tdir
 
 git_archive_into $xen_root $tdir/xen-$desc
 
-# We can't use git_archive_into with qemu upstream because it uses
-# git-submodules.  git-submodules are an inherently broken git feature
-# which should never be used in any circumstance.  Unfortunately, qemu
-# upstream uses them.  Relevantly for us, git archive does not work
-# properly when there are submodules.
-(
-    cd $xen_root/tools/qemu-xen-dir-remote
-    # if it's not clean, the qemu script will call `git stash' !
-    git --no-pager diff --stat HEAD
-    scripts/archive-source.sh $tdir/xen-$desc/tools/qemu-xen.tar
-    cd $tdir/xen-$desc/tools
-    mkdir qemu-xen
-    tar <qemu-xen.tar Cxf qemu-xen -
-    rm qemu-xen.tar
-)
-
-git_archive_into $xen_root/tools/qemu-xen-traditional-dir-remote $tdir/xen-$desc/tools/qemu-xen-traditional
-
-git_archive_into $xen_root/extras/mini-os-remote $tdir/xen-$desc/extras/mini-os
-
 GZIP=-9v tar cz -f $xen_root/dist/xen-$desc.tar.gz -C $tdir xen-$desc
 
 echo "Source tarball in $xen_root/dist/xen-$desc.tar.gz"

On 19/08/2024 4:10 pm, Jan Beulich wrote:
> As was basically decided already a while ago, remove - in the simplest
> possible way - the archiving of both qemu-s and mini-os from tarball
> generation.
>
> With this the subtree-force-update-all prereq isn't needed anymore in
> the top level Makefile. That goal, including the respective ones
> underneath tools/, then also are unreferenced and hence are being
> dropped, too.
>
> Signed-off-by: Jan Beulich <jbeulich@suse.com>
> ---
> This is the simplistic approach; I'm sure this could now be done quite a
> bit more efficiently. I also expect there's no longer a need to run
> ./configure ahead of the invocation of this script, but since I have no
> idea why it was needed earlier on, I'm not removing that here from the
> doc. IOW all cleanup that's probably better done separately, by someone
> actually using all of that machinery.
> ---
> v3: Drop subtree-force-update-all rules from underneath tools/.
> v2: Drop subtree-force-update-all prereqs from ./Makefile.

Removing the archiving is one thing, and we have agreed that's a good
thing to do.

However, the makefile rules might be used manually.  While we have the
other trees cloned in Xen, I'd be wary about dropping the rules.

I guess the real question is how long we expect this interim state to last.

SeaBIOS and iPXE could disappear right away; there's nothing xen-local
there at all.

OVMF doesn't have anything xen-local, but there's a bit of complexity
setting up a Xen build IIRC.

Qemu-upstream won't be going anywhere until a) a release of Xen with
stabilised hypercalls, b) a release of Qemu after that

~Andrew

On 04.09.2024 11:59, Andrew Cooper wrote:
> On 19/08/2024 4:10 pm, Jan Beulich wrote:
>> As was basically decided already a while ago, remove - in the simplest
>> possible way - the archiving of both qemu-s and mini-os from tarball
>> generation.
>>
>> With this the subtree-force-update-all prereq isn't needed anymore in
>> the top level Makefile. That goal, including the respective ones
>> underneath tools/, then also are unreferenced and hence are being
>> dropped, too.
>>
>> Signed-off-by: Jan Beulich <jbeulich@suse.com>
>> ---
>> This is the simplistic approach; I'm sure this could now be done quite a
>> bit more efficiently. I also expect there's no longer a need to run
>> ./configure ahead of the invocation of this script, but since I have no
>> idea why it was needed earlier on, I'm not removing that here from the
>> doc. IOW all cleanup that's probably better done separately, by someone
>> actually using all of that machinery.
>> ---
>> v3: Drop subtree-force-update-all rules from underneath tools/.
>> v2: Drop subtree-force-update-all prereqs from ./Makefile.
> 
> Removing the archiving is one thing, and we have agreed that's a good
> thing to do.
> 
> However, the makefile rules might be used manually.  While we have the
> other trees cloned in Xen, I'd be wary about dropping the rules.

They're not mentioned in what the help: goal displays, which makes me
worry not overly much here. And Anthony had basically agreed to their
dropping, in part by asking me to drop more than I did initially.

> I guess the real question is how long we expect this interim state to last.

For quite some more time, I suppose, ...

> SeaBIOS and iPXE could disappear right away; there's nothing xen-local
> there at all.
> 
> OVMF doesn't have anything xen-local, but there's a bit of complexity
> setting up a Xen build IIRC.
> 
> Qemu-upstream won't be going anywhere until a) a release of Xen with
> stabilised hypercalls, b) a release of Qemu after that

... seeing for how long we've been talking of stabilized hypercalls.
But how does it matter here for how much longer that situation will
remain? We want release tarballs of just xen.git, and that's what
this patch achieves (plus some cleanup of then-orphan logic). How
else would you propose we reach said goal?

Jan

On 04/09/2024 11:26 am, Jan Beulich wrote:
> On 04.09.2024 11:59, Andrew Cooper wrote:
>> On 19/08/2024 4:10 pm, Jan Beulich wrote:
>>> As was basically decided already a while ago, remove - in the simplest
>>> possible way - the archiving of both qemu-s and mini-os from tarball
>>> generation.
>>>
>>> With this the subtree-force-update-all prereq isn't needed anymore in
>>> the top level Makefile. That goal, including the respective ones
>>> underneath tools/, then also are unreferenced and hence are being
>>> dropped, too.
>>>
>>> Signed-off-by: Jan Beulich <jbeulich@suse.com>
>>> ---
>>> This is the simplistic approach; I'm sure this could now be done quite a
>>> bit more efficiently. I also expect there's no longer a need to run
>>> ./configure ahead of the invocation of this script, but since I have no
>>> idea why it was needed earlier on, I'm not removing that here from the
>>> doc. IOW all cleanup that's probably better done separately, by someone
>>> actually using all of that machinery.
>>> ---
>>> v3: Drop subtree-force-update-all rules from underneath tools/.
>>> v2: Drop subtree-force-update-all prereqs from ./Makefile.
>> Removing the archiving is one thing, and we have agreed that's a good
>> thing to do.
>>
>> However, the makefile rules might be used manually.  While we have the
>> other trees cloned in Xen, I'd be wary about dropping the rules.
> They're not mentioned in what the help: goal displays, which makes me
> worry not overly much here. And Anthony had basically agreed to their
> dropping, in part by asking me to drop more than I did initially.

Fine.  Acked-by: Andrew Cooper <andrew.cooper3@citrix.com>

On 19.08.2024 17:10, Jan Beulich wrote:
> As was basically decided already a while ago, remove - in the simplest
> possible way - the archiving of both qemu-s and mini-os from tarball
> generation.
> 
> With this the subtree-force-update-all prereq isn't needed anymore in
> the top level Makefile. That goal, including the respective ones
> underneath tools/, then also are unreferenced and hence are being
> dropped, too.
> 
> Signed-off-by: Jan Beulich <jbeulich@suse.com>
> ---
> This is the simplistic approach; I'm sure this could now be done quite a
> bit more efficiently. I also expect there's no longer a need to run
> ./configure ahead of the invocation of this script, but since I have no
> idea why it was needed earlier on, I'm not removing that here from the
> doc. IOW all cleanup that's probably better done separately, by someone
> actually using all of that machinery.
> ---
> v3: Drop subtree-force-update-all rules from underneath tools/.
> v2: Drop subtree-force-update-all prereqs from ./Makefile.

Are any further adjustments needed?

Jan

> --- a/Makefile
> +++ b/Makefile
> @@ -200,10 +200,6 @@ rpmball: dist
>  subtree-force-update: mini-os-dir-force-update
>  	$(MAKE) -C tools subtree-force-update
>  
> -.PHONY: subtree-force-update-all
> -subtree-force-update-all: mini-os-dir-force-update
> -	$(MAKE) -C tools subtree-force-update-all
> -
>  # Make a source tarball, including qemu sub-trees.
>  #
>  # src-tarball will use "git describe" for the version number.  This
> @@ -214,11 +210,11 @@ subtree-force-update-all: mini-os-dir-fo
>  # src-tarball-release will use "make xenversion" as the version
>  # number.  This is suitable for release tarballs.
>  .PHONY: src-tarball-release
> -src-tarball-release: subtree-force-update-all
> +src-tarball-release:
>  	bash ./tools/misc/mktarball $(XEN_ROOT) $$($(MAKE) -C xen xenversion --no-print-directory)
>  
>  .PHONY: src-tarball
> -src-tarball: subtree-force-update-all
> +src-tarball:
>  	bash ./tools/misc/mktarball $(XEN_ROOT) $$(git describe)
>  
>  .PHONY: clean
> --- a/tools/Makefile
> +++ b/tools/Makefile
> @@ -282,8 +282,3 @@ endif
>  ifeq ($(CONFIG_X86),y)
>  	$(MAKE) -C firmware subtree-force-update
>  endif
> -
> -subtree-force-update-all:
> -	$(MAKE) qemu-xen-dir-force-update
> -	$(MAKE) qemu-xen-traditional-dir-force-update
> -	$(MAKE) -C firmware subtree-force-update-all
> --- a/tools/Rules.mk
> +++ b/tools/Rules.mk
> @@ -206,7 +206,7 @@ subdir-all-% subdir-clean-% subdir-insta
>  subdir-distclean-%: .phony
>  	$(MAKE) -C $* distclean
>  
> -no-configure-targets := distclean subdir-distclean% clean subdir-clean% subtree-force-update-all %-dir-force-update
> +no-configure-targets := distclean subdir-distclean% clean subdir-clean% %-dir-force-update
>  ifeq (,$(filter $(no-configure-targets),$(MAKECMDGOALS)))
>  $(XEN_ROOT)/config/Tools.mk:
>  	$(error You have to run ./configure before building or installing the tools)
> --- a/tools/firmware/Makefile
> +++ b/tools/firmware/Makefile
> @@ -127,9 +127,5 @@ ifeq ($(CONFIG_OVMF),y)
>  	$(MAKE) ovmf-dir-force-update
>  endif
>  
> -subtree-force-update-all:
> -	$(MAKE) seabios-dir-force-update
> -	$(MAKE) ovmf-dir-force-update
> -
>  subdir-all-seabios-dir: seabios-dir
>  	$(MAKE) -C $< CC=$(SEABIOSCC) LD=$(SEABIOSLD) PYTHON=$(PYTHON) EXTRAVERSION=$(SEABIOS_EXTRAVERSION) all;
> --- a/tools/misc/mktarball
> +++ b/tools/misc/mktarball
> @@ -1,6 +1,6 @@
>  #!/bin/bash
>  #
> -# mktarball: Make a release tarball (including xen, qemu, and qemu-traditional)
> +# mktarball: Make a release tarball (including just xen)
>  #
>  # Takes 2 arguments, the path to the dist directory and the version
>  set -ex
> @@ -29,26 +29,6 @@ mkdir -p $tdir
>  
>  git_archive_into $xen_root $tdir/xen-$desc
>  
> -# We can't use git_archive_into with qemu upstream because it uses
> -# git-submodules.  git-submodules are an inherently broken git feature
> -# which should never be used in any circumstance.  Unfortunately, qemu
> -# upstream uses them.  Relevantly for us, git archive does not work
> -# properly when there are submodules.
> -(
> -    cd $xen_root/tools/qemu-xen-dir-remote
> -    # if it's not clean, the qemu script will call `git stash' !
> -    git --no-pager diff --stat HEAD
> -    scripts/archive-source.sh $tdir/xen-$desc/tools/qemu-xen.tar
> -    cd $tdir/xen-$desc/tools
> -    mkdir qemu-xen
> -    tar <qemu-xen.tar Cxf qemu-xen -
> -    rm qemu-xen.tar
> -)
> -
> -git_archive_into $xen_root/tools/qemu-xen-traditional-dir-remote $tdir/xen-$desc/tools/qemu-xen-traditional
> -
> -git_archive_into $xen_root/extras/mini-os-remote $tdir/xen-$desc/extras/mini-os
> -
>  GZIP=-9v tar cz -f $xen_root/dist/xen-$desc.tar.gz -C $tdir xen-$desc
>  
>  echo "Source tarball in $xen_root/dist/xen-$desc.tar.gz"