To avoid lots of copy and paste lets deal with artefacts in a
template. This way we can filter out most of the pre-binary object and
library files we no longer need as we have the final binaries.
build-system-alpine also saved .git-submodule-status so for simplicity
we bring that into the template as well.
As an example the build-system-ubuntu artefacts before this patch
where around 1.3 GB, after dropping the object files it comes to 970
MB.
Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
---
.gitlab-ci.d/buildtest-template.yml | 16 ++++++
.gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------
2 files changed, 46 insertions(+), 51 deletions(-)
diff --git a/.gitlab-ci.d/buildtest-template.yml b/.gitlab-ci.d/buildtest-template.yml
index cb96b55c3f..a6cfe9be97 100644
--- a/.gitlab-ci.d/buildtest-template.yml
+++ b/.gitlab-ci.d/buildtest-template.yml
@@ -25,6 +25,22 @@
make -j"$JOBS" $MAKE_CHECK_ARGS ;
fi
+# We jump some hoops in common_test_job_template to avoid
+# rebuilding all the object files we skip in the artifacts
+.native_build_artifact_template:
+ artifacts:
+ expire_in: 2 days
+ paths:
+ - build
+ - .git-submodule-status
+ exclude:
+ - build/**/*.p
+ - build/**/*.a.p
+ - build/**/*.fa.p
+ - build/**/*.c.o
+ - build/**/*.c.o.d
+ - build/**/*.fa
+
.common_test_job_template:
extends: .base_job_template
stage: test
diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml
index 43f9e4a81d..44b8275299 100644
--- a/.gitlab-ci.d/buildtest.yml
+++ b/.gitlab-ci.d/buildtest.yml
@@ -2,7 +2,9 @@ include:
- local: '/.gitlab-ci.d/buildtest-template.yml'
build-system-alpine:
- extends: .native_build_job_template
+ extends:
+ - .native_build_job_template
+ - .native_build_artifact_template
needs:
- job: amd64-alpine-container
variables:
@@ -11,11 +13,6 @@ build-system-alpine:
microblazeel-softmmu mips64el-softmmu
MAKE_CHECK_ARGS: check-build
CONFIGURE_ARGS: --enable-docs --enable-trace-backends=log,simple,syslog
- artifacts:
- expire_in: 2 days
- paths:
- - .git-submodule-status
- - build
check-system-alpine:
extends: .native_test_job_template
@@ -36,7 +33,9 @@ avocado-system-alpine:
MAKE_CHECK_ARGS: check-avocado
build-system-ubuntu:
- extends: .native_build_job_template
+ extends:
+ - .native_build_job_template
+ - .native_build_artifact_template
needs:
job: amd64-ubuntu2204-container
variables:
@@ -45,10 +44,6 @@ build-system-ubuntu:
TARGETS: alpha-softmmu cris-softmmu hppa-softmmu
microblazeel-softmmu mips64el-softmmu
MAKE_CHECK_ARGS: check-build
- artifacts:
- expire_in: 2 days
- paths:
- - build
check-system-ubuntu:
extends: .native_test_job_template
@@ -69,7 +64,9 @@ avocado-system-ubuntu:
MAKE_CHECK_ARGS: check-avocado
build-system-debian:
- extends: .native_build_job_template
+ extends:
+ - .native_build_job_template
+ - .native_build_artifact_template
needs:
job: amd64-debian-container
variables:
@@ -78,10 +75,6 @@ build-system-debian:
TARGETS: arm-softmmu avr-softmmu i386-softmmu mipsel-softmmu
riscv64-softmmu sh4eb-softmmu sparc-softmmu xtensaeb-softmmu
MAKE_CHECK_ARGS: check-build
- artifacts:
- expire_in: 2 days
- paths:
- - build
check-system-debian:
extends: .native_test_job_template
@@ -114,7 +107,9 @@ crash-test-debian:
- tests/venv/bin/python3 scripts/device-crash-test -q ./qemu-system-i386
build-system-fedora:
- extends: .native_build_job_template
+ extends:
+ - .native_build_job_template
+ - .native_build_artifact_template
needs:
job: amd64-fedora-container
variables:
@@ -123,10 +118,6 @@ build-system-fedora:
TARGETS: tricore-softmmu microblaze-softmmu mips-softmmu
xtensa-softmmu m68k-softmmu riscv32-softmmu ppc-softmmu sparc64-softmmu
MAKE_CHECK_ARGS: check-build
- artifacts:
- expire_in: 2 days
- paths:
- - build
check-system-fedora:
extends: .native_test_job_template
@@ -160,7 +151,9 @@ crash-test-fedora:
- tests/venv/bin/python3 scripts/device-crash-test -q ./qemu-system-riscv32
build-system-centos:
- extends: .native_build_job_template
+ extends:
+ - .native_build_job_template
+ - .native_build_artifact_template
needs:
job: amd64-centos8-container
variables:
@@ -170,10 +163,6 @@ build-system-centos:
TARGETS: ppc64-softmmu or1k-softmmu s390x-softmmu
x86_64-softmmu rx-softmmu sh4-softmmu nios2-softmmu
MAKE_CHECK_ARGS: check-build
- artifacts:
- expire_in: 2 days
- paths:
- - build
check-system-centos:
extends: .native_test_job_template
@@ -194,17 +183,15 @@ avocado-system-centos:
MAKE_CHECK_ARGS: check-avocado
build-system-opensuse:
- extends: .native_build_job_template
+ extends:
+ - .native_build_job_template
+ - .native_build_artifact_template
needs:
job: amd64-opensuse-leap-container
variables:
IMAGE: opensuse-leap
TARGETS: s390x-softmmu x86_64-softmmu aarch64-softmmu
MAKE_CHECK_ARGS: check-build
- artifacts:
- expire_in: 2 days
- paths:
- - build
check-system-opensuse:
extends: .native_test_job_template
@@ -339,7 +326,9 @@ clang-user:
# Split in three sets of build/check/avocado to limit the execution time of each
# job
build-cfi-aarch64:
- extends: .native_build_job_template
+ extends:
+ - .native_build_job_template
+ - .native_build_artifact_template
needs:
- job: amd64-fedora-container
variables:
@@ -355,10 +344,6 @@ build-cfi-aarch64:
# skipped until the situation has been solved.
QEMU_JOB_SKIPPED: 1
timeout: 90m
- artifacts:
- expire_in: 2 days
- paths:
- - build
check-cfi-aarch64:
extends: .native_test_job_template
@@ -379,7 +364,9 @@ avocado-cfi-aarch64:
MAKE_CHECK_ARGS: check-avocado
build-cfi-ppc64-s390x:
- extends: .native_build_job_template
+ extends:
+ - .native_build_job_template
+ - .native_build_artifact_template
needs:
- job: amd64-fedora-container
variables:
@@ -395,10 +382,6 @@ build-cfi-ppc64-s390x:
# skipped until the situation has been solved.
QEMU_JOB_SKIPPED: 1
timeout: 80m
- artifacts:
- expire_in: 2 days
- paths:
- - build
check-cfi-ppc64-s390x:
extends: .native_test_job_template
@@ -419,7 +402,9 @@ avocado-cfi-ppc64-s390x:
MAKE_CHECK_ARGS: check-avocado
build-cfi-x86_64:
- extends: .native_build_job_template
+ extends:
+ - .native_build_job_template
+ - .native_build_artifact_template
needs:
- job: amd64-fedora-container
variables:
@@ -431,10 +416,6 @@ build-cfi-x86_64:
TARGETS: x86_64-softmmu
MAKE_CHECK_ARGS: check-build
timeout: 70m
- artifacts:
- expire_in: 2 days
- paths:
- - build
check-cfi-x86_64:
extends: .native_test_job_template
@@ -564,7 +545,9 @@ build-libvhost-user:
# No targets are built here, just tools, docs, and unit tests. This
# also feeds into the eventual documentation deployment steps later
build-tools-and-docs-debian:
- extends: .native_build_job_template
+ extends:
+ - .native_build_job_template
+ - .native_build_artifact_template
needs:
job: amd64-debian-container
# when running on 'master' we use pre-existing container
@@ -574,10 +557,6 @@ build-tools-and-docs-debian:
MAKE_CHECK_ARGS: check-unit ctags TAGS cscope
CONFIGURE_ARGS: --disable-system --disable-user --enable-docs --enable-tools
QEMU_JOB_PUBLISH: 1
- artifacts:
- expire_in: 2 days
- paths:
- - build
# Prepare for GitLab pages deployment. Anything copied into the
# "public" directory will be deployed to $USER.gitlab.io/$PROJECT
--
2.39.2
On 28/2/23 20:06, Alex Bennée wrote: > To avoid lots of copy and paste lets deal with artefacts in a > template. This way we can filter out most of the pre-binary object and > library files we no longer need as we have the final binaries. > > build-system-alpine also saved .git-submodule-status so for simplicity > we bring that into the template as well. > > As an example the build-system-ubuntu artefacts before this patch > where around 1.3 GB, after dropping the object files it comes to 970 > MB. > > Signed-off-by: Alex Bennée <alex.bennee@linaro.org> > --- > .gitlab-ci.d/buildtest-template.yml | 16 ++++++ > .gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------ > 2 files changed, 46 insertions(+), 51 deletions(-) > > diff --git a/.gitlab-ci.d/buildtest-template.yml b/.gitlab-ci.d/buildtest-template.yml > index cb96b55c3f..a6cfe9be97 100644 > --- a/.gitlab-ci.d/buildtest-template.yml > +++ b/.gitlab-ci.d/buildtest-template.yml > @@ -25,6 +25,22 @@ > make -j"$JOBS" $MAKE_CHECK_ARGS ; > fi > > +# We jump some hoops in common_test_job_template to avoid > +# rebuilding all the object files we skip in the artifacts > +.native_build_artifact_template: > + artifacts: > + expire_in: 2 days > + paths: > + - build > + - .git-submodule-status > + exclude: > + - build/**/*.p > + - build/**/*.a.p > + - build/**/*.fa.p > + - build/**/*.c.o > + - build/**/*.c.o.d > + - build/**/*.fa Why not exclude "build/**/*.a", was there some issue? Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
On 28/2/23 20:06, Alex Bennée wrote: > To avoid lots of copy and paste lets deal with artefacts in a > template. This way we can filter out most of the pre-binary object and > library files we no longer need as we have the final binaries. > > build-system-alpine also saved .git-submodule-status so for simplicity > we bring that into the template as well. > > As an example the build-system-ubuntu artefacts before this patch > where around 1.3 GB, after dropping the object files it comes to 970 > MB. > > Signed-off-by: Alex Bennée <alex.bennee@linaro.org> > --- > .gitlab-ci.d/buildtest-template.yml | 16 ++++++ > .gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------ > 2 files changed, 46 insertions(+), 51 deletions(-) > > diff --git a/.gitlab-ci.d/buildtest-template.yml b/.gitlab-ci.d/buildtest-template.yml > index cb96b55c3f..a6cfe9be97 100644 > --- a/.gitlab-ci.d/buildtest-template.yml > +++ b/.gitlab-ci.d/buildtest-template.yml > @@ -25,6 +25,22 @@ > make -j"$JOBS" $MAKE_CHECK_ARGS ; > fi > > +# We jump some hoops in common_test_job_template to avoid > +# rebuilding all the object files we skip in the artifacts > +.native_build_artifact_template: > + artifacts: > + expire_in: 2 days > + paths: > + - build > + - .git-submodule-status > + exclude: > + - build/**/*.p > + - build/**/*.a.p > + - build/**/*.fa.p > + - build/**/*.c.o > + - build/**/*.c.o.d > + - build/**/*.fa > + > .common_test_job_template: > extends: .base_job_template > stage: test > diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml > index 43f9e4a81d..44b8275299 100644 > --- a/.gitlab-ci.d/buildtest.yml > +++ b/.gitlab-ci.d/buildtest.yml > @@ -2,7 +2,9 @@ include: > - local: '/.gitlab-ci.d/buildtest-template.yml' > > build-system-alpine: > - extends: .native_build_job_template > + extends: > + - .native_build_job_template > + - .native_build_artifact_template I'm confused... Apparently this doesn't work: https://gitlab.com/stsquad/qemu/-/jobs/3847747681/artifacts/browse
Philippe Mathieu-Daudé <philmd@linaro.org> writes: > On 28/2/23 20:06, Alex Bennée wrote: >> To avoid lots of copy and paste lets deal with artefacts in a >> template. This way we can filter out most of the pre-binary object and >> library files we no longer need as we have the final binaries. >> build-system-alpine also saved .git-submodule-status so for >> simplicity >> we bring that into the template as well. >> As an example the build-system-ubuntu artefacts before this patch >> where around 1.3 GB, after dropping the object files it comes to 970 >> MB. >> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> >> --- >> .gitlab-ci.d/buildtest-template.yml | 16 ++++++ >> .gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------ >> 2 files changed, 46 insertions(+), 51 deletions(-) >> diff --git a/.gitlab-ci.d/buildtest-template.yml >> b/.gitlab-ci.d/buildtest-template.yml >> index cb96b55c3f..a6cfe9be97 100644 >> --- a/.gitlab-ci.d/buildtest-template.yml >> +++ b/.gitlab-ci.d/buildtest-template.yml >> @@ -25,6 +25,22 @@ >> make -j"$JOBS" $MAKE_CHECK_ARGS ; >> fi >> +# We jump some hoops in common_test_job_template to avoid >> +# rebuilding all the object files we skip in the artifacts >> +.native_build_artifact_template: >> + artifacts: >> + expire_in: 2 days >> + paths: >> + - build >> + - .git-submodule-status >> + exclude: >> + - build/**/*.p >> + - build/**/*.a.p >> + - build/**/*.fa.p >> + - build/**/*.c.o >> + - build/**/*.c.o.d >> + - build/**/*.fa >> + >> .common_test_job_template: >> extends: .base_job_template >> stage: test >> diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml >> index 43f9e4a81d..44b8275299 100644 >> --- a/.gitlab-ci.d/buildtest.yml >> +++ b/.gitlab-ci.d/buildtest.yml >> @@ -2,7 +2,9 @@ include: >> - local: '/.gitlab-ci.d/buildtest-template.yml' >> build-system-alpine: >> - extends: .native_build_job_template >> + extends: >> + - .native_build_job_template >> + - .native_build_artifact_template > > I'm confused... Apparently this doesn't work: > https://gitlab.com/stsquad/qemu/-/jobs/3847747681/artifacts/browse Nope you are one run behind ;-) -- Alex Bennée Virtualisation Tech Lead @ Linaro
On 28/2/23 20:06, Alex Bennée wrote: > To avoid lots of copy and paste lets deal with artefacts in a > template. This way we can filter out most of the pre-binary object and > library files we no longer need as we have the final binaries. > > build-system-alpine also saved .git-submodule-status so for simplicity > we bring that into the template as well. > > As an example the build-system-ubuntu artefacts before this patch > where around 1.3 GB, after dropping the object files it comes to 970 > MB. > > Signed-off-by: Alex Bennée <alex.bennee@linaro.org> > --- > .gitlab-ci.d/buildtest-template.yml | 16 ++++++ > .gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------ > 2 files changed, 46 insertions(+), 51 deletions(-) This is still kludging the fact that 'make check-qtest' rebuild the world even if QEMU binaries are present. Still an improvement, so Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
On 28/02/2023 22.41, Philippe Mathieu-Daudé wrote: > On 28/2/23 20:06, Alex Bennée wrote: >> To avoid lots of copy and paste lets deal with artefacts in a >> template. This way we can filter out most of the pre-binary object and >> library files we no longer need as we have the final binaries. >> >> build-system-alpine also saved .git-submodule-status so for simplicity >> we bring that into the template as well. >> >> As an example the build-system-ubuntu artefacts before this patch >> where around 1.3 GB, after dropping the object files it comes to 970 >> MB. >> >> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> >> --- >> .gitlab-ci.d/buildtest-template.yml | 16 ++++++ >> .gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------ >> 2 files changed, 46 insertions(+), 51 deletions(-) > > This is still kludging the fact that 'make check-qtest' rebuild > the world even if QEMU binaries are present. Did you try? ... if so, that would kill even more CI minutes, i.e. it would IMHO be a show-stopper for this patch. Thomas
On 1/3/23 07:51, Thomas Huth wrote: > On 28/02/2023 22.41, Philippe Mathieu-Daudé wrote: >> On 28/2/23 20:06, Alex Bennée wrote: >>> To avoid lots of copy and paste lets deal with artefacts in a >>> template. This way we can filter out most of the pre-binary object and >>> library files we no longer need as we have the final binaries. >>> >>> build-system-alpine also saved .git-submodule-status so for simplicity >>> we bring that into the template as well. >>> >>> As an example the build-system-ubuntu artefacts before this patch >>> where around 1.3 GB, after dropping the object files it comes to 970 >>> MB. >>> >>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> >>> --- >>> .gitlab-ci.d/buildtest-template.yml | 16 ++++++ >>> .gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------ >>> 2 files changed, 46 insertions(+), 51 deletions(-) >> >> This is still kludging the fact that 'make check-qtest' rebuild >> the world even if QEMU binaries are present. > > Did you try? ... if so, that would kill even more CI minutes, i.e. it > would IMHO be a show-stopper for this patch. I remember we had to pass all build/ otherwise calling 'make check-qtest check-tcg' would rebuild the same binaries, it it was pointless to split the jobs in 2 stages. I might have missed when that was fixed. I haven't tried, however I see in Alex job: $ scripts/git-submodule.sh update $(sed -n '/GIT_SUBMODULES=/ s/.*=// p' build/config-host.mak) sed: can't read build/config-host.mak: No such file or directory $ cd build /bin/bash: line 144: cd: build: No such file or directory ERROR: Job failed: exit code 1 https://gitlab.com/stsquad/qemu/-/jobs/3847747769
Philippe Mathieu-Daudé <philmd@linaro.org> writes: > On 1/3/23 07:51, Thomas Huth wrote: >> On 28/02/2023 22.41, Philippe Mathieu-Daudé wrote: >>> On 28/2/23 20:06, Alex Bennée wrote: >>>> To avoid lots of copy and paste lets deal with artefacts in a >>>> template. This way we can filter out most of the pre-binary object and >>>> library files we no longer need as we have the final binaries. >>>> >>>> build-system-alpine also saved .git-submodule-status so for simplicity >>>> we bring that into the template as well. >>>> >>>> As an example the build-system-ubuntu artefacts before this patch >>>> where around 1.3 GB, after dropping the object files it comes to 970 >>>> MB. >>>> >>>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> >>>> --- >>>> .gitlab-ci.d/buildtest-template.yml | 16 ++++++ >>>> .gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------ >>>> 2 files changed, 46 insertions(+), 51 deletions(-) >>> >>> This is still kludging the fact that 'make check-qtest' rebuild >>> the world even if QEMU binaries are present. >> Did you try? ... if so, that would kill even more CI minutes, i.e. >> it would IMHO be a show-stopper for this patch. > > I remember we had to pass all build/ otherwise calling 'make > check-qtest check-tcg' would rebuild the same binaries, it it > was pointless to split the jobs in 2 stages. I might have missed > when that was fixed. > > I haven't tried, however I see in Alex job: > > $ scripts/git-submodule.sh update $(sed -n '/GIT_SUBMODULES=/ s/.*=// > p' build/config-host.mak) > sed: can't read build/config-host.mak: No such file or directory > $ cd build > /bin/bash: line 144: cd: build: No such file or directory > ERROR: Job failed: exit code 1 > > https://gitlab.com/stsquad/qemu/-/jobs/3847747769 That was an older run of the job where I thought the skip path would merge from the template. In the end I've just include git-submodules in all the build aretfacts instead of just for alpine. It ran fine: https://gitlab.com/stsquad/qemu/-/jobs/3849512854 -- Alex Bennée Virtualisation Tech Lead @ Linaro
On 1/3/23 11:00, Alex Bennée wrote: > > Philippe Mathieu-Daudé <philmd@linaro.org> writes: > >> On 1/3/23 07:51, Thomas Huth wrote: >>> On 28/02/2023 22.41, Philippe Mathieu-Daudé wrote: >>>> On 28/2/23 20:06, Alex Bennée wrote: >>>>> To avoid lots of copy and paste lets deal with artefacts in a >>>>> template. This way we can filter out most of the pre-binary object and >>>>> library files we no longer need as we have the final binaries. >>>>> >>>>> build-system-alpine also saved .git-submodule-status so for simplicity >>>>> we bring that into the template as well. >>>>> >>>>> As an example the build-system-ubuntu artefacts before this patch >>>>> where around 1.3 GB, after dropping the object files it comes to 970 >>>>> MB. >>>>> >>>>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> >>>>> --- >>>>> .gitlab-ci.d/buildtest-template.yml | 16 ++++++ >>>>> .gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------ >>>>> 2 files changed, 46 insertions(+), 51 deletions(-) >>>> >>>> This is still kludging the fact that 'make check-qtest' rebuild >>>> the world even if QEMU binaries are present. >>> Did you try? ... if so, that would kill even more CI minutes, i.e. >>> it would IMHO be a show-stopper for this patch. >> >> I remember we had to pass all build/ otherwise calling 'make >> check-qtest check-tcg' would rebuild the same binaries, it it >> was pointless to split the jobs in 2 stages. I might have missed >> when that was fixed. >> >> I haven't tried, however I see in Alex job: >> >> $ scripts/git-submodule.sh update $(sed -n '/GIT_SUBMODULES=/ s/.*=// >> p' build/config-host.mak) >> sed: can't read build/config-host.mak: No such file or directory >> $ cd build >> /bin/bash: line 144: cd: build: No such file or directory >> ERROR: Job failed: exit code 1 >> >> https://gitlab.com/stsquad/qemu/-/jobs/3847747769 > > That was an older run of the job where I thought the skip path would > merge from the template. In the end I've just include git-submodules in > all the build aretfacts instead of just for alpine. > > It ran fine: > > https://gitlab.com/stsquad/qemu/-/jobs/3849512854 Indeed: https://gitlab.com/stsquad/qemu/-/jobs/3849512800/artifacts/browse/build/ So a good improvement, but I'd prefer having a stricter "only include what we need" policy rather than "exclude what we don't". The former will catch missing additions, while the later won't notice until someone spend time excluding pointless files manually again.
Philippe Mathieu-Daudé <philmd@linaro.org> writes: > On 1/3/23 11:00, Alex Bennée wrote: >> Philippe Mathieu-Daudé <philmd@linaro.org> writes: >> >>> On 1/3/23 07:51, Thomas Huth wrote: >>>> On 28/02/2023 22.41, Philippe Mathieu-Daudé wrote: >>>>> On 28/2/23 20:06, Alex Bennée wrote: >>>>>> To avoid lots of copy and paste lets deal with artefacts in a >>>>>> template. This way we can filter out most of the pre-binary object and >>>>>> library files we no longer need as we have the final binaries. >>>>>> >>>>>> build-system-alpine also saved .git-submodule-status so for simplicity >>>>>> we bring that into the template as well. >>>>>> >>>>>> As an example the build-system-ubuntu artefacts before this patch >>>>>> where around 1.3 GB, after dropping the object files it comes to 970 >>>>>> MB. >>>>>> >>>>>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> >>>>>> --- >>>>>> .gitlab-ci.d/buildtest-template.yml | 16 ++++++ >>>>>> .gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------ >>>>>> 2 files changed, 46 insertions(+), 51 deletions(-) >>>>> >>>>> This is still kludging the fact that 'make check-qtest' rebuild >>>>> the world even if QEMU binaries are present. >>>> Did you try? ... if so, that would kill even more CI minutes, i.e. >>>> it would IMHO be a show-stopper for this patch. >>> >>> I remember we had to pass all build/ otherwise calling 'make >>> check-qtest check-tcg' would rebuild the same binaries, it it >>> was pointless to split the jobs in 2 stages. I might have missed >>> when that was fixed. >>> >>> I haven't tried, however I see in Alex job: >>> >>> $ scripts/git-submodule.sh update $(sed -n '/GIT_SUBMODULES=/ s/.*=// >>> p' build/config-host.mak) >>> sed: can't read build/config-host.mak: No such file or directory >>> $ cd build >>> /bin/bash: line 144: cd: build: No such file or directory >>> ERROR: Job failed: exit code 1 >>> >>> https://gitlab.com/stsquad/qemu/-/jobs/3847747769 >> That was an older run of the job where I thought the skip path would >> merge from the template. In the end I've just include git-submodules in >> all the build aretfacts instead of just for alpine. >> It ran fine: >> https://gitlab.com/stsquad/qemu/-/jobs/3849512854 > > Indeed: > https://gitlab.com/stsquad/qemu/-/jobs/3849512800/artifacts/browse/build/ > > So a good improvement, but I'd prefer having a stricter "only include > what we need" policy rather than "exclude what we don't". The former > will catch missing additions, while the later won't notice until someone > spend time excluding pointless files manually again. I'd argue the opposite because its hard to come up with patterns for all the binaries we build but easy for all the object files. -- Alex Bennée Virtualisation Tech Lead @ Linaro
Philippe Mathieu-Daudé <philmd@linaro.org> writes:
> On 28/2/23 20:06, Alex Bennée wrote:
>> To avoid lots of copy and paste lets deal with artefacts in a
>> template. This way we can filter out most of the pre-binary object and
>> library files we no longer need as we have the final binaries.
>> build-system-alpine also saved .git-submodule-status so for
>> simplicity
>> we bring that into the template as well.
>> As an example the build-system-ubuntu artefacts before this patch
>> where around 1.3 GB, after dropping the object files it comes to 970
>> MB.
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> ---
>> .gitlab-ci.d/buildtest-template.yml | 16 ++++++
>> .gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------
>> 2 files changed, 46 insertions(+), 51 deletions(-)
>
> This is still kludging the fact that 'make check-qtest' rebuild
> the world even if QEMU binaries are present. Still an improvement, so
> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Is something being missed by:
# Avoid recompiling by hiding ninja with NINJA=":"
- make NINJA=":" $MAKE_CHECK_ARGS
?
--
Alex Bennée
Virtualisation Tech Lead @ Linaro
On 28/2/23 22:55, Alex Bennée wrote: > > Philippe Mathieu-Daudé <philmd@linaro.org> writes: > >> On 28/2/23 20:06, Alex Bennée wrote: >>> To avoid lots of copy and paste lets deal with artefacts in a >>> template. This way we can filter out most of the pre-binary object and >>> library files we no longer need as we have the final binaries. >>> build-system-alpine also saved .git-submodule-status so for >>> simplicity >>> we bring that into the template as well. >>> As an example the build-system-ubuntu artefacts before this patch >>> where around 1.3 GB, after dropping the object files it comes to 970 >>> MB. >>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> >>> --- >>> .gitlab-ci.d/buildtest-template.yml | 16 ++++++ >>> .gitlab-ci.d/buildtest.yml | 81 +++++++++++------------------ >>> 2 files changed, 46 insertions(+), 51 deletions(-) >> >> This is still kludging the fact that 'make check-qtest' rebuild >> the world even if QEMU binaries are present. Still an improvement, so >> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> > > Is something being missed by: > > # Avoid recompiling by hiding ninja with NINJA=":" > - make NINJA=":" $MAKE_CHECK_ARGS > > ? Sorry, I wasn't looking at the correct pipeline.
© 2016 - 2026 Red Hat, Inc.