Series comparison

Future patches will do more code extraction to the template using YAML
anchors so it's better that templates would live separately from the
job definitions.

Signed-off-by: Erik Skultety <eskultet@redhat.com>
---
 ci/integration-template.yml | 52 +++++++++++++++++++++++++++++++++++
 ci/integration.yml          | 54 ++-----------------------------------
 2 files changed, 54 insertions(+), 52 deletions(-)
 create mode 100644 ci/integration-template.yml

diff --git a/ci/integration-template.yml b/ci/integration-template.yml
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/ci/integration-template.yml
@@ -XXX,XX +XXX,XX @@
+.integration_tests:
+  stage: integration_tests
+  before_script:
+    - mkdir "$SCRATCH_DIR"
+    - sudo sh -c "echo DefaultLimitCORE=infinity >> /etc/systemd/system.conf" # Explicitly allow storing cores globally
+    - sudo systemctl daemon-reexec # need to reexec systemd after changing config
+    - sudo dnf install -y libvirt-rpms/* libvirt-perl-rpms/*
+    - sudo pip3 install --prefix=/usr avocado-framework
+    - source /etc/os-release  # in order to query the vendor-provided variables
+    - if test "$ID" = "centos" && test "$VERSION_ID" -lt 9 ||
+         test "$ID" = "fedora" && test "$VERSION_ID" -lt 35;
+      then
+        DAEMONS="libvirtd virtlogd virtlockd";
+      else
+        DAEMONS="virtproxyd virtqemud virtinterfaced virtsecretd virtstoraged virtnwfilterd virtnodedevd virtlogd virtlockd";
+      fi
+    - for daemon in $DAEMONS;
+      do
+        LOG_OUTPUTS="1:file:/var/log/libvirt/${daemon}.log";
+        LOG_FILTERS="3:remote 4:event 3:util.json 3:util.object 3:util.dbus 3:util.netlink 3:node_device 3:rpc 3:access 1:*";
+        sudo augtool set /files/etc/libvirt/${daemon}.conf/log_filters "$LOG_FILTERS" &>/dev/null;
+        sudo augtool set /files/etc/libvirt/${daemon}.conf/log_outputs "$LOG_OUTPUTS" &>/dev/null;
+        sudo systemctl --quiet stop ${daemon}.service;
+        sudo systemctl restart ${daemon}.socket;
+      done
+    - sudo virsh net-start default &>/dev/null || true;
+  script:
+    - mkdir logs
+    - cd "$SCRATCH_DIR"
+    - git clone --depth 1 https://gitlab.com/libvirt/libvirt-tck.git
+    - cd libvirt-tck
+    - sudo avocado --config avocado.config run --job-results-dir "$SCRATCH_DIR"/avocado
+  after_script:
+    - test "$CI_JOB_STATUS" = "success" && exit 0;
+    - test -e "$SCRATCH_DIR"/avocado && sudo mv "$SCRATCH_DIR"/avocado/latest/test-results logs/avocado;
+    - sudo coredumpctl info --no-pager > logs/coredumpctl.txt
+    - sudo mv /var/log/libvirt logs/libvirt
+    - sudo chown -R $(whoami):$(whoami) logs
+      # rename all Avocado stderr/stdout logs to *.log so that GitLab's web UI doesn't mangle the MIME type
+    - find logs/avocado/ -type f ! -name "*.log" -exec
+        sh -c 'DIR=$(dirname {}); NAME=$(basename {}); mv $DIR/$NAME{,.log}' \;
+  variables:
+    SCRATCH_DIR: "/tmp/scratch"
+  artifacts:
+    name: logs
+    paths:
+      - logs
+    when: on_failure
+  rules:
+    - if: '$LIBVIRT_CI_INTEGRATION'
+      when: on_success
+    - when: never
diff --git a/ci/integration.yml b/ci/integration.yml
index XXXXXXX..XXXXXXX 100644
--- a/ci/integration.yml
+++ b/ci/integration.yml
@@ -XXX,XX +XXX,XX @@
-.integration_tests:
-  stage: integration_tests
-  before_script:
-    - mkdir "$SCRATCH_DIR"
-    - sudo sh -c "echo DefaultLimitCORE=infinity >> /etc/systemd/system.conf" # Explicitly allow storing cores globally
-    - sudo systemctl daemon-reexec # need to reexec systemd after changing config
-    - sudo dnf install -y libvirt-rpms/* libvirt-perl-rpms/*
-    - sudo pip3 install --prefix=/usr avocado-framework
-    - source /etc/os-release  # in order to query the vendor-provided variables
-    - if test "$ID" = "centos" && test "$VERSION_ID" -lt 9 ||
-         test "$ID" = "fedora" && test "$VERSION_ID" -lt 35;
-      then
-        DAEMONS="libvirtd virtlogd virtlockd";
-      else
-        DAEMONS="virtproxyd virtqemud virtinterfaced virtsecretd virtstoraged virtnwfilterd virtnodedevd virtlogd virtlockd";
-      fi
-    - for daemon in $DAEMONS;
-      do
-        LOG_OUTPUTS="1:file:/var/log/libvirt/${daemon}.log";
-        LOG_FILTERS="3:remote 4:event 3:util.json 3:util.object 3:util.dbus 3:util.netlink 3:node_device 3:rpc 3:access 1:*";
-        sudo augtool set /files/etc/libvirt/${daemon}.conf/log_filters "$LOG_FILTERS" &>/dev/null;
-        sudo augtool set /files/etc/libvirt/${daemon}.conf/log_outputs "$LOG_OUTPUTS" &>/dev/null;
-        sudo systemctl --quiet stop ${daemon}.service;
-        sudo systemctl restart ${daemon}.socket;
-      done
-    - sudo virsh net-start default &>/dev/null || true;
-  script:
-    - mkdir logs
-    - cd "$SCRATCH_DIR"
-    - git clone --depth 1 https://gitlab.com/libvirt/libvirt-tck.git
-    - cd libvirt-tck
-    - sudo avocado --config avocado.config run --job-results-dir "$SCRATCH_DIR"/avocado
-  after_script:
-    - test "$CI_JOB_STATUS" = "success" && exit 0;
-    - test -e "$SCRATCH_DIR"/avocado && sudo mv "$SCRATCH_DIR"/avocado/latest/test-results logs/avocado;
-    - sudo coredumpctl info --no-pager > logs/coredumpctl.txt
-    - sudo mv /var/log/libvirt logs/libvirt
-    - sudo chown -R $(whoami):$(whoami) logs
-      # rename all Avocado stderr/stdout logs to *.log so that GitLab's web UI doesn't mangle the MIME type
-    - find logs/avocado/ -type f ! -name "*.log" -exec
-        sh -c 'DIR=$(dirname {}); NAME=$(basename {}); mv $DIR/$NAME{,.log}' \;
-  variables:
-    SCRATCH_DIR: "/tmp/scratch"
-  artifacts:
-    name: logs
-    paths:
-      - logs
-    when: on_failure
-  rules:
-    - if: '$LIBVIRT_CI_INTEGRATION'
-      when: on_success
-    - when: never
+include:
+  - 'ci/integration-template.yml'
 
 centos-stream-8-tests:
   extends: .integration_tests
-- 
2.34.1

There's quite a lot happening int the .integration_tests template
already even without adding upstream QEMU build to the mix. Let's break
the template into more pieces which can be put together using the YAML
anchors.

Signed-off-by: Erik Skultety <eskultet@redhat.com>
---
 ci/integration-template.yml | 76 +++++++++++++++++++++++--------------
 1 file changed, 47 insertions(+), 29 deletions(-)

diff --git a/ci/integration-template.yml b/ci/integration-template.yml
index XXXXXXX..XXXXXXX 100644
--- a/ci/integration-template.yml
+++ b/ci/integration-template.yml
@@ -XXX,XX +XXX,XX @@
+
+
+.install-deps: &install-deps
+  - sudo dnf install -y libvirt-rpms/* libvirt-perl-rpms/*
+  - sudo pip3 install --prefix=/usr avocado-framework
+
+
+.enable-core-dumps: &enable-core-dumps
+  - sudo sh -c "echo DefaultLimitCORE=infinity >> /etc/systemd/system.conf" # Explicitly allow storing cores globally
+  - sudo systemctl daemon-reexec # need to reexec systemd after changing config
+
+
+.enable-libvirt-debugging: &enable-libvirt-debugging
+  - source /etc/os-release  # in order to query the vendor-provided variables
+  - if test "$ID" = "centos" && test "$VERSION_ID" -lt 9 ||
+       test "$ID" = "fedora" && test "$VERSION_ID" -lt 35;
+    then
+      DAEMONS="libvirtd virtlogd virtlockd";
+    else
+      DAEMONS="virtproxyd virtqemud virtinterfaced virtsecretd virtstoraged virtnwfilterd virtnodedevd virtlogd virtlockd";
+    fi
+  - for daemon in $DAEMONS;
+    do
+      LOG_OUTPUTS="1:file:/var/log/libvirt/${daemon}.log";
+      LOG_FILTERS="3:remote 4:event 3:util.json 3:util.object 3:util.dbus 3:util.netlink 3:node_device 3:rpc 3:access 1:*";
+      sudo augtool set /files/etc/libvirt/${daemon}.conf/log_filters "$LOG_FILTERS" &>/dev/null;
+      sudo augtool set /files/etc/libvirt/${daemon}.conf/log_outputs "$LOG_OUTPUTS" &>/dev/null;
+      sudo systemctl --quiet stop ${daemon}.service;
+      sudo systemctl restart ${daemon}.socket;
+    done
+
+
+.collect-logs: &collect-logs
+  - mkdir logs
+  - test -e "$SCRATCH_DIR"/avocado && sudo mv "$SCRATCH_DIR"/avocado/latest/test-results logs/avocado;
+  - sudo coredumpctl info --no-pager > logs/coredumpctl.txt
+  - sudo mv /var/log/libvirt logs/libvirt
+  - sudo chown -R $(whoami):$(whoami) logs
+    # rename all Avocado stderr/stdout logs to *.log so that GitLab's web UI doesn't mangle the MIME type
+  - find logs/avocado/ -type f ! -name "*.log" -exec
+      sh -c 'DIR=$(dirname {}); NAME=$(basename {}); mv $DIR/$NAME{,.log}' \;
+
+
 .integration_tests:
   stage: integration_tests
   before_script:
     - mkdir "$SCRATCH_DIR"
-    - sudo sh -c "echo DefaultLimitCORE=infinity >> /etc/systemd/system.conf" # Explicitly allow storing cores globally
-    - sudo systemctl daemon-reexec # need to reexec systemd after changing config
-    - sudo dnf install -y libvirt-rpms/* libvirt-perl-rpms/*
-    - sudo pip3 install --prefix=/usr avocado-framework
-    - source /etc/os-release  # in order to query the vendor-provided variables
-    - if test "$ID" = "centos" && test "$VERSION_ID" -lt 9 ||
-         test "$ID" = "fedora" && test "$VERSION_ID" -lt 35;
-      then
-        DAEMONS="libvirtd virtlogd virtlockd";
-      else
-        DAEMONS="virtproxyd virtqemud virtinterfaced virtsecretd virtstoraged virtnwfilterd virtnodedevd virtlogd virtlockd";
-      fi
-    - for daemon in $DAEMONS;
-      do
-        LOG_OUTPUTS="1:file:/var/log/libvirt/${daemon}.log";
-        LOG_FILTERS="3:remote 4:event 3:util.json 3:util.object 3:util.dbus 3:util.netlink 3:node_device 3:rpc 3:access 1:*";
-        sudo augtool set /files/etc/libvirt/${daemon}.conf/log_filters "$LOG_FILTERS" &>/dev/null;
-        sudo augtool set /files/etc/libvirt/${daemon}.conf/log_outputs "$LOG_OUTPUTS" &>/dev/null;
-        sudo systemctl --quiet stop ${daemon}.service;
-        sudo systemctl restart ${daemon}.socket;
-      done
+    - *install-deps
+    - *enable-core-dumps
+    - *enable-libvirt-debugging
     - sudo virsh net-start default &>/dev/null || true;
   script:
-    - mkdir logs
     - cd "$SCRATCH_DIR"
     - git clone --depth 1 https://gitlab.com/libvirt/libvirt-tck.git
     - cd libvirt-tck
     - sudo avocado --config avocado.config run --job-results-dir "$SCRATCH_DIR"/avocado
   after_script:
     - test "$CI_JOB_STATUS" = "success" && exit 0;
-    - test -e "$SCRATCH_DIR"/avocado && sudo mv "$SCRATCH_DIR"/avocado/latest/test-results logs/avocado;
-    - sudo coredumpctl info --no-pager > logs/coredumpctl.txt
-    - sudo mv /var/log/libvirt logs/libvirt
-    - sudo chown -R $(whoami):$(whoami) logs
-      # rename all Avocado stderr/stdout logs to *.log so that GitLab's web UI doesn't mangle the MIME type
-    - find logs/avocado/ -type f ! -name "*.log" -exec
-        sh -c 'DIR=$(dirname {}); NAME=$(basename {}); mv $DIR/$NAME{,.log}' \;
+    - *collect-logs
   variables:
     SCRATCH_DIR: "/tmp/scratch"
   artifacts:
-- 
2.34.1

Rather than writing down the steps to build QEMU from source ourselves
let's recycle what upstream QEMU uses in their CI. This has the obvious
advantage that if QEMU changes something in their upstream CI build
process we'll pull the change automatically.
Since YAML anchors cannot be used in a cross-file scenario this patch
makes use of the '!reference' keyword which allows us go even further
and thus pick useful parts from the upstream QEMU's CI build template.
In addition, we're also able to add more intermediary steps to these
hand-picked QEMU build job sections.

Signed-off-by: Erik Skultety <eskultet@redhat.com>
---
 ci/integration-template.yml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/ci/integration-template.yml b/ci/integration-template.yml
index XXXXXXX..XXXXXXX 100644
--- a/ci/integration-template.yml
+++ b/ci/integration-template.yml
@@ -XXX,XX +XXX,XX @@
+# We want to reuse QEMU's .native_build_job_template to build it from source
+include:
+  - project: 'qemu-project/qemu'
+    file: '.gitlab-ci.d/buildtest-template.yml'
+
+
+.qemu-build-template: &qemu-build-template
+  - git clone --depth 1 https://gitlab.com/qemu-project/qemu.git
+  - cd qemu
+  #
+  # QEMU's .native_build_job_template bits
+  - export TARGETS="x86_64-softmmu"
+  - export CONFIGURE_ARGS="--prefix=/usr"
+  - export MAKE_CHECK_ARGS="check-build"
+  - !reference [.native_build_job_template, before_script]
+  - !reference [.native_build_job_template, script]
+  - sudo make install
 
 
 .install-deps: &install-deps
-- 
2.34.1

YAML anchors don't work with Shell conditions structures, so we cannot
simply pull in the QEMU build template conditionally like we do for
the other anchors. Instead we have to 'subclass' the .integration_tests
template and inject the QEMU building bits.

Signed-off-by: Erik Skultety <eskultet@redhat.com>
---
 ci/integration-template.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ci/integration-template.yml b/ci/integration-template.yml
index XXXXXXX..XXXXXXX 100644
--- a/ci/integration-template.yml
+++ b/ci/integration-template.yml
@@ -XXX,XX +XXX,XX @@ include:
     - if: '$LIBVIRT_CI_INTEGRATION'
       when: on_success
     - when: never
+
+
+# YAML anchors don't work with Shell conditions so we can't use a variable
+# to conditionally build+install QEMU from source.
+# Instead, create a new test job template for this scenario.
+.integration_tests_upstream_qemu:
+  extends: .integration_tests
+  before_script:
+    - !reference [.integration_tests, before_script]
+    - cd "$SCRATCH_DIR"
+    - *qemu-build-template
-- 
2.34.1

Since QEMU doesn't maintain a spec file in upstream, we cannot build RPM
artifacts as part of the CI as we do for libvirt. Instead, we build QEMU from
source manually.

Since v1:
- hard-code the QEMU build steps rather than pulling the build template from
upstream QEMU's CI due to a possibility of QEMU moving the template file or
renaming the build template itself
    -> only patch 3/5 was modified, the rest remained the same

Successful pipeline:
https://gitlab.com/eskultety/libvirt/-/pipelines/537225094

Erik Skultety (5):
  ci: Separate the integration job template to a separate file
  ci: Break off the integration_tests template into more templates
  ci: Introduce a template for upstream QEMU build
  ci: Add a new integration job template for the upstream QEMU scenario
  ci: Add a Fedora integration test job utilizing upstream QEMU

ci/integration-template.yml | 100 ++++++++++++++++++++++++++++++++++++
 ci/integration.yml          |  70 +++++++------------------
 2 files changed, 118 insertions(+), 52 deletions(-)
 create mode 100644 ci/integration-template.yml

-- 
2.34.3

Future patches will do more code extraction from the existing template
using YAML anchors so it'd be better that the templates would live
separately from job definitions.

Signed-off-by: Erik Skultety <eskultet@redhat.com>
Reviewed-by: Michal Privoznik <mprivozn@redhat.com>
Reviewed-by: Daniel P. Berrangé <berrange@redhat.com>
---
 ci/integration-template.yml | 52 +++++++++++++++++++++++++++++++++++
 ci/integration.yml          | 54 ++-----------------------------------
 2 files changed, 54 insertions(+), 52 deletions(-)
 create mode 100644 ci/integration-template.yml

There's quite a lot happening in the .integration_tests template
already even without adding upstream QEMU build into the mix.
Let's break the template into more pieces which can then reference
in the .integration_tests template when putting all the pieces back
together using YAML anchors.

Signed-off-by: Erik Skultety <eskultet@redhat.com>
Reviewed-by: Michal Privoznik <mprivozn@redhat.com>
Reviewed-by: Daniel P. Berrangé <berrange@redhat.com>
---
 ci/integration-template.yml | 76 +++++++++++++++++++++++--------------
 1 file changed, 47 insertions(+), 29 deletions(-)

This was heavily inspired by QEMU's upstream CI buildtest-template.yml.
Rather than referencing QEMU's template directly (which GitLab can do),
this patch resorts to hard-coding the build steps ourselves, solely
because there's no guarantee QEMU will keep either the template file
name or the template name from which the build steps were mostly copied
from.

Signed-off-by: Erik Skultety <eskultet@redhat.com>
---
 ci/integration-template.yml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/ci/integration-template.yml b/ci/integration-template.yml
index XXXXXXX..XXXXXXX 100644
--- a/ci/integration-template.yml
+++ b/ci/integration-template.yml
@@ -XXX,XX +XXX,XX @@
+.qemu-build-template: &qemu-build-template
+  - git clone --depth 1 https://gitlab.com/qemu-project/qemu.git
+  - cd qemu
+  #
+  # inspired by upstream QEMU's buildtest-template.yml
+  - export JOBS="$(expr $(nproc) + 1)"
+  - mkdir build
+  - cd build
+  - ../configure --prefix=/usr
+                 --enable-werror
+                 --disable-tcg
+                 --disable-docs
+                 --target-list=x86_64-softmmu || (cat config.log meson-logs/meson-log.txt && exit 1)
+  - make -j"$JOBS"
+  - if test -n "$MAKE_CHECK_ARGS";
+    then
+      make -j"$JOBS" check-build;
+    fi
+  - sudo make install
 
 
 .install-deps: &install-deps
-- 
2.34.3

YAML anchors don't work with Shell condition structures, so we cannot
simply reference the QEMU build template YAML anchor conditionally and
hence have everything as part of a single job template.
Instead, we have to "subclass" the .integration_tests template and
inject the QEMU building bits explicitly.

diff --git a/ci/integration-template.yml b/ci/integration-template.yml
index XXXXXXX..XXXXXXX 100644
--- a/ci/integration-template.yml
+++ b/ci/integration-template.yml
@@ -XXX,XX +XXX,XX @@
     - if: '$LIBVIRT_CI_INTEGRATION'
       when: on_success
     - when: never
+
+
+# YAML anchors don't work with Shell conditions so we can't use a variable
+# to conditionally build+install QEMU from source.
+# Instead, create a new test job template for this scenario.
+.integration_tests_upstream_qemu:
+  extends: .integration_tests
+  before_script:
+    - !reference [.integration_tests, before_script]
+    - cd "$SCRATCH_DIR"
+    - *qemu-build-template
-- 
2.34.3