[libvirt] [PATCH v5 16/23] tests: rewrite qemu capability grouper in Python

Daniel P. Berrangé posted 23 patches 5 years ago
There is a newer version of this series
[libvirt] [PATCH v5 16/23] tests: rewrite qemu capability grouper in Python
Posted by Daniel P. Berrangé 5 years ago
As part of an goal to eliminate Perl from libvirt build tools,
rewrite the group-qemu-caps.pl tool in Python.

This was a straight conversion, manually going line-by-line to
change the syntax from Perl to Python. Thus the overall structure
of the file and approach is the same.

Signed-off-by: Daniel P. Berrangé <berrange@redhat.com>
---
 Makefile.am                |   1 +
 build-aux/syntax-check.mk  |   3 +-
 scripts/group-qemu-caps.py | 123 ++++++++++++++++++++++++++++++++++++
 tests/group-qemu-caps.pl   | 124 -------------------------------------
 4 files changed, 126 insertions(+), 125 deletions(-)
 create mode 100755 scripts/group-qemu-caps.py
 delete mode 100755 tests/group-qemu-caps.pl

diff --git a/Makefile.am b/Makefile.am
index 6f6cead526..769cd4ce64 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -57,6 +57,7 @@ EXTRA_DIST = \
   scripts/dtrace2systemtap.py \
   scripts/genpolkit.py \
   scripts/gensystemtap.py \
+  scripts/group-qemu-caps.py \
   scripts/header-ifdef.py \
   scripts/minimize-po.py \
   scripts/mock-noinline.py \
diff --git a/build-aux/syntax-check.mk b/build-aux/syntax-check.mk
index 7d54df182a..44639f499e 100644
--- a/build-aux/syntax-check.mk
+++ b/build-aux/syntax-check.mk
@@ -2176,7 +2176,8 @@ test-wrap-argv:
 	$(PYTHON) $(top_srcdir)/scripts/test-wrap-argv.py --check
 
 group-qemu-caps:
-	$(AM_V_GEN)$(PERL) $(top_srcdir)/tests/group-qemu-caps.pl --check $(top_srcdir)/
+	$(AM_V_GEN)$(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/group-qemu-caps.py \
+		--check --prefix $(top_srcdir)/
 
 # List all syntax-check exemptions:
 exclude_file_name_regexp--sc_avoid_strcase = ^tools/vsh\.h$$
diff --git a/scripts/group-qemu-caps.py b/scripts/group-qemu-caps.py
new file mode 100755
index 0000000000..3edaf5d09f
--- /dev/null
+++ b/scripts/group-qemu-caps.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see
+# <http://www.gnu.org/licenses/>.
+#
+#
+# Regroup array values into smaller groups separated by numbered comments.
+#
+# If --check is the first parameter, the script will return
+# a non-zero value if a file is not grouped correctly.
+# Otherwise the files are regrouped in place.
+
+from __future__ import print_function
+
+import argparse
+import re
+import subprocess
+import sys
+
+
+def regroup_caps(check, filename, start_regex, end_regex,
+                 trailing_newline, counter_prefix):
+    step = 5
+
+    original = []
+    with open(filename, "r") as fh:
+        for line in fh:
+            original.append(line)
+
+    fixed = []
+    game_on = False
+    counter = 0
+    for line in original:
+        line = line.rstrip("\n")
+        if game_on:
+            if re.search(r'''.*/\* [0-9]+ \*/.*''', line):
+                continue
+            if re.search(r'''^\s*$''', line):
+                continue
+            if counter % step == 0:
+                if counter != 0:
+                    fixed.append("\n")
+                fixed.append("%s/* %d */\n" % (counter_prefix, counter))
+
+            if not (line.find("/*") != -1 and line.find("*/") == -1):
+                # count two-line comments as one line
+                counter = counter + 1
+
+        if re.search(start_regex, line):
+            game_on = True
+        elif game_on and re.search(end_regex, line):
+            if (counter - 1) % step == 0:
+                fixed = fixed[:-1]  # /* $counter */
+                if counter != 1:
+                    fixed = fixed[:-1]  # \n
+
+            if trailing_newline:
+                fixed.append("\n")
+
+            game_on = False
+
+        fixed.append(line + "\n")
+
+    if check:
+        orig = "".join(original)
+        new = "".join(fixed)
+        if new != orig:
+            diff = subprocess.Popen(["diff", "-u", filename, "-"],
+                                    stdin=subprocess.PIPE)
+            diff.communicate(input=new.encode('utf-8'))
+
+            print("Incorrect line wrapping in $file",
+                  file=sys.stderr)
+            print("Use test-wrap-argv.py to wrap test data files",
+                  file=sys.stderr)
+            return False
+    else:
+        with open(filename, "w") as fh:
+            for line in fixed:
+                print(line, file=fh, end='')
+
+    return True
+
+
+parser = argparse.ArgumentParser(description='Test arg line wrapper')
+parser.add_argument('--check', action="store_true",
+                    help='check existing files only')
+parser.add_argument('--prefix', default='',
+                    help='source code tree prefix')
+args = parser.parse_args()
+
+errs = False
+
+if not regroup_caps(args.check,
+                    args.prefix + 'src/qemu/qemu_capabilities.c',
+                    r'virQEMUCaps grouping marker',
+                    r'\);',
+                    0,
+                    "              "):
+    errs = True
+
+if not regroup_caps(args.check,
+                    args.prefix + 'src/qemu/qemu_capabilities.h',
+                    r'virQEMUCapsFlags grouping marker',
+                    r'QEMU_CAPS_LAST \/\* this must',
+                    1,
+                    "    "):
+    errs = True
+
+if errs:
+    sys.exit(1)
+sys.exit(0)
diff --git a/tests/group-qemu-caps.pl b/tests/group-qemu-caps.pl
deleted file mode 100755
index 829e63a562..0000000000
--- a/tests/group-qemu-caps.pl
+++ /dev/null
@@ -1,124 +0,0 @@
-#!/usr/bin/env perl
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library.  If not, see
-# <http://www.gnu.org/licenses/>.
-#
-#
-# Regroup array values into smaller groups separated by numbered comments.
-#
-# If --check is the first parameter, the script will return
-# a non-zero value if a file is not grouped correctly.
-# Otherwise the files are regrouped in place.
-
-use strict;
-use warnings;
-
-my $check = 0;
-
-if (defined $ARGV[0] && $ARGV[0] eq "--check") {
-    $check = 1;
-    shift @ARGV;
-}
-
-my $prefix = '';
-if (defined $ARGV[0]) {
-    $prefix = $ARGV[0];
-    shift @ARGV;
-}
-
-my $ret = 0;
-if (&regroup_caps($prefix . 'src/qemu/qemu_capabilities.c',
-                  'virQEMUCaps grouping marker',
-                  '\);',
-                  0,
-                  "              ") < 0) {
-    $ret = 1;
-}
-if (&regroup_caps($prefix . 'src/qemu/qemu_capabilities.h',
-                  'virQEMUCapsFlags grouping marker',
-                  'QEMU_CAPS_LAST \/\* this must',
-                  1,
-                  "    ") < 0) {
-    $ret = 1;
-}
-
-exit $ret;
-
-sub regroup_caps {
-    my $filename = shift;
-    my $start_regex = shift;
-    my $end_regex = shift;
-    my $trailing_newline = shift;
-    my $counter_prefix = shift;
-    my $step = 5;
-
-    open FILE, '<', $filename or die "cannot open $filename: $!";
-    my @original = <FILE>;
-    close FILE;
-
-    my @fixed;
-    my $game_on = 0;
-    my $counter = 0;
-    foreach (@original) {
-        if ($game_on) {
-            next if ($_ =~ '/\* [0-9]+ \*/');
-            next if (/^\s+$/);
-            if ($counter % $step == 0) {
-                if ($counter != 0) {
-                    push @fixed, "\n";
-                }
-                push @fixed, "$counter_prefix/* $counter */\n";
-            }
-            if (!($_ =~ '/\*' && !($_ =~ '\*/'))) {
-                # count two-line comments as one line
-                $counter++;
-            }
-        }
-        if (/$start_regex/) {
-            $game_on = 1;
-        } elsif ($game_on && $_ =~ /$end_regex/) {
-            if (($counter -1) % $step == 0) {
-                pop @fixed; # /* $counter */
-                if ($counter != 1) {
-                    pop @fixed; # \n
-                }
-            }
-            if ($trailing_newline) {
-                push @fixed, "\n";
-            }
-            $game_on = 0;
-        }
-        push @fixed, $_;
-    }
-
-    if ($check) {
-        my $nl = join('', @fixed);
-        my $ol = join('', @original);
-        unless ($nl eq $ol) {
-            open DIFF, "| diff -u $filename -" or die "cannot run diff: $!";
-            print DIFF $nl;
-            close DIFF;
-
-            print STDERR "Incorrect array grouping in $filename\n";
-            print STDERR "Use group-qemu-caps.pl to group long array members\n";
-            return -1;
-        }
-    } else {
-        open FILE, '>', $filename or die "cannot open $filename: $!";
-        foreach my $line (@fixed) {
-            print FILE $line;
-        }
-        close FILE;
-    }
-}
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list
Re: [libvirt] [PATCH v5 16/23] tests: rewrite qemu capability grouper in Python
Posted by Cole Robinson 5 years ago
On 11/11/19 9:38 AM, Daniel P. Berrangé wrote:
> As part of an goal to eliminate Perl from libvirt build tools,
> rewrite the group-qemu-caps.pl tool in Python.
> 
> This was a straight conversion, manually going line-by-line to
> change the syntax from Perl to Python. Thus the overall structure
> of the file and approach is the same.
> 
> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com>
> ---
>  Makefile.am                |   1 +
>  build-aux/syntax-check.mk  |   3 +-
>  scripts/group-qemu-caps.py | 123 ++++++++++++++++++++++++++++++++++++
>  tests/group-qemu-caps.pl   | 124 -------------------------------------
>  4 files changed, 126 insertions(+), 125 deletions(-)
>  create mode 100755 scripts/group-qemu-caps.py
>  delete mode 100755 tests/group-qemu-caps.pl
> 
> diff --git a/Makefile.am b/Makefile.am
> index 6f6cead526..769cd4ce64 100644
> --- a/Makefile.am
> +++ b/Makefile.am
> @@ -57,6 +57,7 @@ EXTRA_DIST = \
>    scripts/dtrace2systemtap.py \
>    scripts/genpolkit.py \
>    scripts/gensystemtap.py \
> +  scripts/group-qemu-caps.py \
>    scripts/header-ifdef.py \
>    scripts/minimize-po.py \
>    scripts/mock-noinline.py \
> diff --git a/build-aux/syntax-check.mk b/build-aux/syntax-check.mk
> index 7d54df182a..44639f499e 100644
> --- a/build-aux/syntax-check.mk
> +++ b/build-aux/syntax-check.mk
> @@ -2176,7 +2176,8 @@ test-wrap-argv:
>  	$(PYTHON) $(top_srcdir)/scripts/test-wrap-argv.py --check
>  
>  group-qemu-caps:
> -	$(AM_V_GEN)$(PERL) $(top_srcdir)/tests/group-qemu-caps.pl --check $(top_srcdir)/
> +	$(AM_V_GEN)$(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/group-qemu-caps.py \
> +		--check --prefix $(top_srcdir)/
>  
>  # List all syntax-check exemptions:
>  exclude_file_name_regexp--sc_avoid_strcase = ^tools/vsh\.h$$
> diff --git a/scripts/group-qemu-caps.py b/scripts/group-qemu-caps.py
> new file mode 100755
> index 0000000000..3edaf5d09f
> --- /dev/null
> +++ b/scripts/group-qemu-caps.py
> @@ -0,0 +1,123 @@
> +#!/usr/bin/env python
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library.  If not, see
> +# <http://www.gnu.org/licenses/>.
> +#
> +#
> +# Regroup array values into smaller groups separated by numbered comments.
> +#
> +# If --check is the first parameter, the script will return
> +# a non-zero value if a file is not grouped correctly.
> +# Otherwise the files are regrouped in place.
> +
> +from __future__ import print_function
> +
> +import argparse
> +import re
> +import subprocess
> +import sys
> +
> +
> +def regroup_caps(check, filename, start_regex, end_regex,
> +                 trailing_newline, counter_prefix):
> +    step = 5
> +
> +    original = []
> +    with open(filename, "r") as fh:
> +        for line in fh:
> +            original.append(line)
> +
> +    fixed = []
> +    game_on = False
> +    counter = 0
> +    for line in original:
> +        line = line.rstrip("\n")
> +        if game_on:
> +            if re.search(r'''.*/\* [0-9]+ \*/.*''', line):
> +                continue
> +            if re.search(r'''^\s*$''', line):
> +                continue
> +            if counter % step == 0:
> +                if counter != 0:
> +                    fixed.append("\n")
> +                fixed.append("%s/* %d */\n" % (counter_prefix, counter))
> +
> +            if not (line.find("/*") != -1 and line.find("*/") == -1):
> +                # count two-line comments as one line
> +                counter = counter + 1
> +
> +        if re.search(start_regex, line):
> +            game_on = True
> +        elif game_on and re.search(end_regex, line):
> +            if (counter - 1) % step == 0:
> +                fixed = fixed[:-1]  # /* $counter */
> +                if counter != 1:
> +                    fixed = fixed[:-1]  # \n
> +
> +            if trailing_newline:
> +                fixed.append("\n")
> +
> +            game_on = False
> +
> +        fixed.append(line + "\n")
> +
> +    if check:
> +        orig = "".join(original)
> +        new = "".join(fixed)
> +        if new != orig:
> +            diff = subprocess.Popen(["diff", "-u", filename, "-"],
> +                                    stdin=subprocess.PIPE)
> +            diff.communicate(input=new.encode('utf-8'))
> +
> +            print("Incorrect line wrapping in $file",
> +                  file=sys.stderr)
> +            print("Use test-wrap-argv.py to wrap test data files",
> +                  file=sys.stderr)

test-wrap-arg reference here

> +            return False
> +    else:
> +        with open(filename, "w") as fh:
> +            for line in fixed:
> +                print(line, file=fh, end='')
> +
> +    return True
> +
> +
> +parser = argparse.ArgumentParser(description='Test arg line wrapper')

test-wrap-arg reference here, should be group-qemu-caps

Otherwise it seems to work as expected

Tested-by: Cole Robinson <crobinso@redhat.com>

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list