Make generation of JUnit test reports more robust.

* lisp/emacs-lisp/ert.el (ert-load-file-name): New defvar.
(ert-write-junit-test-report): Use it.  Make detection of selector
more robust.  Protect calls of `xml-escape-string' with 'noerror.

* test/infra/Makefile.in (subdir_template): Use "make -k ...".

* test/infra/gitlab-ci.yml (test-filenotify-gio):
Adapt make_params in order to get a JUnit test report.
(test-native-comp-speed0): Use "make -k ...".

* test/infra/test-jobs.yml: Regenerate.

* test/lisp/progmodes/perl-mode-tests.el (top):
Set `ert-load-file-name'.

diff --git a/lisp/emacs-lisp/ert.el b/lisp/emacs-lisp/ert.el
index 019916e6172c9295f9e2820d5dcd0d17c5d53d55..cc464a0f819de052b9ca911927eda2075e74cb7b 100644 (file)
--- a/lisp/emacs-lisp/ert.el
+++ b/lisp/emacs-lisp/ert.el
@@ -1527,13 +1527,20 @@ the tests)."
           (backtrace))
       (kill-emacs 2))))
 
+(defvar ert-load-file-name nil
+  "The name of the loaded ERT test file, a string.
+Usually, it is not needed to be defined, but if different ERT
+test packages depend on each other, it might be helpful.")
+
 (defun ert-write-junit-test-report (stats)
   "Write a JUnit test report, generated from STATS."
   ;; https://www.ibm.com/docs/en/developer-for-zos/14.1.0?topic=formats-junit-xml-format
   ;; https://llg.cubic.org/docs/junit/
   (when-let ((symbol (car (apropos-internal "" #'ert-test-boundp)))
              (test-file (symbol-file symbol 'ert--test))
-             (test-report (file-name-with-extension test-file "xml")))
+             (test-report
+              (file-name-with-extension
+               (or ert-load-file-name test-file) "xml")))
     (with-temp-file test-report
       (insert "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
       (insert (format "<testsuites name=\"%s\" tests=\"%s\" errors=\"%s\" failures=\"%s\" skipped=\"%s\" time=\"%s\">\n"
@@ -1557,15 +1564,19 @@ the tests)."
                         (ert--stats-end-time stats)
                         (ert--stats-start-time stats)))
                       (ert--format-time-iso8601 (ert--stats-end-time stats))))
-      (insert "    <properties>\n"
-              (format "      <property name=\"selector\" value=\"%s\"/>\n"
-                      (ert--stats-selector stats))
-              "    </properties>\n")
+      ;; If the test has aborted, `ert--stats-selector' might return
+      ;; huge junk.  Skip this.
+      (when (< (length (format "%s" (ert--stats-selector stats))) 1024)
+        (insert "    <properties>\n"
+                (format "      <property name=\"selector\" value=\"%s\"/>\n"
+                        (xml-escape-string
+                         (format "%s" (ert--stats-selector stats)) 'noerror))
+                "    </properties>\n"))
       (cl-loop for test across (ert--stats-tests stats)
                for result = (ert-test-most-recent-result test) do
                (insert (format "    <testcase name=\"%s\" status=\"%s\" time=\"%s\""
                                (xml-escape-string
-                                (symbol-name (ert-test-name test)))
+                                (symbol-name (ert-test-name test)) 'noerror)
                                (ert-string-for-test-result
                                 result
                                 (ert-test-result-expected-p test result))
@@ -1581,14 +1592,16 @@ the tests)."
                    (insert (format "      <skipped message=\"%s\" type=\"%s\">\n"
                                    (xml-escape-string
                                     (string-trim
-                                     (ert-reason-for-test-result result)))
+                                     (ert-reason-for-test-result result))
+                                    'noerror)
                                    (ert-string-for-test-result
                                     result
                                     (ert-test-result-expected-p
                                      test result)))
                            (xml-escape-string
                             (string-trim
-                             (ert-reason-for-test-result result)))
+                             (ert-reason-for-test-result result))
+                            'noerror)
                            "\n"
                            "      </skipped>\n"))
                   ((ert-test-aborted-with-non-local-exit-p result)
@@ -1600,27 +1613,29 @@ the tests)."
                                      test result)))
                            (format "Test %s aborted with non-local exit\n"
                                    (xml-escape-string
-                                    (symbol-name (ert-test-name test))))
+                                    (symbol-name (ert-test-name test)) 'noerror))
                            "      </error>\n"))
                   ((not (ert-test-result-type-p
                          result (ert-test-expected-result-type test)))
                    (insert (format "      <failure message=\"%s\" type=\"%s\">\n"
                                    (xml-escape-string
                                     (string-trim
-                                     (ert-reason-for-test-result result)))
+                                     (ert-reason-for-test-result result))
+                                    'noerror)
                                    (ert-string-for-test-result
                                     result
                                     (ert-test-result-expected-p
                                      test result)))
                            (xml-escape-string
                             (string-trim
-                             (ert-reason-for-test-result result)))
+                             (ert-reason-for-test-result result))
+                            'noerror)
                            "\n"
                            "      </failure>\n")))
                  (unless (zerop (length (ert-test-result-messages result)))
                    (insert "      <system-out>\n"
                            (xml-escape-string
-                            (ert-test-result-messages result))
+                            (ert-test-result-messages result) 'noerror)
                            "      </system-out>\n"))
                  (insert "    </testcase>\n")))
       (insert "  </testsuite>\n")
@@ -1653,7 +1668,7 @@ the tests)."
                   (insert (format "      <error message=\"Test report missing %s\" type=\"error\">\n"
                                   (file-name-nondirectory test-report)))
                   (when logfile-contents
-                    (insert (xml-escape-string logfile-contents)))
+                    (insert (xml-escape-string logfile-contents 'noerror)))
                   (insert "      </error>\n"
                           "    </testcase>\n"
                           "  </testsuite>\n")

diff --git a/test/infra/Makefile.in b/test/infra/Makefile.in
index e4f99743e0915acc984093908f064ea09cf70df2..368be7392b2ffd1172c06a0c972b28ab17bbf4c0 100644 (file)
--- a/test/infra/Makefile.in
+++ b/test/infra/Makefile.in
@@ -84,7 +84,7 @@ define subdir_template
        @echo '        - test/$(1)/*resources/**' >>$(FILE)
        @echo '  variables:' >>$(FILE)
        @echo '    target: emacs-inotify' >>$(FILE)
-       @echo '    make_params: "-C test $(target)"' >>$(FILE)
+       @echo '    make_params: "-k -C test $(target)"' >>$(FILE)
 endef
 
 $(foreach subdir, $(SUBDIRS), $(eval $(call subdir_template,$(subdir))))

diff --git a/test/infra/gitlab-ci.yml b/test/infra/gitlab-ci.yml
index 3903642c792495725bba6fa8844c01af4300b264..dd3f517e74ab23d1d98b68f39c6830782245795c 100644 (file)
--- a/test/infra/gitlab-ci.yml
+++ b/test/infra/gitlab-ci.yml
@@ -226,7 +226,8 @@ test-filenotify-gio:
       optional: true
   variables:
     target: emacs-filenotify-gio
-    make_params: "-k -C test autorevert-tests.log filenotify-tests.log"
+    # This is needed in order to get a JUnit test report.
+    make_params: '-k -C test check-expensive LOGFILES="lisp/autorevert-tests.log lisp/filenotify-tests.log"'
 
 build-image-gnustep:
   stage: platform-images
@@ -271,7 +272,7 @@ test-native-comp-speed0:
       optional: true
   variables:
     target: emacs-native-comp-speed0
-    make_params: "-C test check SELECTOR='(not (tag :unstable))'"
+    make_params: "-k -C test check SELECTOR='(not (tag :unstable))'"
 
 # Local Variables:
 # add-log-current-defun-header-regexp: "^\\([-_.[:alnum:]]+\\)[ \t]*:"

diff --git a/test/infra/test-jobs.yml b/test/infra/test-jobs.yml
index 63b052bf8c7123cc78e47ea8afa6785c6a26f16b..51707c181b123551ac07f1cf9082a7a23283f0b6 100644 (file)
--- a/test/infra/test-jobs.yml
+++ b/test/infra/test-jobs.yml
@@ -15,7 +15,7 @@ test-lib-src-inotify:
         - test/lib-src/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lib-src"
+    make_params: "-k -C test check-lib-src"
 
 test-lisp-inotify:
   stage: normal
@@ -32,7 +32,7 @@ test-lisp-inotify:
         - test/lisp/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp"
+    make_params: "-k -C test check-lisp"
 
 test-lisp-calc-inotify:
   stage: normal
@@ -49,7 +49,7 @@ test-lisp-calc-inotify:
         - test/lisp/calc/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-calc"
+    make_params: "-k -C test check-lisp-calc"
 
 test-lisp-calendar-inotify:
   stage: normal
@@ -66,7 +66,7 @@ test-lisp-calendar-inotify:
         - test/lisp/calendar/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-calendar"
+    make_params: "-k -C test check-lisp-calendar"
 
 test-lisp-cedet-inotify:
   stage: normal
@@ -83,7 +83,7 @@ test-lisp-cedet-inotify:
         - test/lisp/cedet/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-cedet"
+    make_params: "-k -C test check-lisp-cedet"
 
 test-lisp-cedet-semantic-inotify:
   stage: normal
@@ -100,7 +100,7 @@ test-lisp-cedet-semantic-inotify:
         - test/lisp/cedet/semantic/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-cedet-semantic"
+    make_params: "-k -C test check-lisp-cedet-semantic"
 
 test-lisp-cedet-semantic-bovine-inotify:
   stage: normal
@@ -117,7 +117,7 @@ test-lisp-cedet-semantic-bovine-inotify:
         - test/lisp/cedet/semantic/bovine/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-cedet-semantic-bovine"
+    make_params: "-k -C test check-lisp-cedet-semantic-bovine"
 
 test-lisp-cedet-srecode-inotify:
   stage: normal
@@ -134,7 +134,7 @@ test-lisp-cedet-srecode-inotify:
         - test/lisp/cedet/srecode/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-cedet-srecode"
+    make_params: "-k -C test check-lisp-cedet-srecode"
 
 test-lisp-emacs-lisp-inotify:
   stage: normal
@@ -151,7 +151,7 @@ test-lisp-emacs-lisp-inotify:
         - test/lisp/emacs-lisp/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-emacs-lisp"
+    make_params: "-k -C test check-lisp-emacs-lisp"
 
 test-lisp-emacs-lisp-eieio-tests-inotify:
   stage: normal
@@ -168,7 +168,7 @@ test-lisp-emacs-lisp-eieio-tests-inotify:
         - test/lisp/emacs-lisp/eieio-tests/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-emacs-lisp-eieio-tests"
+    make_params: "-k -C test check-lisp-emacs-lisp-eieio-tests"
 
 test-lisp-emacs-lisp-faceup-tests-inotify:
   stage: normal
@@ -185,7 +185,7 @@ test-lisp-emacs-lisp-faceup-tests-inotify:
         - test/lisp/emacs-lisp/faceup-tests/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-emacs-lisp-faceup-tests"
+    make_params: "-k -C test check-lisp-emacs-lisp-faceup-tests"
 
 test-lisp-emulation-inotify:
   stage: normal
@@ -202,7 +202,7 @@ test-lisp-emulation-inotify:
         - test/lisp/emulation/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-emulation"
+    make_params: "-k -C test check-lisp-emulation"
 
 test-lisp-erc-inotify:
   stage: normal
@@ -219,7 +219,7 @@ test-lisp-erc-inotify:
         - test/lisp/erc/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-erc"
+    make_params: "-k -C test check-lisp-erc"
 
 test-lisp-eshell-inotify:
   stage: normal
@@ -236,7 +236,7 @@ test-lisp-eshell-inotify:
         - test/lisp/eshell/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-eshell"
+    make_params: "-k -C test check-lisp-eshell"
 
 test-lisp-gnus-inotify:
   stage: normal
@@ -253,7 +253,7 @@ test-lisp-gnus-inotify:
         - test/lisp/gnus/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-gnus"
+    make_params: "-k -C test check-lisp-gnus"
 
 test-lisp-image-inotify:
   stage: normal
@@ -270,7 +270,7 @@ test-lisp-image-inotify:
         - test/lisp/image/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-image"
+    make_params: "-k -C test check-lisp-image"
 
 test-lisp-international-inotify:
   stage: normal
@@ -287,7 +287,7 @@ test-lisp-international-inotify:
         - test/lisp/international/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-international"
+    make_params: "-k -C test check-lisp-international"
 
 test-lisp-mail-inotify:
   stage: normal
@@ -304,7 +304,7 @@ test-lisp-mail-inotify:
         - test/lisp/mail/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-mail"
+    make_params: "-k -C test check-lisp-mail"
 
 test-lisp-mh-e-inotify:
   stage: normal
@@ -321,7 +321,7 @@ test-lisp-mh-e-inotify:
         - test/lisp/mh-e/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-mh-e"
+    make_params: "-k -C test check-lisp-mh-e"
 
 test-lisp-net-inotify:
   stage: normal
@@ -338,7 +338,7 @@ test-lisp-net-inotify:
         - test/lisp/net/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-net"
+    make_params: "-k -C test check-lisp-net"
 
 test-lisp-nxml-inotify:
   stage: normal
@@ -355,7 +355,7 @@ test-lisp-nxml-inotify:
         - test/lisp/nxml/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-nxml"
+    make_params: "-k -C test check-lisp-nxml"
 
 test-lisp-obsolete-inotify:
   stage: normal
@@ -372,7 +372,7 @@ test-lisp-obsolete-inotify:
         - test/lisp/obsolete/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-obsolete"
+    make_params: "-k -C test check-lisp-obsolete"
 
 test-lisp-org-inotify:
   stage: normal
@@ -389,7 +389,7 @@ test-lisp-org-inotify:
         - test/lisp/org/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-org"
+    make_params: "-k -C test check-lisp-org"
 
 test-lisp-play-inotify:
   stage: normal
@@ -406,7 +406,7 @@ test-lisp-play-inotify:
         - test/lisp/play/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-play"
+    make_params: "-k -C test check-lisp-play"
 
 test-lisp-progmodes-inotify:
   stage: normal
@@ -423,7 +423,7 @@ test-lisp-progmodes-inotify:
         - test/lisp/progmodes/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-progmodes"
+    make_params: "-k -C test check-lisp-progmodes"
 
 test-lisp-so-long-tests-inotify:
   stage: normal
@@ -440,7 +440,7 @@ test-lisp-so-long-tests-inotify:
         - test/lisp/so-long-tests/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-so-long-tests"
+    make_params: "-k -C test check-lisp-so-long-tests"
 
 test-lisp-term-inotify:
   stage: normal
@@ -457,7 +457,7 @@ test-lisp-term-inotify:
         - test/lisp/term/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-term"
+    make_params: "-k -C test check-lisp-term"
 
 test-lisp-textmodes-inotify:
   stage: normal
@@ -474,7 +474,7 @@ test-lisp-textmodes-inotify:
         - test/lisp/textmodes/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-textmodes"
+    make_params: "-k -C test check-lisp-textmodes"
 
 test-lisp-url-inotify:
   stage: normal
@@ -491,7 +491,7 @@ test-lisp-url-inotify:
         - test/lisp/url/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-url"
+    make_params: "-k -C test check-lisp-url"
 
 test-lisp-vc-inotify:
   stage: normal
@@ -508,7 +508,7 @@ test-lisp-vc-inotify:
         - test/lisp/vc/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-lisp-vc"
+    make_params: "-k -C test check-lisp-vc"
 
 test-misc-inotify:
   stage: normal
@@ -525,7 +525,7 @@ test-misc-inotify:
         - test/misc/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-misc"
+    make_params: "-k -C test check-misc"
 
 test-src-inotify:
   stage: normal
@@ -542,4 +542,4 @@ test-src-inotify:
         - test/src/*resources/**
   variables:
     target: emacs-inotify
-    make_params: "-C test check-src"
+    make_params: "-k -C test check-src"

diff --git a/test/lisp/progmodes/perl-mode-tests.el b/test/lisp/progmodes/perl-mode-tests.el
index 3f4af5e1f613a312ff0e3d2c61ae305ed8d094b8..b059f539159e580e787d4cf92931303924689f06 100644 (file)
--- a/test/lisp/progmodes/perl-mode-tests.el
+++ b/test/lisp/progmodes/perl-mode-tests.el
@@ -37,4 +37,6 @@
                               (file-name-directory (or load-file-name
                                                        buffer-file-name)))))
 
+(setq ert-load-file-name load-file-name)
+
 ;;; perl-mode-tests.el ends here