From 55f652c856837b903b28917c8f5c6834b1483197 Mon Sep 17 00:00:00 2001 From: Michael Albinus Date: Sat, 18 Dec 2021 18:43:18 +0100 Subject: [PATCH] Make generation of JUnit test reports more robust. * lisp/emacs-lisp/ert.el (ert-load-file-name): New defvar. (ert-write-junit-test-report): Use it. Make detection of selector more robust. Protect calls of `xml-escape-string' with 'noerror. * test/infra/Makefile.in (subdir_template): Use "make -k ...". * test/infra/gitlab-ci.yml (test-filenotify-gio): Adapt make_params in order to get a JUnit test report. (test-native-comp-speed0): Use "make -k ...". * test/infra/test-jobs.yml: Regenerate. * test/lisp/progmodes/perl-mode-tests.el (top): Set `ert-load-file-name'. --- lisp/emacs-lisp/ert.el | 41 +++++++++++------ test/infra/Makefile.in | 2 +- test/infra/gitlab-ci.yml | 5 +- test/infra/test-jobs.yml | 64 +++++++++++++------------- test/lisp/progmodes/perl-mode-tests.el | 2 + 5 files changed, 66 insertions(+), 48 deletions(-) diff --git a/lisp/emacs-lisp/ert.el b/lisp/emacs-lisp/ert.el index 019916e6172..cc464a0f819 100644 --- a/lisp/emacs-lisp/ert.el +++ b/lisp/emacs-lisp/ert.el @@ -1527,13 +1527,20 @@ the tests)." (backtrace)) (kill-emacs 2)))) +(defvar ert-load-file-name nil + "The name of the loaded ERT test file, a string. +Usually, it is not needed to be defined, but if different ERT +test packages depend on each other, it might be helpful.") + (defun ert-write-junit-test-report (stats) "Write a JUnit test report, generated from STATS." ;; https://www.ibm.com/docs/en/developer-for-zos/14.1.0?topic=formats-junit-xml-format ;; https://llg.cubic.org/docs/junit/ (when-let ((symbol (car (apropos-internal "" #'ert-test-boundp))) (test-file (symbol-file symbol 'ert--test)) - (test-report (file-name-with-extension test-file "xml"))) + (test-report + (file-name-with-extension + (or ert-load-file-name test-file) "xml"))) (with-temp-file test-report (insert "\n") (insert (format "\n" @@ -1557,15 +1564,19 @@ the tests)." (ert--stats-end-time stats) (ert--stats-start-time stats))) (ert--format-time-iso8601 (ert--stats-end-time stats)))) - (insert " \n" - (format " \n" - (ert--stats-selector stats)) - " \n") + ;; If the test has aborted, `ert--stats-selector' might return + ;; huge junk. Skip this. + (when (< (length (format "%s" (ert--stats-selector stats))) 1024) + (insert " \n" + (format " \n" + (xml-escape-string + (format "%s" (ert--stats-selector stats)) 'noerror)) + " \n")) (cl-loop for test across (ert--stats-tests stats) for result = (ert-test-most-recent-result test) do (insert (format " \n" (xml-escape-string (string-trim - (ert-reason-for-test-result result))) + (ert-reason-for-test-result result)) + 'noerror) (ert-string-for-test-result result (ert-test-result-expected-p test result))) (xml-escape-string (string-trim - (ert-reason-for-test-result result))) + (ert-reason-for-test-result result)) + 'noerror) "\n" " \n")) ((ert-test-aborted-with-non-local-exit-p result) @@ -1600,27 +1613,29 @@ the tests)." test result))) (format "Test %s aborted with non-local exit\n" (xml-escape-string - (symbol-name (ert-test-name test)))) + (symbol-name (ert-test-name test)) 'noerror)) " \n")) ((not (ert-test-result-type-p result (ert-test-expected-result-type test))) (insert (format " \n" (xml-escape-string (string-trim - (ert-reason-for-test-result result))) + (ert-reason-for-test-result result)) + 'noerror) (ert-string-for-test-result result (ert-test-result-expected-p test result))) (xml-escape-string (string-trim - (ert-reason-for-test-result result))) + (ert-reason-for-test-result result)) + 'noerror) "\n" " \n"))) (unless (zerop (length (ert-test-result-messages result))) (insert " \n" (xml-escape-string - (ert-test-result-messages result)) + (ert-test-result-messages result) 'noerror) " \n")) (insert " \n"))) (insert " \n") @@ -1653,7 +1668,7 @@ the tests)." (insert (format " \n" (file-name-nondirectory test-report))) (when logfile-contents - (insert (xml-escape-string logfile-contents))) + (insert (xml-escape-string logfile-contents 'noerror))) (insert " \n" " \n" " \n") diff --git a/test/infra/Makefile.in b/test/infra/Makefile.in index e4f99743e09..368be7392b2 100644 --- a/test/infra/Makefile.in +++ b/test/infra/Makefile.in @@ -84,7 +84,7 @@ define subdir_template @echo ' - test/$(1)/*resources/**' >>$(FILE) @echo ' variables:' >>$(FILE) @echo ' target: emacs-inotify' >>$(FILE) - @echo ' make_params: "-C test $(target)"' >>$(FILE) + @echo ' make_params: "-k -C test $(target)"' >>$(FILE) endef $(foreach subdir, $(SUBDIRS), $(eval $(call subdir_template,$(subdir)))) diff --git a/test/infra/gitlab-ci.yml b/test/infra/gitlab-ci.yml index 3903642c792..dd3f517e74a 100644 --- a/test/infra/gitlab-ci.yml +++ b/test/infra/gitlab-ci.yml @@ -226,7 +226,8 @@ test-filenotify-gio: optional: true variables: target: emacs-filenotify-gio - make_params: "-k -C test autorevert-tests.log filenotify-tests.log" + # This is needed in order to get a JUnit test report. + make_params: '-k -C test check-expensive LOGFILES="lisp/autorevert-tests.log lisp/filenotify-tests.log"' build-image-gnustep: stage: platform-images @@ -271,7 +272,7 @@ test-native-comp-speed0: optional: true variables: target: emacs-native-comp-speed0 - make_params: "-C test check SELECTOR='(not (tag :unstable))'" + make_params: "-k -C test check SELECTOR='(not (tag :unstable))'" # Local Variables: # add-log-current-defun-header-regexp: "^\\([-_.[:alnum:]]+\\)[ \t]*:" diff --git a/test/infra/test-jobs.yml b/test/infra/test-jobs.yml index 63b052bf8c7..51707c181b1 100644 --- a/test/infra/test-jobs.yml +++ b/test/infra/test-jobs.yml @@ -15,7 +15,7 @@ test-lib-src-inotify: - test/lib-src/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lib-src" + make_params: "-k -C test check-lib-src" test-lisp-inotify: stage: normal @@ -32,7 +32,7 @@ test-lisp-inotify: - test/lisp/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp" + make_params: "-k -C test check-lisp" test-lisp-calc-inotify: stage: normal @@ -49,7 +49,7 @@ test-lisp-calc-inotify: - test/lisp/calc/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-calc" + make_params: "-k -C test check-lisp-calc" test-lisp-calendar-inotify: stage: normal @@ -66,7 +66,7 @@ test-lisp-calendar-inotify: - test/lisp/calendar/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-calendar" + make_params: "-k -C test check-lisp-calendar" test-lisp-cedet-inotify: stage: normal @@ -83,7 +83,7 @@ test-lisp-cedet-inotify: - test/lisp/cedet/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-cedet" + make_params: "-k -C test check-lisp-cedet" test-lisp-cedet-semantic-inotify: stage: normal @@ -100,7 +100,7 @@ test-lisp-cedet-semantic-inotify: - test/lisp/cedet/semantic/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-cedet-semantic" + make_params: "-k -C test check-lisp-cedet-semantic" test-lisp-cedet-semantic-bovine-inotify: stage: normal @@ -117,7 +117,7 @@ test-lisp-cedet-semantic-bovine-inotify: - test/lisp/cedet/semantic/bovine/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-cedet-semantic-bovine" + make_params: "-k -C test check-lisp-cedet-semantic-bovine" test-lisp-cedet-srecode-inotify: stage: normal @@ -134,7 +134,7 @@ test-lisp-cedet-srecode-inotify: - test/lisp/cedet/srecode/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-cedet-srecode" + make_params: "-k -C test check-lisp-cedet-srecode" test-lisp-emacs-lisp-inotify: stage: normal @@ -151,7 +151,7 @@ test-lisp-emacs-lisp-inotify: - test/lisp/emacs-lisp/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-emacs-lisp" + make_params: "-k -C test check-lisp-emacs-lisp" test-lisp-emacs-lisp-eieio-tests-inotify: stage: normal @@ -168,7 +168,7 @@ test-lisp-emacs-lisp-eieio-tests-inotify: - test/lisp/emacs-lisp/eieio-tests/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-emacs-lisp-eieio-tests" + make_params: "-k -C test check-lisp-emacs-lisp-eieio-tests" test-lisp-emacs-lisp-faceup-tests-inotify: stage: normal @@ -185,7 +185,7 @@ test-lisp-emacs-lisp-faceup-tests-inotify: - test/lisp/emacs-lisp/faceup-tests/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-emacs-lisp-faceup-tests" + make_params: "-k -C test check-lisp-emacs-lisp-faceup-tests" test-lisp-emulation-inotify: stage: normal @@ -202,7 +202,7 @@ test-lisp-emulation-inotify: - test/lisp/emulation/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-emulation" + make_params: "-k -C test check-lisp-emulation" test-lisp-erc-inotify: stage: normal @@ -219,7 +219,7 @@ test-lisp-erc-inotify: - test/lisp/erc/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-erc" + make_params: "-k -C test check-lisp-erc" test-lisp-eshell-inotify: stage: normal @@ -236,7 +236,7 @@ test-lisp-eshell-inotify: - test/lisp/eshell/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-eshell" + make_params: "-k -C test check-lisp-eshell" test-lisp-gnus-inotify: stage: normal @@ -253,7 +253,7 @@ test-lisp-gnus-inotify: - test/lisp/gnus/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-gnus" + make_params: "-k -C test check-lisp-gnus" test-lisp-image-inotify: stage: normal @@ -270,7 +270,7 @@ test-lisp-image-inotify: - test/lisp/image/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-image" + make_params: "-k -C test check-lisp-image" test-lisp-international-inotify: stage: normal @@ -287,7 +287,7 @@ test-lisp-international-inotify: - test/lisp/international/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-international" + make_params: "-k -C test check-lisp-international" test-lisp-mail-inotify: stage: normal @@ -304,7 +304,7 @@ test-lisp-mail-inotify: - test/lisp/mail/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-mail" + make_params: "-k -C test check-lisp-mail" test-lisp-mh-e-inotify: stage: normal @@ -321,7 +321,7 @@ test-lisp-mh-e-inotify: - test/lisp/mh-e/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-mh-e" + make_params: "-k -C test check-lisp-mh-e" test-lisp-net-inotify: stage: normal @@ -338,7 +338,7 @@ test-lisp-net-inotify: - test/lisp/net/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-net" + make_params: "-k -C test check-lisp-net" test-lisp-nxml-inotify: stage: normal @@ -355,7 +355,7 @@ test-lisp-nxml-inotify: - test/lisp/nxml/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-nxml" + make_params: "-k -C test check-lisp-nxml" test-lisp-obsolete-inotify: stage: normal @@ -372,7 +372,7 @@ test-lisp-obsolete-inotify: - test/lisp/obsolete/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-obsolete" + make_params: "-k -C test check-lisp-obsolete" test-lisp-org-inotify: stage: normal @@ -389,7 +389,7 @@ test-lisp-org-inotify: - test/lisp/org/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-org" + make_params: "-k -C test check-lisp-org" test-lisp-play-inotify: stage: normal @@ -406,7 +406,7 @@ test-lisp-play-inotify: - test/lisp/play/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-play" + make_params: "-k -C test check-lisp-play" test-lisp-progmodes-inotify: stage: normal @@ -423,7 +423,7 @@ test-lisp-progmodes-inotify: - test/lisp/progmodes/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-progmodes" + make_params: "-k -C test check-lisp-progmodes" test-lisp-so-long-tests-inotify: stage: normal @@ -440,7 +440,7 @@ test-lisp-so-long-tests-inotify: - test/lisp/so-long-tests/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-so-long-tests" + make_params: "-k -C test check-lisp-so-long-tests" test-lisp-term-inotify: stage: normal @@ -457,7 +457,7 @@ test-lisp-term-inotify: - test/lisp/term/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-term" + make_params: "-k -C test check-lisp-term" test-lisp-textmodes-inotify: stage: normal @@ -474,7 +474,7 @@ test-lisp-textmodes-inotify: - test/lisp/textmodes/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-textmodes" + make_params: "-k -C test check-lisp-textmodes" test-lisp-url-inotify: stage: normal @@ -491,7 +491,7 @@ test-lisp-url-inotify: - test/lisp/url/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-url" + make_params: "-k -C test check-lisp-url" test-lisp-vc-inotify: stage: normal @@ -508,7 +508,7 @@ test-lisp-vc-inotify: - test/lisp/vc/*resources/** variables: target: emacs-inotify - make_params: "-C test check-lisp-vc" + make_params: "-k -C test check-lisp-vc" test-misc-inotify: stage: normal @@ -525,7 +525,7 @@ test-misc-inotify: - test/misc/*resources/** variables: target: emacs-inotify - make_params: "-C test check-misc" + make_params: "-k -C test check-misc" test-src-inotify: stage: normal @@ -542,4 +542,4 @@ test-src-inotify: - test/src/*resources/** variables: target: emacs-inotify - make_params: "-C test check-src" + make_params: "-k -C test check-src" diff --git a/test/lisp/progmodes/perl-mode-tests.el b/test/lisp/progmodes/perl-mode-tests.el index 3f4af5e1f61..b059f539159 100644 --- a/test/lisp/progmodes/perl-mode-tests.el +++ b/test/lisp/progmodes/perl-mode-tests.el @@ -37,4 +37,6 @@ (file-name-directory (or load-file-name buffer-file-name))))) +(setq ert-load-file-name load-file-name) + ;;; perl-mode-tests.el ends here -- 2.39.2