(require 'pp)
(require 'map)
+(autoload 'xml-escape-string "xml.el")
+
;;; UI customization options.
(defgroup ert ()
"%s\\(\\s-\\|$\\)")
"The regexp the `find-function' mechanisms use for finding test definitions.")
-
(define-error 'ert-test-failed "Test failed")
(define-error 'ert-test-skipped "Test skipped")
,@body))
-
;;; Facilities for running a single test.
(defvar ert-debug-on-error nil
(if (getenv "EMACS_TEST_VERBOSE")
(ert-reason-for-test-result result)
""))))
- (message "%s" "")))))
+ (message "%s" ""))
+ (when (getenv "EMACS_TEST_JUNIT_REPORT")
+ (ert-write-junit-test-report stats)))))
(test-started)
(test-ended
(cl-destructuring-bind (stats test result) event-args
(backtrace))
(kill-emacs 2))))
+(defun ert-write-junit-test-report (stats)
+ "Write a JUnit test report, generated from STATS."
+ ;; https://www.ibm.com/docs/de/developer-for-zos/14.1.0?topic=formats-junit-xml-format
+ ;; https://llg.cubic.org/docs/junit/
+ (unless (zerop (length (ert--stats-tests stats)))
+ (when-let ((test-file
+ (symbol-file
+ (ert-test-name (aref (ert--stats-tests stats) 0)) 'ert--test)))
+ (with-temp-file (file-name-with-extension test-file "xml")
+ (insert "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
+ (insert (format "<testsuites name=\"%s\" tests=\"%s\" failures=\"%s\" skipped=\"%s\" time=\"%s\">\n"
+ (file-name-nondirectory test-file)
+ (ert-stats-total stats)
+ (ert-stats-completed-unexpected stats)
+ (ert-stats-skipped stats)
+ (float-time
+ (time-subtract
+ (ert--stats-end-time stats)
+ (ert--stats-start-time stats)))))
+ (insert (format " <testsuite id=\"0\" name=\"%s\" tests=\"%s\" failures=\"%s\" skipped=\"%s\" time=\"%s\" timestamp=\"%s\">\n"
+ (file-name-nondirectory test-file)
+ (ert-stats-total stats)
+ (ert-stats-completed-unexpected stats)
+ (ert-stats-skipped stats)
+ (float-time
+ (time-subtract
+ (ert--stats-end-time stats)
+ (ert--stats-start-time stats)))
+ (ert--format-time-iso8601 (ert--stats-end-time stats))))
+ (insert " <properties>\n"
+ (format " <property name=\"selector\" value=\"%s\"/>\n"
+ (ert--stats-selector stats))
+ " </properties>\n")
+ (cl-loop for test across (ert--stats-tests stats)
+ for result = (ert-test-most-recent-result test) do
+ (insert (format " <testcase name=\"%s\" status=\"%s\" time=\"%s\""
+ (xml-escape-string
+ (symbol-name (ert-test-name test)))
+ (ert-string-for-test-result
+ result
+ (ert-test-result-expected-p test result))
+ (ert-test-result-duration result)))
+ (if (and (ert-test-result-expected-p test result)
+ (not (ert-test-skipped-p result))
+ (zerop (length (ert-test-result-messages result))))
+ (insert "/>\n")
+ (insert ">\n")
+ (if (ert-test-skipped-p result)
+ (insert (format " <skipped message=\"%s\" type=\"%s\">\n"
+ (xml-escape-string
+ (string-trim
+ (ert-reason-for-test-result result)))
+ (ert-string-for-test-result
+ result
+ (ert-test-result-expected-p
+ test result)))
+ (xml-escape-string
+ (string-trim
+ (ert-reason-for-test-result result)))
+ "\n"
+ " </skipped>\n")
+ (unless
+ (ert-test-result-type-p
+ result (ert-test-expected-result-type test))
+ (insert (format " <failure message=\"%s\" type=\"%s\">\n"
+ (xml-escape-string
+ (string-trim
+ (ert-reason-for-test-result result)))
+ (ert-string-for-test-result
+ result
+ (ert-test-result-expected-p
+ test result)))
+ (xml-escape-string
+ (string-trim
+ (ert-reason-for-test-result result)))
+ "\n"
+ " </failure>\n")))
+ (unless (zerop (length (ert-test-result-messages result)))
+ (insert " <system-out>\n"
+ (xml-escape-string
+ (ert-test-result-messages result))
+ " </system-out>\n"))
+ (insert " </testcase>\n")))
+ (insert " </testsuite>\n")
+ (insert "</testsuites>\n")))))
+
+(defun ert-write-junit-test-summary-report (&rest logfiles)
+ "Write a JUnit summary test report, generated from LOGFILES."
+ (let ((report (file-name-with-extension
+ (getenv "EMACS_TEST_JUNIT_REPORT") "xml"))
+ (tests 0) (failures 0) (skipped 0) (time 0) (id 0))
+ (with-temp-file report
+ (dolist (logfile logfiles)
+ (let ((test-file (file-name-with-extension logfile "xml")))
+ (when (file-readable-p test-file)
+ (insert-file-contents-literally test-file)
+ (when (looking-at-p
+ (regexp-quote "<?xml version=\"1.0\" encoding=\"utf-8\"?>"))
+ (delete-region (point) (line-beginning-position 2)))
+ (when (looking-at
+ "<testsuites name=\".+\" tests=\"\\(.+\\)\" failures=\"\\(.+\\)\" skipped=\"\\(.+\\)\" time=\"\\(.+\\)\">")
+ (cl-incf tests (string-to-number (match-string 1)))
+ (cl-incf failures (string-to-number (match-string 2)))
+ (cl-incf skipped (string-to-number (match-string 3)))
+ (cl-incf time (string-to-number (match-string 4)))
+ (delete-region (point) (line-beginning-position 2)))
+ (when (looking-at " <testsuite id=\"\\(0\\)\"")
+ (replace-match (number-to-string id) nil nil nil 1)
+ (cl-incf id 1))
+ (goto-char (point-max))
+ (beginning-of-line 0)
+ (when (looking-at-p "</testsuites>")
+ (delete-region (point) (line-beginning-position 2)))
+ (narrow-to-region (point-max) (point-max)))))
+
+ (insert "</testsuites>\n")
+ (widen)
+ (goto-char (point-min))
+ (insert "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
+ (insert (format "<testsuites name=\"%s\" tests=\"%s\" failures=\"%s\" skipped=\"%s\" time=\"%s\">\n"
+ (file-name-nondirectory report)
+ tests failures skipped time)))))
(defun ert-summarize-tests-batch-and-exit (&optional high)
"Summarize the results of testing.
;; behavior.
(setq attempt-stack-overflow-recovery nil
attempt-orderly-shutdown-on-fatal-signal nil)
+ (when (getenv "EMACS_TEST_JUNIT_REPORT")
+ (apply #'ert-write-junit-test-summary-report command-line-args-left))
(let ((nlogs (length command-line-args-left))
(ntests 0) (nrun 0) (nexpected 0) (nunexpected 0) (nskipped 0)
nnotrun logfile notests badtests unexpected skipped tests)
;; should test it again.)
"\n")))
-
(defvar ert-test-run-redisplay-interval-secs .1
"How many seconds ERT should wait between redisplays while running tests.
(goto-char (1- (point-max)))
buffer)))))
-
(defvar ert--selector-history nil
"List of recent test selectors read from terminal.")
variables:
GIT_STRATEGY: fetch
EMACS_EMBA_CI: 1
+ EMACS_TEST_JUNIT_REPORT: junit-test-report.xml
EMACS_TEST_TIMEOUT: 3600
EMACS_TEST_VERBOSE: 1
# Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled
# TODO: with make -j4 several of the tests were failing, for
# example shadowfile-tests, but passed without it.
- 'export PWD=$(pwd)'
- - 'docker run -i -e EMACS_EMBA_CI=${EMACS_EMBA_CI} -e EMACS_TEST_TIMEOUT=${EMACS_TEST_TIMEOUT} -e EMACS_TEST_VERBOSE=${EMACS_TEST_VERBOSE} --volumes-from $(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=${CI_JOB_ID}"):ro --name ${test_name} ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} /bin/bash -c "git fetch ${PWD} HEAD && echo checking out these updated files && git diff --name-only FETCH_HEAD && ( git diff --name-only FETCH_HEAD | xargs git checkout -f FETCH_HEAD ) && make -j4 && make ${make_params}"'
+ - 'docker run -i -e EMACS_EMBA_CI=${EMACS_EMBA_CI} -e EMACS_TEST_JUNIT_REPORT=${EMACS_TEST_JUNIT_REPORT} -e EMACS_TEST_TIMEOUT=${EMACS_TEST_TIMEOUT} -e EMACS_TEST_VERBOSE=${EMACS_TEST_VERBOSE} --volumes-from $(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=${CI_JOB_ID}"):ro --name ${test_name} ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} /bin/bash -c "git fetch ${PWD} HEAD && echo checking out these updated files && git diff --name-only FETCH_HEAD && ( git diff --name-only FETCH_HEAD | xargs git checkout -f FETCH_HEAD ) && make -j4 && make ${make_params}"'
after_script:
# - docker ps -a
# - printenv
# Prepare test artifacts.
- test -n "$(docker ps -aq -f name=${test_name})" && docker cp ${test_name}:checkout/test ${test_name}
- test -n "$(docker ps -aq -f name=${test_name})" && docker rm ${test_name}
- - find ${test_name} ! -name "*.log" -type f -delete
+ - find ${test_name} ! \( -name "*.log" -o -name ${EMACS_TEST_JUNIT_REPORT} \) -type f -delete
+ # BusyBox find does not know -empty.
- find ${test_name} -type d -depth -exec rmdir {} + 2>/dev/null
.build-template:
when: always
- changes:
- "**.in"
- - "**.yml"
- GNUmakefile
- aclocal.m4
- autogen.sh
- lib/malloc/*.{h,c}
- lisp/emacs-lisp/*.el
- src/*.{h,c}
- - test/infra/Dockerfile.emba
+ - test/infra/*
- changes:
# gfilemonitor, kqueue
- src/gfilenotify.c
name: ${test_name}
public: true
expire_in: 1 week
+ when: always
paths:
- ${test_name}/
- when: always
+ reports:
+ junit: ${test_name}/${EMACS_TEST_JUNIT_REPORT}
.gnustep-template:
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule"'
changes:
- "**.in"
- - "**.yml"
- src/ns*.{h,m}
- src/macfont.{h,m}
- lisp/term/ns-win.el
- nextstep/**
- - test/infra/Dockerfile.emba
+ - test/infra/*
.filenotify-gio-template:
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule"'
changes:
- "**.in"
- - "**.yml"
- lisp/autorevert.el
- lisp/filenotify.el
- lisp/net/tramp-sh.el
- src/gfilenotify.c
- - test/infra/Dockerfile.emba
+ - test/infra/*
- test/lisp/autorevert-tests.el
- test/lisp/filenotify-tests.el
- if: '$CI_PIPELINE_SOURCE == "schedule"'
changes:
- "**.in"
- - "**.yml"
- lisp/emacs-lisp/comp.el
- lisp/emacs-lisp/comp-cstr.el
- src/comp.{h,m}
- - test/infra/Dockerfile.emba
+ - test/infra/*
- test/src/comp-resources/*.el
- test/src/comp-tests.el
timeout: 8 hours
projects / emacs.git / commitdiff