Tag some unstable tests, and skip by default (bug#24503)

* Makefile.in (check-all): New phony target.
* test/Makefile.in (SELECTOR_DEFAULT, SELECTOR_EXPENSIVE):
Also skip unstable tests.
(SELECTOR_ALL): New variable.
(check-all): New phony target.
* test/lisp/emacs-lisp/eieio-tests/eieio-test-methodinvoke.el
(eieio-test-method-order-list-6):
* test/lisp/emacs-lisp/eieio-tests/eieio-tests.el
(eieio-test-37-obsolete-name-in-constructor):
Mark as unstable rather than skipping on hydra.nixos.org.

diff --git a/Makefile.in b/Makefile.in
index 89277e2d66aa03ea87808cffe037e6f699ee8a36..238df40ded898b18c2360d41df3cbf02097534d3 100644 (file)
--- a/Makefile.in
+++ b/Makefile.in
@@ -58,9 +58,11 @@
 # make docs
 #      Make Emacs documentation files from their sources; requires makeinfo.
 #
-# make check  or  make check-expensive
-#      Run Emacs test suite.  check-expensive runs also tests which
-#      take more time to perform.
+# make check (or check-expensive or check-all)
+#      Run the Emacs test suite.
+#      check-expensive includes additional tests that can be slow.
+#      check-all runs all tests, including ones that can be slow, or
+#        fail unpredictably
 
 SHELL = @SHELL@
 
@@ -938,7 +940,8 @@ have-tests:
         exit 1; \
        fi
 
-check check-maybe check-expensive: have-tests all
+.PHONY: check check-maybe check-expensive check-all
+check check-maybe check-expensive check-all: have-tests all
        $(MAKE) -C test $@
 
 dist:
@@ -955,7 +958,7 @@ $(DOCS):
        $(MAKE) -C doc/$(subst -, ,$@)
 
 .PHONY: $(DOCS) docs pdf ps
-.PHONY: info dvi dist check check-maybe check-expensive html info-real info-dir check-info
+.PHONY: info dvi dist html info-real info-dir check-info
 
 ## TODO add etc/refcards.
 docs: $(DOCS)

diff --git a/test/Makefile.in b/test/Makefile.in
index 7bc99fb5cb821562581ede18536aaf572332c260..e03ffc5ab01884beacb39a77c042712704535bd1 100644 (file)
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -125,8 +125,9 @@ test_module_dir := $(srcdir)/data/emacs-module
 
 all: check
 
-SELECTOR_DEFAULT = (quote (not (tag :expensive-test)))
-SELECTOR_EXPENSIVE = nil
+SELECTOR_DEFAULT = (quote (not (or (tag :expensive-test) (tag :unstable))))
+SELECTOR_EXPENSIVE = (quote (not (tag :unstable)))
+SELECTOR_ALL = nil
 ifdef SELECTOR
 SELECTOR_ACTUAL=$(SELECTOR)
 else ifndef MAKECMDGOALS
@@ -241,6 +242,11 @@ check: mostlyclean check-no-automated-subdir
 check-expensive: mostlyclean check-no-automated-subdir
        @${MAKE} check-doit SELECTOR="${SELECTOR_EXPENSIVE}"
 
+## Run all tests, regardless of tag.
+.PHONY: check-all
+check-all: mostlyclean check-no-automated-subdir
+       @${MAKE} check-doit SELECTOR="${SELECTOR_ALL}"
+
 ## Re-run all tests which are outdated. A test is outdated if its
 ## logfile is out-of-date with either the test file, or the source
 ## files that the tests depend on.  See test_template.

diff --git a/test/lisp/emacs-lisp/eieio-tests/eieio-test-methodinvoke.el b/test/lisp/emacs-lisp/eieio-tests/eieio-test-methodinvoke.el
index 84661e8770b358ddcd0dbe75f0c6c09aa66c44e9..c6da9e15fa3fd74378432b6251a67e29e9efaeeb 100644 (file)
--- a/test/lisp/emacs-lisp/eieio-tests/eieio-test-methodinvoke.el
+++ b/test/lisp/emacs-lisp/eieio-tests/eieio-test-methodinvoke.el
@@ -192,7 +192,7 @@
 (ert-deftest eieio-test-method-order-list-6 ()
   ;; FIXME repeated intermittent failures on hydra (bug#24503)
   ;; ((:STATIC C) (:STATIC C-base1) (:STATIC C-base2)) != ((:STATIC C))")
-  (skip-unless (not (getenv "EMACS_HYDRA_CI")))
+  :tags '(:unstable)
   (let ((eieio-test-method-order-list nil)
        (ans '(
               (:STATIC C)

diff --git a/test/lisp/emacs-lisp/eieio-tests/eieio-tests.el b/test/lisp/emacs-lisp/eieio-tests/eieio-tests.el
index ee739740ae754540e8dea42b84aad01ed20603bc..5ba094c007228e0403b3e423162df6a6ce74d431 100644 (file)
--- a/test/lisp/emacs-lisp/eieio-tests/eieio-tests.el
+++ b/test/lisp/emacs-lisp/eieio-tests/eieio-tests.el
@@ -893,8 +893,8 @@ Subclasses to override slot attributes.")
   (list newname 2))
 
 (ert-deftest eieio-test-37-obsolete-name-in-constructor ()
-  ;; FIXME repeated intermittent failures on hydra (bug#24503)
-  (skip-unless (not (getenv "EMACS_HYDRA_CI")))
+  ;; FIXME repeated intermittent failures on hydra and elsewhere (bug#24503).
+  :tags '(:unstable)
   (should (equal (eieio--testing "toto") '("toto" 2))))
 
 (ert-deftest eieio-autoload ()