From 5d408f1a24fd6e0fe0ea4acd6c02887332de1494 Mon Sep 17 00:00:00 2001 From: Stephen Gildea Date: Sat, 9 Oct 2021 11:36:03 -0700 Subject: [PATCH] Expanded testing of MH-E with multiple MH variants * test/lisp/mh-e/mh-utils-tests.el: Environment variable TEST_MH_PATH controls which installed MH variant to test with. Moved the commentary about testing with different MH variants from above 'with-mh-test-env' definition to "Commentary" section at the top of the file. * test/lisp/mh-e/test-all-mh-variants.sh: New script to test all installed MH variants. --- test/lisp/mh-e/mh-utils-tests.el | 94 +++++++++++++++++----- test/lisp/mh-e/test-all-mh-variants.sh | 104 +++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 19 deletions(-) create mode 100755 test/lisp/mh-e/test-all-mh-variants.sh diff --git a/test/lisp/mh-e/mh-utils-tests.el b/test/lisp/mh-e/mh-utils-tests.el index bf684dbbea8..a10c29fcf71 100644 --- a/test/lisp/mh-e/mh-utils-tests.el +++ b/test/lisp/mh-e/mh-utils-tests.el @@ -17,6 +17,34 @@ ;; You should have received a copy of the GNU General Public License ;; along with GNU Emacs. If not, see . +;;; Commentary: + +;; This test suite runs tests that use and depend on MH programs +;; installed on the system. + +;; When running such tests, MH-E can use a particular MH variant +;; installed on the system, or it can use the mocks provided here. +;; (Setup is done by the `with-mh-test-env' macro.) + +;; By setting environment variable TEST_MH_PATH, you can select which of +;; the installed MH variants to use, or ignore them all and use mocks. +;; See also the script test-all-mh-variants.sh in this directory. + +;; 1. To run these tests against the default MH variant installed on +;; this system: +;; cd ../.. && make lisp/mh-e/mh-utils-tests + +;; 2. To run these tests against an MH variant installed in a +;; specific directory, set TEST_MH_PATH, as in this example: +;; cd ../.. && make lisp/mh-e/mh-utils-tests TEST_MH_PATH=/usr/local/nmh/bin + +;; 3. To search for and run these tests against all MH variants +;; installed on this system: +;; ./test-all-mh-variants.sh + +;; Setting the environment variable TEST_MH_DEBUG or the Lisp variable +;; mh-test-utils-debug-mocks logs access to the file system during the test. + ;;; Code: (require 'ert) @@ -56,34 +84,32 @@ ;; Folder names that are used by the following tests. (defvar mh-test-rel-folder "rela-folder") (defvar mh-test-abs-folder "/abso-folder") -(defvar mh-test-no-such-folder "/testdir/none" - "Name of a folder that the user does not have.") +(defvar mh-test-no-such-folder "/testdir/none" "A folder that does not exist.") + +(defvar mh-test-utils-variants nil + "The value of `mh-variants' used for these tests. +This variable allows setting `mh-variants' to a limited set for targeted +testing. Its value can be different from the normal value when +environment variable TEST_MH_PATH is set. By remembering the value, we +can log the choice only once, which makes the batch log easier to read.") (defvar mh-test-variant-logged-already nil "Whether `with-mh-test-env' has written the MH variant to the log.") -(setq mh-test-variant-logged-already nil) ;reset if buffer is re-evaluated -(defvar mh-test-utils-debug-mocks nil +(defvar mh-test-utils-debug-mocks (> (length (getenv "TEST_MH_DEBUG")) 0) "Whether to log detailed behavior of mock functions.") (defvar mh-test-call-process-real (symbol-function 'call-process)) (defvar mh-test-file-directory-p-real (symbol-function 'file-directory-p)) - -;;; This macro wraps tests that touch the file system and/or run programs. -;;; When running such tests, MH-E can use a particular MH variant -;;; installed on the system, or it can use the mocks provided below. - -;;; By setting PATH and mh-sys-path, you can select which of the -;;; installed MH variants to use or ignore them all and use mocks. +;;; The macro with-mh-test-env wraps tests that touch the file system +;;; and/or run programs. (defmacro with-mh-test-env (&rest body) "Evaluate BODY with a test mail environment. Functions that touch the file system or run MH programs are either -mocked out or pointed at a test tree. When called from Emacs's batch -testing infrastructure, this will use mocks and thus run on systems -that do not have any MH variant installed. MH-E developers can -install an MH variant and test it interactively." +mocked out or pointed at a test tree. Uses `mh-test-utils-setup' to +select which." (declare (indent defun)) `(cl-letf ((temp-home-dir nil) ;; make local bindings for things we will modify for test env @@ -93,26 +119,56 @@ install an MH variant and test it interactively." ((symbol-function 'file-directory-p)) ;; the test always gets its own sub-folders cache (mh-sub-folders-cache (make-hash-table :test #'equal)) + ;; Allow envvar TEST_MH_PATH to control mh-variants. + (mh-variants mh-test-utils-variants) ;; remember the original value + (original-mh-test-variant-logged mh-test-variant-logged-already) + (original-mh-path mh-path) + (original-mh-sys-path mh-sys-path) + (original-exec-path exec-path) + (original-mh-variant-in-use mh-variant-in-use) + (original-mh-progs mh-progs) + (original-mh-lib mh-lib) + (original-mh-lib-progs mh-lib-progs) (original-mh-envvar (getenv "MH"))) (unwind-protect (progn (setq temp-home-dir (mh-test-utils-setup)) ,@body) + (unless noninteractive + ;; If interactive, forget that we logged the variant and + ;; restore any changes TEST_MH_PATH made. + (setq mh-test-variant-logged-already original-mh-test-variant-logged + mh-path original-mh-path + mh-sys-path original-mh-sys-path + exec-path original-exec-path + mh-variant-in-use original-mh-variant-in-use + mh-progs original-mh-progs + mh-lib original-mh-lib + mh-lib-progs original-mh-lib-progs)) (if temp-home-dir (delete-directory temp-home-dir t)) (setenv "MH" original-mh-envvar)))) (defun mh-test-utils-setup () "Set dynamically bound variables needed by mock and/or variants. +Call `mh-variant-set' to look through the directories named by +envionment variable `TEST_MH_PATH' (default: `mh-path' and `mh-sys-path') +to find the MH variant to use, if any. Return the name of the root of the created directory tree, if any." + (when (getenv "TEST_MH_PATH") + ;; force mh-variants to use only TEST_MH_PATH + (setq mh-path (split-string (getenv "TEST_MH_PATH") path-separator t) + mh-sys-path nil + exec-path '("/bin" "/usr/bin"))) (unless mh-test-variant-logged-already (mh-variant-set mh-variant) + (setq mh-test-utils-variants mh-variants) (setq mh-test-variant-logged-already t)) - ;; As `call-process'' and `file-directory-p' will be redefined, the - ;; native compiler will invoke `call-process' to compile the - ;; respective trampolines. To avoid interference with the - ;; `call-process' mocking, we build these ahead of time. (when (native-comp-available-p) + ;; As `call-process'' and `file-directory-p' will be redefined, the + ;; native compiler will invoke `call-process' to compile the + ;; respective trampolines. To avoid interference with the + ;; `call-process' mocking, we build these ahead of time. (mapc #'comp-subr-trampoline-install '(call-process file-directory-p))) (if mh-variant-in-use (mh-test-utils-setup-with-variant) diff --git a/test/lisp/mh-e/test-all-mh-variants.sh b/test/lisp/mh-e/test-all-mh-variants.sh new file mode 100755 index 00000000000..e917d8155bc --- /dev/null +++ b/test/lisp/mh-e/test-all-mh-variants.sh @@ -0,0 +1,104 @@ +#! /bin/bash +# Run the mh-utils-tests against all MH variants found on this system. + +# Copyright (C) 2021 Free Software Foundation, Inc. + +# This file is part of GNU Emacs. + +# GNU Emacs is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# GNU Emacs is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with GNU Emacs. If not, see . + +# Commentary: + +# By default runs all tests; test names or Emacs-style regexps may be +# given on the command line to run just those tests. +# +# Option -d turns on Emacs variable mh-test-utils-debug-mocks, which +# causes the tests to output all interactions with the file system. + +# If you want to run the tests for only one MH variant, you don't need +# to use this script, because "make" can do it. See the commentary at +# the top of ./mh-utils-tests.el for the recipe. + +debug= +if [[ "$1" = -* ]]; then + if [[ "$1" != -d ]]; then + echo "Usage: $(basename "$0") [-d] [test ...]" >&2 + exit 2 + fi + debug=t + shift +fi + +shopt -s extglob +ert_test_list=() +for tst; do + # Guess the type the test spec + case $tst in + *[\[\].*+\\]*) # Regexp: put in string quotes + ert_test_list+=("\"$tst\"") + ;; + *) # Lisp expression, keyword, or symbol: use as is + ert_test_list+=("$tst") + ;; + esac +done +if [[ ${#ert_test_list[@]} -eq 0 ]]; then + # t means true for all tests, runs everything + ert_test_list=(t) +fi + +# This script is 3 directories down in the Emacs source tree. +cd "$(dirname "$0")" +cd ../../.. +emacs=(src/emacs --batch -Q) + +# MH-E has a good list of directories where an MH variant might be installed, +# so we look in each of those. +read -r -a mh_sys_path \ + < <("${emacs[@]}" -l mh-e --eval "(princ mh-sys-path)" | sed 's/[()]//g') + +have_done_mocked_variant=false +declare -i tests_total=0 tests_passed=0 + +for path in "${mh_sys_path[@]}"; do + if [[ ! -x "$path/mhparam" ]]; then + if [[ "$have_done_mocked_variant" = false ]]; then + have_done_mocked_variant=true + else + continue + fi + fi + echo "Testing with PATH $path" + ((++tests_total)) + # The LD_LIBRARY_PATH setting is needed + # to run locally installed Mailutils. + TEST_MH_PATH=$path TEST_MH_DEBUG=$debug \ + LD_LIBRARY_PATH=/usr/local/lib HOME=/nonexistent \ + "${emacs[@]}" -l ert \ + --eval "(setq load-prefer-newer t)" \ + --eval "(load \"$PWD/test/lisp/mh-e/mh-utils-tests\" nil t)" \ + --eval "(ert-run-tests-batch-and-exit '(or ${ert_test_list[*]}))" \ + && ((++tests_passed)) +done + +if (( tests_total == 0 )); then + echo "NO tests run" + exit 1 +elif (( tests_total == tests_passed )); then + echo "All tested variants pass: $tests_passed/$tests_total" +else + echo "Tested variants passing: $tests_passed/$tests_total," \ + "FAILING: $((tests_total - tests_passed))/$tests_total" + exit 1 +fi -- 2.39.5