EMBA infrastructure improvements for Emacs build testing.

* .gitlab-ci.yml: Use job templates and rules. Split tests into
fast/normal/slow. Make Docker images for each tested platform
(inotify, filenotify-gio, gnustep). Increase timeout.
* test/Makefile.in (check-lisp, check-net): Add new testing
targets.
* test/README: Document them.
* test/file-organization.org: Mention test/infra.
* test/infra/Dockerfile.emba: Add special Docker recipes for EMBA
testing.

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bc18137a4391574f97f0a9f0c92015e83a515c39..eb884767c95f08212547fd820751424e1d23751a 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,89 +24,141 @@
 # Maintainer: Ted Zlatanov <tzz@lifelogs.com>
 # URL: https://emba.gnu.org/emacs/emacs
 
-image: debian:stretch
+# Never run merge request pipelines, they usually duplicate push pipelines
+# see https://docs.gitlab.com/ee/ci/yaml/README.html#common-if-clauses-for-rules
+workflow:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: never
+    - when: always
 
 variables:
   GIT_STRATEGY: fetch
   EMACS_EMBA_CI: 1
 
-before_script:
-  - apt update -qq
-  - DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y -qq -o=Dpkg::Use-Pty=0 libc-dev gcc g++ make autoconf automake libncurses-dev gnutls-dev git
+default:
+  image: docker:19.03.12
+  timeout: 3 hours
+  before_script:
+    - docker info
 
-stages:
-  - test
+.job-template:
+  # these will be cached across builds
+  cache:
+    key: ${CI_COMMIT_REF_SLUG}
+    paths: []
+    policy: pull-push
+  # these will be saved for followup builds
+  artifacts:
+    expire_in: 24 hrs
+    paths: []
+      # - "test/**/*.log"
+      # - "**/*.log"
 
-test-all:
-  # This tests also file monitor libraries inotify and inotifywatch.
-  stage: test
-  only:
-    changes:
-      - "Makefile.in"
-      - .gitlab-ci.yml
-      - aclocal.m4
-      - autogen.sh
-      - configure.ac
-      - lib/*.{h,c}
-      - lisp/*.el
-      - lisp/**/*.el
-      - src/*.{h,c}
-      - test/lisp/*.el
-      - test/lisp/**/*.el
-      - test/src/*.el
-  except:
-    changes:
-      # gfilemonitor, kqueue
-      - src/gfilenotify.c
-      - src/kqueue.c
-      # MS Windows
-      - lisp/w32*.el
-      - lisp/term/w32*.el
-      - src/w32*.{h,c}
-      # GNUstep
-      - lisp/term/ns-win.el
-      - src/ns*.{h,m}
-      - src/macfont.{h,m}
+.test-template:
+  rules:
+    - changes:
+        - "**/Makefile.in"
+        - .gitlab-ci.yml
+        - aclocal.m4
+        - autogen.sh
+        - configure.ac
+        - lib/*.{h,c}
+        - lisp/**/*.el
+        - src/*.{h,c}
+        - test/infra/*
+        - test/lisp/**/*.el
+        - test/src/*.el
+    - changes:
+        # gfilemonitor, kqueue
+        - src/gfilenotify.c
+        - src/kqueue.c
+        # MS Windows
+        - "**/w32*"
+        # GNUstep
+        - lisp/term/ns-win.el
+        - src/ns*.{h,m}
+        - src/macfont.{h,m}
+      when: never
+
+  # using the variables for each job
   script:
-    - DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y -qq -o=Dpkg::Use-Pty=0 inotify-tools
-    - ./autogen.sh autoconf
-    - ./configure --without-makeinfo
-    - make bootstrap
-    - make check-expensive
+    - docker build --target ${target} -t ${target}:${CI_COMMIT_REF_SLUG} -t ${target}:${CI_COMMIT_SHA} -f test/infra/Dockerfile.emba .
+    # TODO: with make -j4 several of the tests were failing, for example shadowfile-tests, but passed without it
+    - docker run -i --rm -e EMACS_EMBA_CI=${EMACS_EMBA_CI} ${target}:${CI_COMMIT_SHA} make ${make_params}
+
+stages:
+  - fast
+  - normal
+  - slow
+
+test-fast:
+  stage: fast
+  extends: [.job-template, .test-template]
+  variables:
+    target: emacs-inotify
+    make_params: "-C test check"
+
+test-lisp:
+  stage: normal
+  extends: [.job-template, .test-template]
+  variables:
+    target: emacs-inotify
+    make_params: "-C test check-lisp"
+
+test-net:
+  stage: normal
+  extends: [.job-template, .test-template]
+  variables:
+    target: emacs-inotify
+    make_params: "-C test check-net"
 
 test-filenotify-gio:
-  stage: test
   # This tests file monitor libraries gfilemonitor and gio.
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - lisp/autorevert.el
-      - lisp/filenotify.el
-      - lisp/net/tramp-sh.el
-      - src/gfilenotify.c
-      - test/lisp/autorevert-tests.el
-      - test/lisp/filenotify-tests.el
-  script:
-    - DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y -qq -o=Dpkg::Use-Pty=0 libglib2.0-dev libglib2.0-bin libglib2.0-0
-    - ./autogen.sh autoconf
-    - ./configure --without-makeinfo --with-file-notification=gfile
-    - make bootstrap
-    - make -k -C test autorevert-tests filenotify-tests
+  stage: normal
+  extends: [.job-template, .test-template]
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      changes:
+        - "**/Makefile.in"
+        - .gitlab-ci.yml
+        - lisp/autorevert.el
+        - lisp/filenotify.el
+        - lisp/net/tramp-sh.el
+        - src/gfilenotify.c
+        - test/infra/*
+        - test/lisp/autorevert-tests.el
+        - test/lisp/filenotify-tests.el
+  variables:
+    target: emacs-filenotify-gio
+    make_params: "-k -C test autorevert-tests filenotify-tests"
 
 test-gnustep:
-  stage: test
   # This tests the GNUstep build process
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - configure.ac
-      - src/ns*.{h,m}
-      - src/macfont.{h,m}
-      - lisp/term/ns-win.el
-      - nextstep/**/*
-  script:
-    - DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y -qq -o=Dpkg::Use-Pty=0 gnustep-devel
-    - ./autogen.sh autoconf
-    - ./configure --without-makeinfo --with-ns
-    - make bootstrap
-    - make install
+  stage: normal
+  extends: [.job-template, .test-template]
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      changes:
+        - "**/Makefile.in"
+        - .gitlab-ci.yml
+        - configure.ac
+        - src/ns*.{h,m}
+        - src/macfont.{h,m}
+        - lisp/term/ns-win.el
+        - nextstep/**/*
+        - test/infra/*
+  variables:
+    target: emacs-gnustep
+    make_params: install
+
+test-all:
+  # This tests also file monitor libraries inotify and inotifywatch.
+  stage: slow
+  extends: [.job-template, .test-template]
+  rules:
+    # note there's no "changes" section, so this always runs on a schedule
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+  variables:
+    target: emacs-inotify
+    make_params: check-expensive

diff --git a/test/Makefile.in b/test/Makefile.in
index fc40dad5e2eaf81956180d5380b7b1226d7311ae..2d595d9bf162776019c83392b6d80e2538f8bf58 100644 (file)
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -246,6 +246,12 @@ endef
 
 $(foreach test,${TESTS},$(eval $(call test_template,${test})))
 
+# Get the tests for only a specific directory
+NET_TESTS := $(patsubst %.el,%,$(wildcard lisp/net/*.el))
+LISP_TESTS := $(patsubst %.el,%,$(wildcard lisp/*.el))
+check-net: ${NET_TESTS}
+check-lisp: ${LISP_TESTS}
+
 ifeq (@HAVE_MODULES@, yes)
 # -fPIC is a no-op on Windows, but causes a compiler warning
 ifeq ($(SO),.dll)

diff --git a/test/README b/test/README
index ec566cb58dcc538efa2035c3da25c35cb0739f7a..38f4a109701ad8866e4e43569968056e133223c4 100644 (file)
--- a/test/README
+++ b/test/README
@@ -39,6 +39,12 @@ The Makefile in this directory supports the following targets:
 * make check-all
   Like "make check", but run all tests.
 
+* make check-lisp
+  Like "make check", but run only the tests in test/lisp/*.el
+
+* make check-net
+  Like "make check", but run only the tests in test/lisp/net/*.el
+
 * make <filename>  -or-  make <filename>.log
   Run all tests declared in <filename>.el.  This includes expensive
   tests.  In the former case the output is shown on the terminal, in

diff --git a/test/file-organization.org b/test/file-organization.org
index 64c0755b3bc01364fc6645b87512316a823ee2e9..efc354529c55269c64f1a2964e1809bf7d2b1a8d 100644 (file)
--- a/test/file-organization.org
+++ b/test/file-organization.org
@@ -57,3 +57,8 @@ directory called ~test/lisp/progmodes/flymake-resources~.
 No guidance is given for the organization of resource files inside the
 ~-resources~ directory; files can be organized at the author's
 discretion.
+
+** Testing Infrastructure Files
+
+Files used to support testing infrastructure such as EMBA should be
+placed in ~infra~.

diff --git a/test/infra/Dockerfile.emba b/test/infra/Dockerfile.emba
new file mode 100644 (file)
index 0000000..dd41982
--- /dev/null
+++ b/test/infra/Dockerfile.emba
@@ -0,0 +1,71 @@
+# Copyright (C) 2021 Free Software Foundation, Inc.
+#
+#  This file is part of GNU Emacs.
+#
+#  GNU Emacs is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  GNU Emacs is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+# GNU Emacs support for the GitLab-specific build of Docker images.
+
+# The presence of this file does not imply any FSF/GNU endorsement of
+# Docker or any other particular tool.  Also, it is intended for
+# evaluation purposes, thus possibly temporary.
+
+# Maintainer: Ted Zlatanov <tzz@lifelogs.com>
+# URL: https://emba.gnu.org/emacs/emacs
+
+FROM debian:stretch as emacs-base
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends -o=Dpkg::Use-Pty=0 \
+      libc-dev gcc g++ make autoconf automake libncurses-dev gnutls-dev git \
+    && rm -rf /var/lib/apt/lists/*
+
+FROM emacs-base as emacs-inotify
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends -o=Dpkg::Use-Pty=0 inotify-tools \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY . /checkout
+WORKDIR /checkout
+RUN ./autogen.sh autoconf
+RUN ./configure --without-makeinfo
+RUN make bootstrap
+RUN make -j4
+
+FROM emacs-base as emacs-filenotify-gio
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends -o=Dpkg::Use-Pty=0 libglib2.0-dev libglib2.0-bin libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY . /checkout
+WORKDIR /checkout
+RUN ./autogen.sh autoconf
+RUN ./configure --without-makeinfo --with-file-notification=gfile
+RUN make bootstrap
+RUN make -j4
+
+FROM emacs-base as emacs-gnustep
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends -o=Dpkg::Use-Pty=0 gnustep-devel \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY . /checkout
+WORKDIR /checkout
+RUN ./autogen.sh autoconf
+RUN ./configure --without-makeinfo --with-ns
+RUN make bootstrap
+RUN make -j4