From: Michael Albinus Date: Sun, 21 Nov 2021 13:39:29 +0000 (+0100) Subject: Reorganize emba control files X-Git-Tag: emacs-29.0.90~2852^2~134 X-Git-Url: http://git.eshelyaron.com/gitweb/?a=commitdiff_plain;h=75d294cd4abb4d4062a1bfcd914229d0f9e142dd;p=emacs.git Reorganize emba control files Using dynamic job generation in GitLab does not work sufficiently. So we generate the jobs in the Emacs sources. * configure.ac (SUBDIR_MAKEFILES): Add test/infra/Makefile. * test/Makefile.in (subdirs, generate-test-jobs): New targets. * test/infra/Makefile.in: * test/infra/test-jobs.yml: New files. * test/infra/default-gitlab-ci.yml: * test/infra/test-jobs-generator.sh: Remove. * test/infra/gitlab-ci.yml: Insert contents of default-gitlab-ci.yml. (stages): Remove generator and trigger. Add normal. (test-jobs-generator, test-jobs-pipeline): Remove jobs. (top): Include test-jobs.yml. --- diff --git a/configure.ac b/configure.ac index 90a487f7ac7..9cf192d4ba9 100644 --- a/configure.ac +++ b/configure.ac @@ -6309,6 +6309,13 @@ if test -f "$srcdir/$opt_makefile.in"; then dnl ", [], [opt_makefile='$opt_makefile']" and it should work. AC_CONFIG_FILES([test/Makefile]) fi +opt_makefile=test/infra/Makefile +if test -f "$srcdir/$opt_makefile.in"; then + SUBDIR_MAKEFILES="$SUBDIR_MAKEFILES $opt_makefile" + dnl Again, it's best not to use a variable. Though you can add + dnl ", [], [opt_makefile='$opt_makefile']" and it should work. + AC_CONFIG_FILES([test/infra/Makefile]) +fi dnl The admin/ directory used to be excluded from tarfiles. diff --git a/test/Makefile.in b/test/Makefile.in index 39d7b1d4e48..51696d7faaf 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -371,7 +371,13 @@ check-declare: $(emacs) -l check-declare \ --eval '(check-declare-directory "$(srcdir)")' -.PHONY: subdir-targets +.PHONY: subdirs subdir-targets generate-test-jobs + +subdirs: + @echo $(SUBDIRS) subdir-targets: @echo $(SUBDIR_TARGETS) + +generate-test-jobs: + @$(MAKE) -C infra generate-test-jobs SUBDIRS="$(SUBDIRS)" diff --git a/test/infra/Makefile.in b/test/infra/Makefile.in new file mode 100644 index 00000000000..e91aea404d4 --- /dev/null +++ b/test/infra/Makefile.in @@ -0,0 +1,86 @@ +### test/infra/Makefile. Generated from Makefile.in by configure. + +# Copyright (C) 2021 Free Software Foundation, Inc. + +# This file is part of GNU Emacs. + +# GNU Emacs is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# GNU Emacs is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with GNU Emacs. If not, see . + +### Commentary: + +## Generate the test-jobs.yml file for emba. + +### Code: + +SHELL = @SHELL@ + +top_builddir = @top_builddir@ + +-include ${top_builddir}/src/verbose.mk + +## Get the tests for only a specific directory. +SUBDIRS ?= $(shell make -s -C .. subdirs) +SUBDIR_TARGETS = +FILE = test-jobs.yml + +define subdir_template + $(eval target = check-$(subst /,-,$(1))) + SUBDIR_TARGETS += $(target) + + $(eval + ifeq ($(findstring src, $(1)), src) + define changes + @echo " - $(1)/*.{h,c}" >>$(FILE) + @echo " - test/$(1)/*.el" >>$(FILE) + endef + else ifeq ($(findstring misc, $(1)), misc) + define changes + @echo " - admin/*.el" >>$(FILE) + @echo " - test/$(1)/*.el" >>$(FILE) + endef + else + define changes + @echo " - $(1)/*.el" >>$(FILE) + @echo " - test/$(1)/*.el" >>$(FILE) + endef + endif) + + .PHONY: $(target) + + $(target): + @echo "test-$(subst /,-,$(1))-inotify:" >>$(FILE) + @echo " stage: normal" >>$(FILE) + @echo " extends: [.job-template, .test-template]" >>$(FILE) + @echo " rules:" >>$(FILE) + @echo " - changes:" >>$(FILE) + $(changes) + @echo " variables:" >>$(FILE) + @echo " target: emacs-inotify" >>$(FILE) + @echo " make_params: \"-C test $(target)\"" >>$(FILE) + @echo >>$(FILE) +endef + +$(foreach subdir, $(SUBDIRS), $(eval $(call subdir_template,$(subdir)))) + +all: generate-test-jobs + +.PHONY: generate-test-jobs $(FILE) + +generate-test-jobs: clean $(FILE) $(SUBDIR_TARGETS) + +$(FILE): + $(AM_V_GEN) + +clean: + @rm -f $(FILE) diff --git a/test/infra/default-gitlab-ci.yml b/test/infra/default-gitlab-ci.yml deleted file mode 100644 index f6fadee27f3..00000000000 --- a/test/infra/default-gitlab-ci.yml +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright (C) 2017-2021 Free Software Foundation, Inc. -# -# This file is part of GNU Emacs. -# -# GNU Emacs is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# GNU Emacs is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with GNU Emacs. If not, see . - -# GNU Emacs support for the GitLab protocol for CI. - -# The presence of this file does not imply any FSF/GNU endorsement of -# any particular service that uses that protocol. Also, it is intended for -# evaluation purposes, thus possibly temporary. - -# Maintainer: Ted Zlatanov -# URL: https://emba.gnu.org/emacs/emacs - -# Never run merge request pipelines, they usually duplicate push pipelines -# see https://docs.gitlab.com/ee/ci/yaml/README.html#common-if-clauses-for-rules - -# Rules: always run tags and branches named master*, emacs*, feature*, fix* -# Test that it triggers by pushing a tag: `git tag mytag; git push origin mytag` -# Test that it triggers by pushing to: feature/emba, feature1, master, master-2, fix/emba, emacs-299, fix-2 -# Test that it doesn't trigger by pushing to: scratch-2, scratch/emba, oldbranch, dev -workflow: - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: never - - if: '$CI_COMMIT_TAG' - when: always - - if: '$CI_COMMIT_BRANCH !~ /^(master|emacs|feature|fix)/' - when: never - - when: always - -variables: - GIT_STRATEGY: fetch - EMACS_EMBA_CI: 1 - # Three hours, see below. - EMACS_TEST_TIMEOUT: 10800 - EMACS_TEST_VERBOSE: 1 - # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled - # DOCKER_HOST: tcp://docker:2376 - # DOCKER_TLS_CERTDIR: "/certs" - # Put the configuration for each run in a separate directory to - # avoid conflicts. - DOCKER_CONFIG: "/.docker-config-${CI_COMMIT_SHA}" - DOCKER_BUILDKIT: 1 - # We don't use ${CI_COMMIT_SHA} to be able to do one bootstrap - # across multiple builds. - BUILD_TAG: ${CI_COMMIT_REF_SLUG} - # Disable if you don't need it, it can be a security risk. - CI_DEBUG_TRACE: "true" - -default: - image: docker:19.03.12 - timeout: 3 hours - before_script: - - docker info - - echo "docker registry is ${CI_REGISTRY}" - - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} - -.job-template: - variables: - test_name: ${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA} - rules: - - changes: - - "**Makefile.in" - - .gitlab-ci.yml - - aclocal.m4 - - autogen.sh - - configure.ac - - lib/*.{h,c} - - lisp/**.el - - src/*.{h,c} - - test/infra/* - - test/lib-src/*.el - - test/lisp/**.el - - test/misc/*.el - - test/src/*.el - - changes: - # gfilemonitor, kqueue - - src/gfilenotify.c - - src/kqueue.c - # MS Windows - - "**w32*" - # GNUstep - - lisp/term/ns-win.el - - src/ns*.{h,m} - - src/macfont.{h,m} - when: never - # These will be cached across builds. - cache: - key: ${CI_COMMIT_SHA} - paths: [] - policy: pull-push - # These will be saved for followup builds. - artifacts: - expire_in: 24 hrs - paths: [] - # Using the variables for each job. - script: - - docker pull ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} - # TODO: with make -j4 several of the tests were failing, for - # example shadowfile-tests, but passed without it. - - 'export PWD=$(pwd)' - - 'docker run -i -e EMACS_EMBA_CI=${EMACS_EMBA_CI} -e EMACS_TEST_TIMEOUT=${EMACS_TEST_TIMEOUT} -e EMACS_TEST_VERBOSE=${EMACS_TEST_VERBOSE} --volumes-from $(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=${CI_JOB_ID}"):ro --name ${test_name} ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} /bin/bash -c "git fetch ${PWD} HEAD && echo checking out these updated files && git diff --name-only FETCH_HEAD && ( git diff --name-only FETCH_HEAD | xargs git checkout -f FETCH_HEAD ) && make -j4 && make ${make_params}"' - after_script: - # - docker ps -a - # - printenv - # - test -n "$(docker ps -aq -f name=${test_name})" && ( docker export ${test_name} | tar -tvf - ) - - test -n "$(docker ps -aq -f name=${test_name})" && docker cp ${test_name}:checkout/test ${test_name} - - test -n "$(docker ps -aq -f name=${test_name})" && docker rm ${test_name} - # - ls -alR ${test_name} - -.build-template: - needs: [] - rules: - - if: '$CI_PIPELINE_SOURCE == "web"' - when: always - - changes: - - "**Makefile.in" - - .gitlab-ci.yml - - aclocal.m4 - - autogen.sh - - configure.ac - - lib/*.{h,c} - - lisp/emacs-lisp/*.el - - src/*.{h,c} - - test/infra/* - - changes: - # gfilemonitor, kqueue - - src/gfilenotify.c - - src/kqueue.c - # MS Windows - - "**w32*" - # GNUstep - - lisp/term/ns-win.el - - src/ns*.{h,m} - - src/macfont.{h,m} - when: never - script: - - docker build --pull --target ${target} -t ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} -f test/infra/Dockerfile.emba . - - docker push ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} - -.test-template: - # Do not run fast and normal test jobs when scheduled. - rules: - - if: '$CI_JOB_STAGE =~ "fast|normal" && $CI_PIPELINE_SOURCE == "schedule"' - when: never - - when: always - artifacts: - name: ${test_name} - public: true - expire_in: 1 week - paths: - - ${test_name}/**/*.log - - ${test_name}/**/core - - ${test_name}/core - when: always - -.gnustep-template: - rules: - - if: '$CI_PIPELINE_SOURCE == "web"' - - if: '$CI_PIPELINE_SOURCE == "schedule"' - changes: - - "**Makefile.in" - - .gitlab-ci.yml - - configure.ac - - src/ns*.{h,m} - - src/macfont.{h,m} - - lisp/term/ns-win.el - - nextstep/** - - test/infra/* - -.filenotify-gio-template: - rules: - - if: '$CI_PIPELINE_SOURCE == "web"' - - if: '$CI_PIPELINE_SOURCE == "schedule"' - changes: - - "**Makefile.in" - - .gitlab-ci.yml - - lisp/autorevert.el - - lisp/filenotify.el - - lisp/net/tramp-sh.el - - src/gfilenotify.c - - test/infra/* - - test/lisp/autorevert-tests.el - - test/lisp/filenotify-tests.el - -.native-comp-template: - rules: - - if: '$CI_PIPELINE_SOURCE == "web"' - - if: '$CI_PIPELINE_SOURCE == "schedule"' - changes: - - "**Makefile.in" - - .gitlab-ci.yml - - lisp/emacs-lisp/comp.el - - lisp/emacs-lisp/comp-cstr.el - - src/comp.{h,m} - - test/infra/* - - test/src/comp-resources/*.el - - test/src/comp-tests.el - timeout: 8 hours - -# Local Variables: -# add-log-current-defun-header-regexp: "^\\([-_.[:alnum:]]+\\)[ \t]*:" -# End: diff --git a/test/infra/gitlab-ci.yml b/test/infra/gitlab-ci.yml index ebfe9965139..a0e2c283cde 100644 --- a/test/infra/gitlab-ci.yml +++ b/test/infra/gitlab-ci.yml @@ -24,15 +24,197 @@ # Maintainer: Ted Zlatanov # URL: https://emba.gnu.org/emacs/emacs -# Include defaults. -include: '/test/infra/default-gitlab-ci.yml' +# Never run merge request pipelines, they usually duplicate push pipelines +# see https://docs.gitlab.com/ee/ci/yaml/README.html#common-if-clauses-for-rules + +# Rules: always run tags and branches named master*, emacs*, feature*, fix* +# Test that it triggers by pushing a tag: `git tag mytag; git push origin mytag` +# Test that it triggers by pushing to: feature/emba, feature1, master, master-2, fix/emba, emacs-299, fix-2 +# Test that it doesn't trigger by pushing to: scratch-2, scratch/emba, oldbranch, dev +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - if: '$CI_COMMIT_TAG' + when: always + - if: '$CI_COMMIT_BRANCH !~ /^(master|emacs|feature|fix)/' + when: never + - when: always + +variables: + GIT_STRATEGY: fetch + EMACS_EMBA_CI: 1 + # Three hours, see below. + EMACS_TEST_TIMEOUT: 10800 + EMACS_TEST_VERBOSE: 1 + # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled + # DOCKER_HOST: tcp://docker:2376 + # DOCKER_TLS_CERTDIR: "/certs" + # Put the configuration for each run in a separate directory to + # avoid conflicts. + DOCKER_CONFIG: "/.docker-config-${CI_COMMIT_SHA}" + DOCKER_BUILDKIT: 1 + # We don't use ${CI_COMMIT_SHA} to be able to do one bootstrap + # across multiple builds. + BUILD_TAG: ${CI_COMMIT_REF_SLUG} + # Disable if you don't need it, it can be a security risk. + # CI_DEBUG_TRACE: "true" + +default: + image: docker:19.03.12 + timeout: 3 hours + before_script: + - docker info + - echo "docker registry is ${CI_REGISTRY}" + - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + +.job-template: + variables: + test_name: ${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA} + rules: + - changes: + - "**Makefile.in" + - .gitlab-ci.yml + - aclocal.m4 + - autogen.sh + - configure.ac + - lib/*.{h,c} + - lisp/**.el + - src/*.{h,c} + - test/infra/* + - test/lib-src/*.el + - test/lisp/**.el + - test/misc/*.el + - test/src/*.el + - changes: + # gfilemonitor, kqueue + - src/gfilenotify.c + - src/kqueue.c + # MS Windows + - "**w32*" + # GNUstep + - lisp/term/ns-win.el + - src/ns*.{h,m} + - src/macfont.{h,m} + when: never + # These will be cached across builds. + cache: + key: ${CI_COMMIT_SHA} + paths: [] + policy: pull-push + # These will be saved for followup builds. + artifacts: + expire_in: 24 hrs + paths: [] + # Using the variables for each job. + script: + - docker pull ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} + # TODO: with make -j4 several of the tests were failing, for + # example shadowfile-tests, but passed without it. + - 'export PWD=$(pwd)' + - 'docker run -i -e EMACS_EMBA_CI=${EMACS_EMBA_CI} -e EMACS_TEST_TIMEOUT=${EMACS_TEST_TIMEOUT} -e EMACS_TEST_VERBOSE=${EMACS_TEST_VERBOSE} --volumes-from $(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=${CI_JOB_ID}"):ro --name ${test_name} ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} /bin/bash -c "git fetch ${PWD} HEAD && echo checking out these updated files && git diff --name-only FETCH_HEAD && ( git diff --name-only FETCH_HEAD | xargs git checkout -f FETCH_HEAD ) && make -j4 && make ${make_params}"' + after_script: + # - docker ps -a + # - printenv + # - test -n "$(docker ps -aq -f name=${test_name})" && ( docker export ${test_name} | tar -tvf - ) + - test -n "$(docker ps -aq -f name=${test_name})" && docker cp ${test_name}:checkout/test ${test_name} + - test -n "$(docker ps -aq -f name=${test_name})" && docker rm ${test_name} + # - ls -alR ${test_name} + +.build-template: + needs: [] + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - changes: + - "**Makefile.in" + - .gitlab-ci.yml + - aclocal.m4 + - autogen.sh + - configure.ac + - lib/*.{h,c} + - lisp/emacs-lisp/*.el + - src/*.{h,c} + - test/infra/* + - changes: + # gfilemonitor, kqueue + - src/gfilenotify.c + - src/kqueue.c + # MS Windows + - "**w32*" + # GNUstep + - lisp/term/ns-win.el + - src/ns*.{h,m} + - src/macfont.{h,m} + when: never + script: + - docker build --pull --target ${target} -t ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} -f test/infra/Dockerfile.emba . + - docker push ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} + +.test-template: + # Do not run fast and normal test jobs when scheduled. + rules: + - if: '$CI_JOB_STAGE =~ "fast|normal" && $CI_PIPELINE_SOURCE == "schedule"' + when: never + - when: always + artifacts: + name: ${test_name} + public: true + expire_in: 1 week + paths: + - ${test_name}/**/*.log + - ${test_name}/**/core + - ${test_name}/core + when: always + +.gnustep-template: + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + - if: '$CI_PIPELINE_SOURCE == "schedule"' + changes: + - "**Makefile.in" + - .gitlab-ci.yml + - configure.ac + - src/ns*.{h,m} + - src/macfont.{h,m} + - lisp/term/ns-win.el + - nextstep/** + - test/infra/* + +.filenotify-gio-template: + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + - if: '$CI_PIPELINE_SOURCE == "schedule"' + changes: + - "**Makefile.in" + - .gitlab-ci.yml + - lisp/autorevert.el + - lisp/filenotify.el + - lisp/net/tramp-sh.el + - src/gfilenotify.c + - test/infra/* + - test/lisp/autorevert-tests.el + - test/lisp/filenotify-tests.el + +.native-comp-template: + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + - if: '$CI_PIPELINE_SOURCE == "schedule"' + changes: + - "**Makefile.in" + - .gitlab-ci.yml + - lisp/emacs-lisp/comp.el + - lisp/emacs-lisp/comp-cstr.el + - src/comp.{h,m} + - test/infra/* + - test/src/comp-resources/*.el + - test/src/comp-tests.el + timeout: 8 hours stages: - build-images - - generator - - trigger # - fast -# - normal + - normal - platform-images - platforms - native-comp-images @@ -52,21 +234,7 @@ build-image-inotify: # target: emacs-inotify # make_params: "-C test check" -test-jobs-generator: - stage: generator - script: - - test/infra/test-jobs-generator.sh > test-jobs.yml - artifacts: - paths: - - test-jobs.yml - -test-jobs-pipeline: - stage: trigger - trigger: - include: - - artifact: test-jobs.yml - job: test-jobs-generator - strategy: depend +include: '/test/infra/test-jobs.yml' # test-lisp-inotify: # stage: normal diff --git a/test/infra/test-jobs-generator.sh b/test/infra/test-jobs-generator.sh deleted file mode 100755 index c40570cbc30..00000000000 --- a/test/infra/test-jobs-generator.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2021 Free Software Foundation, Inc. -# -# This file is part of GNU Emacs. -# -# GNU Emacs is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# GNU Emacs is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with GNU Emacs. If not, see . - -# GNU Emacs support for the gitlab-ci.yml template generation. - -# The presence of this file does not imply any FSF/GNU endorsement of -# any particular service that uses that protocol. Also, it is intended for -# evaluation purposes, thus possibly temporary. - -# Maintainer: Michael Albinus -# URL: https://emba.gnu.org/emacs/emacs - -cd test -SUBDIRS=\ -$(find lib-src lisp misc src -type d \ - ! \( -path "*resources*" -o -path "*auto-save-list" \) -print | sort -) - -for subdir in $SUBDIRS; do - target=check-$(echo $subdir | tr '/' '-') - - case $target in - check*-src) - changes=" - - $subdir/*.{h,c} - - test/$subdir/*.el" - ;; - check-misc) - changes=" - - admin/*.el - - test/$subdir/*.el" - ;; - *) - changes=" - - $subdir/*.el - - test/$subdir/*.el" - ;; - esac - - cat <