From: Corwin Brust Date: Wed, 25 Dec 2024 05:55:21 +0000 (-0600) Subject: * admin/nt/dist-build/build-dep-zips.py: (deps src) handle zst X-Git-Url: http://git.eshelyaron.com/gitweb/?a=commitdiff_plain;h=96645a0266a4f8188537209c2a4ef27f9d2e5157;p=emacs.git * admin/nt/dist-build/build-dep-zips.py: (deps src) handle zst This patch fixes various problems preventing this script from working. The job of this script is to collect Emacs' dependanicies and the (MSYS2/MinGW64) source archives for dependancies (including 2nd etc. order dependancies) into zip archives. GPL requires distributing such archives (along with Emacs' own sources) when distributing GPL binaries. Most principle of the changes is support for MSYS2 "w64" source archives in zst format; previously this script assumed all sources were available as tar.gz files. Additionaly several DLLs are added to list of required, including sqllite3 and tree-sitter. In general this is a maintaince fix to ensure others have access to the code used to create pretest (and, presumably release) version of Emacs 30. Additional changes are required to progmatically identify changes in Emacs' dependancies (e.g. after building a new Emacs binary for distribution). There is commented out code to this effect in the changes I've installed which did work, for a brief period in time; I suspect some changes in MSYS2 made it more difficult (or impossible?) to invoke the MSYS2 shell (under which this python script must run) from the MINGW64 (under which Emacs is built). I may resort to a powershell script as a way to orchistrate/invoke both MSYS2 and MING64 but I'm currently still messing with a Makefile based approach (which would run under MINGW64), in any event I believe we're all agreed this script must interigate Emacs to create the "starting list" of dependancies to include as a way to resolve this issue (missing/incorrect deps are frequent source of broken binary releases). (bug#65206) (cherry picked from commit 8b95549e9087fc54be3d849470652d38ec64cbff) --- diff --git a/admin/nt/dist-build/build-dep-zips.py b/admin/nt/dist-build/build-dep-zips.py index 0b1cc4d8695..c145cd3fc00 100755 --- a/admin/nt/dist-build/build-dep-zips.py +++ b/admin/nt/dist-build/build-dep-zips.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -## Copyright (C) 2017-2024 Free Software Foundation, Inc. +## Copyright (C) 2017-2023 Free Software Foundation, Inc. ## This file is part of GNU Emacs. @@ -20,41 +20,56 @@ import argparse import os import shutil import re -import functools -import operator +import subprocess from subprocess import check_output ## Constants -EMACS_MAJOR_VERSION="28" +EMACS_MAJOR_VERSION= os.getenv('EMACS_MAJOR_VERSION') or "30" -# This list derives from the features we want Emacs to compile with. +# Base URI for the package sources mapped in PKG_REQ +SRC_REPO="https://repo.msys2.org/mingw/sources" + +# Map items in `dynamic-library-alist' to source pakages PKG_REQ='''mingw-w64-x86_64-giflib mingw-w64-x86_64-gnutls mingw-w64-x86_64-harfbuzz +mingw-w64-x86_64-jansson mingw-w64-x86_64-lcms2 mingw-w64-x86_64-libjpeg-turbo mingw-w64-x86_64-libpng mingw-w64-x86_64-librsvg +mingw-w64-x86_64-libwebp mingw-w64-x86_64-libtiff mingw-w64-x86_64-libxml2 -mingw-w64-x86_64-xpm-nox'''.split() - -DLL_REQ='''libgif -libgnutls -libharfbuzz -liblcms2 -libturbojpeg -libpng -librsvg -libtiff -libxml -libXpm'''.split() - +mingw-w64-x86_64-gmp +mingw-w64-x86_64-xpm-nox +mingw-w64-x86_64-tree-sitter +mingw-w64-x86_64-sqlite3'''.split() + +# Emacs style path to dependancy DLLs on build system +DLL_SRC="c:/msys64/mingw64/bin" + +# libraries we never include +DLL_SKIP=["libgccjit-0.dll"] + +# Report first existing file for entries in dynamic-library-alist +# ELISP_PROG=""" +# (message "%s" (mapconcat 'identity (remove nil +# (mapcar (lambda(lib) +# (seq-find +# (lambda(file) +# (file-exists-p +# (file-name-concat "{}" +# file))) +# (cdr lib))) +# dynamic-library-alist) +# ) "\\n")) +# """.format(DLL_SRC) ## Options DRY_RUN=False - +# NEW_EMACS="bin/emacs.exe" def check_output_maybe(*args,**kwargs): if(DRY_RUN): @@ -62,31 +77,76 @@ def check_output_maybe(*args,**kwargs): else: return check_output(*args,**kwargs) +#################### ## DLL Capture + +# entry point def gather_deps(): os.mkdir("x86_64") os.chdir("x86_64") - for dep in full_dll_dependency(): - check_output_maybe(["cp /mingw64/bin/{}*.dll .".format(dep)], - shell=True) - - print("Zipping") - check_output_maybe("zip -9r ../emacs-{}-{}deps.zip *" - .format(EMACS_MAJOR_VERSION, DATE), - shell=True) + #full=full_dll_dependency(init_deps()) + #filtered=filter(lambda x: x not in DLL_SKIP, full) + #print("full:",full.len(), " filtered:",filtered.len()) + #exit + + for dep in full_dll_dependency(init_deps()): + if dep not in DLL_SKIP: + if args.l != True: + print("Adding dep", dep) + check_output_maybe(["cp /mingw64/bin/{} .".format(dep)], shell=True) + else: + if args.l != True: + print("Skipping dep", dep) + + zipfile="../emacs-{}-{}deps.zip".format(EMACS_MAJOR_VERSION, DATE) + tmpfile="{}.tmp".format(zipfile) + print("Zipping deps in", os.getcwd(), "as", tmpfile) + check_output_maybe("zip -9vr {} *.dll".format(tmpfile), shell=True) + if os.path.isfile(zipfile): + os.remove(zipfile) + os.rename(tmpfile, zipfile) + print("Deps updated in", os.getcwd(), "as", zipfile) os.chdir("../") -## Return all Emacs dependencies -def full_dll_dependency(): - deps = [dll_dependency(dep) for dep in DLL_REQ] - return set(sum(deps, []) + DLL_REQ) - -## Dependencies for a given DLL +# Return dependancies listed in Emacs +def init_deps(): + return '''libXpm-nox4.dll +libpng16-16.dll +libjpeg-8.dll +libgif-7.dll +librsvg-2-2.dll +libwebp-7.dll +libwebpdemux-2.dll +libsqlite3-0.dll +libgdk_pixbuf-2.0-0.dll +libglib-2.0-0.dll +libgio-2.0-0.dll +libgobject-2.0-0.dll +libgnutls-30.dll +libxml2-2.dll +zlib1.dll +liblcms2-2.dll +libgccjit-0.dll +libtree-sitter.dll'''.split() + # job_args=[NEW_EMACS, "--batch", "--eval", ELISP_PROG] + # #print("args: ", job_args) + # return subprocess.check_output(job_args, stderr=subprocess.STDOUT + # ).decode('utf-8').splitlines() + +# Return all second order dependencies +def full_dll_dependency(dlls): + deps = [dll_dependency(dep) for dep in dlls] + return set(sum(deps, []) + dlls) + +#xs = filter(lambda x: x.attribute == value, xs) + +# Dependencies for a given DLL def dll_dependency(dll): output = check_output(["/mingw64/bin/ntldd", "--recursive", - "/mingw64/bin/{}*.dll".format(dll)]).decode("utf-8") + "/mingw64/bin/{}".format(dll)] + ).decode("utf-8") ## munge output return ntldd_munge(output) @@ -101,9 +161,8 @@ def ntldd_munge(out): ## if it's the former, we want it, if its the later we don't splt = dep.split() - if len(splt) > 2 and "msys64" in splt[2]: - print("Adding dep", splt[0]) - rtn.append(splt[0].split(".")[0]) + if len(splt) > 2 and "mingw64" in splt[2]: + rtn.append(splt[0]) return rtn @@ -112,26 +171,92 @@ def ntldd_munge(out): ## Packages to fiddle with ## Source for gcc-libs is part of gcc SKIP_SRC_PKGS=["mingw-w64-gcc-libs"] -SKIP_DEP_PKGS=frozenset(["mingw-w64-x86_64-glib2"]) -MUNGE_SRC_PKGS={"mingw-w64-libwinpthread-git":"mingw-w64-winpthreads-git"} +SKIP_DEP_PKGS=["mingw-w64-glib2", "mingw-w64-ca-certificates-20211016-3"] +MUNGE_SRC_PKGS={ + "mingw-w64-libwinpthread-git":"mingw-w64-winpthreads-git", + "mingw-w64-gettext-runtime":"mingw-w64-gettext" +} MUNGE_DEP_PKGS={ "mingw-w64-x86_64-libwinpthread":"mingw-w64-x86_64-libwinpthread-git", "mingw-w64-x86_64-libtre": "mingw-w64-x86_64-libtre-git", } +SRC_EXT={ + "mingw-w64-freetype": ".src.tar.zst", + "mingw-w64-fribidi": ".src.tar.zst", + "mingw-w64-glib2": ".src.tar.zst", + "mingw-w64-harfbuzz": ".src.tar.zst", + "mingw-w64-libunistring": ".src.tar.zst", + "mingw-w64-winpthreads-git": ".src.tar.zst", + "mingw-w64-ca-certificates": ".src.tar.zst", + "mingw-w64-libxml2": ".src.tar.zst", + "mingw-w64-ncurses": ".src.tar.zst", + "mingw-w64-openssl": ".src.tar.zst", + "mingw-w64-pango": ".src.tar.zst", + "mingw-w64-python": ".src.tar.zst", + "mingw-w64-sqlite3": ".src.tar.zst", + "mingw-w64-xpm-nox": ".src.tar.zst", + "mingw-w64-xz": ".src.tar.zst", + "mingw-w64-bzip2": ".src.tar.zst", + "mingw-w64-cairo": ".src.tar.zst", + "mingw-w64-expat": ".src.tar.zst", + "mingw-w64-fontconfig": ".src.tar.zst", + "mingw-w64-gdk-pixbuf2": ".src.tar.zst", + "mingw-w64-giflib": ".src.tar.zst", + "mingw-w64-gmp": ".src.tar.zst", + "mingw-w64-gnutls": ".src.tar.zst", + "mingw-w64-graphite2": ".src.tar.zst", + "mingw-w64-jbigkit": ".src.tar.zst", + "mingw-w64-lcms2": ".src.tar.zst", + "mingw-w64-lerc": ".src.tar.zst", + "mingw-w64-libdatrie": ".src.tar.zst", + "mingw-w64-libffi": ".src.tar.zst", + "mingw-w64-libiconv": ".src.tar.zst", + "mingw-w64-libiconv": ".src.tar.zst", + "mingw-w64-libpng": ".src.tar.zst", + "mingw-w64-librsvg": ".src.tar.zst", + "mingw-w64-libsystre": ".src.tar.zst", + "mingw-w64-libtasn": ".src.tar.zst", + "mingw-w64-libthai": ".src.tar.zst", + "mingw-w64-libtiff": ".src.tar.zst", + "mingw-w64-libtre-git": ".src.tar.zst", + "mingw-w64-libwebp": ".src.tar.zst", + "mingw-w64-mpdecimal": ".src.tar.zst", + "mingw-w64-nettle": ".src.tar.zst", + "mingw-w64-p11-kit": ".src.tar.zst", + "mingw-w64-pcre": ".src.tar.zst", + "mingw-w64-pixman": ".src.tar.zst", + "mingw-w64-python-packaging": ".src.tar.zst", + "mingw-w64-readline": ".src.tar.zst", + "mingw-w64-tcl": ".src.tar.zst", + "mingw-w64-termcap": ".src.tar.zst", + "mingw-w64-tk": ".src.tar.zst", + "mingw-w64-tree-sitter": ".src.tar.zst", + "mingw-w64-tzdata": ".src.tar.zst", + "mingw-w64-wineditline": ".src.tar.zst", + "mingw-w64-zlib": ".src.tar.zst", + "mingw-w64-zstd": ".src.tar.zst", + "mingw-w64-brotli": ".src.tar.zst", + "mingw-w64-gettext": ".src.tar.zst", + "mingw-w64-libdeflate": ".src.tar.zst", + "mingw-w64-libidn2": ".src.tar.zst", + "mingw-w64-libjpeg-turbo": ".src.tar.zst", + "mingw-w64-libtasn1": ".src.tar.zst", + "mingw-w64-pcre2": ".src.tar.zst", +} ## Currently no packages seem to require this! ARCH_PKGS=[] -SRC_REPO="https://repo.msys2.org/mingw/sources" +def immediate_deps(pkg): + package_info = check_output(["pacman", "-Si", pkg]).decode("utf-8").split("\n") -def immediate_deps(pkgs): - package_info = check_output(["pacman", "-Si"] + pkgs).decode("utf-8").splitlines() + ## Extract the "Depends On" line + depends_on = [x for x in package_info if x.startswith("Depends On")][0] + ## Remove "Depends On" prefix + dependencies = depends_on.split(":")[1] - ## Extract the packages listed for "Depends On:" lines. - dependencies = [line.split(":")[1].split() for line in package_info - if line.startswith("Depends On")] - ## Flatten dependency lists from multiple packages into one list. - dependencies = functools.reduce(operator.iconcat, dependencies, []) + ## Split into dependencies + dependencies = dependencies.strip().split(" ") ## Remove > signs TODO can we get any other punctuation here? dependencies = [d.split(">")[0] for d in dependencies if d] @@ -147,18 +272,16 @@ def extract_deps(): print( "Extracting deps" ) # Get a list of all dependencies needed for packages mentioned above. - pkgs = set(PKG_REQ) - newdeps = pkgs - print("adding...") - while True: - subdeps = frozenset(immediate_deps(list(newdeps))) - newdeps = subdeps - SKIP_DEP_PKGS - pkgs - if not newdeps: - break - print('\n'.join(newdeps)) - pkgs |= newdeps + pkgs = PKG_REQ[:] + n = 0 + while n < len(pkgs): + subdeps = immediate_deps(pkgs[n]) + for p in subdeps: + if not (p in pkgs or p in SKIP_DEP_PKGS): + pkgs.append(p) + n = n + 1 - return list(pkgs) + return sorted(pkgs) def download_source(tarball): @@ -206,14 +329,24 @@ def gather_source(deps): ## Switch names if necessary pkg_name = MUNGE_SRC_PKGS.get(pkg_name,pkg_name) - tarball = "{}-{}.src.tar.gz".format(pkg_name,pkg_version) + ## src archive is usually a .tar.gz + if pkg_name in SRC_EXT.keys(): + src_ext = SRC_EXT[pkg_name] + else: + src_ext = ".src.tar.gz" + + tarball = "{}-{}{}".format(pkg_name,pkg_version,src_ext) download_source(tarball) - print("Zipping") - check_output_maybe("zip -9 ../emacs-{}-{}deps-mingw-w64-src.zip *" - .format(EMACS_MAJOR_VERSION,DATE), - shell=True) + srczip="../emacs-{}-{}deps-mingw-w64-src.zip".format(EMACS_MAJOR_VERSION,DATE) + tmpzip="{}.tmp".format(srczip) + print("Zipping Dsrc in", os.getcwd(), "as", tmpzip) + check_output_maybe("zip -9 {} *".format(tmpzip), shell=True) + if os.path.isfile(srczip): + os.remove(srczip) + os.rename(tmpzip, srczip) + print("Dsrc updated in", os.getcwd(), "as", srczip) os.chdir("..") @@ -231,6 +364,9 @@ if(os.environ["MSYSTEM"] != "MSYS"): parser = argparse.ArgumentParser() + +#parser.add_argument("emacs", help="emacs executable") + parser.add_argument("-s", help="snapshot build", action="store_true") @@ -243,19 +379,29 @@ parser.add_argument("-c", help="clean only", parser.add_argument("-d", help="dry run", action="store_true") -parser.add_argument("-l", help="list dependencies only", +parser.add_argument("-l", help="list dependencies", + action="store_true") + +parser.add_argument("-e", help="extract direct dependancies", action="store_true") args = parser.parse_args() do_all=not (args.c or args.r) - +#NEW_EMACS=args.emacs DRY_RUN=args.d +if( args.e ): + print("\n".join(init_deps())) + if( args.l ): - print("List of dependencies") - print( deps ) + print("List of dependencies:") + print(full_dll_dependency(init_deps())) + print("List of source packages:") + print( extract_deps() ) + +if( args.e or args.l ): exit(0) if args.s: