From 2bf9ee9b997a688fdd6cfbcb0e60db465e76acda Mon Sep 17 00:00:00 2001 From: Phillip Lord Date: Thu, 7 Jan 2021 22:06:53 +0000 Subject: [PATCH] Update dependency capture * admin/nt/dist-build/build-dep-zips.py: Use ntldd to directly determine DLL dependencies --- admin/nt/dist-build/build-dep-zips.py | 122 ++++++++++++++++---------- admin/nt/dist-build/build-zips.sh | 6 +- 2 files changed, 76 insertions(+), 52 deletions(-) diff --git a/admin/nt/dist-build/build-dep-zips.py b/admin/nt/dist-build/build-dep-zips.py index 47185dbb1ba..ec99bd606d8 100755 --- a/admin/nt/dist-build/build-dep-zips.py +++ b/admin/nt/dist-build/build-dep-zips.py @@ -40,10 +40,77 @@ mingw-w64-x86_64-libtiff mingw-w64-x86_64-libxml2 mingw-w64-x86_64-xpm-nox'''.split() +DLL_REQ='''libgif +libgnutls +libharfbuzz +libjansson +liblcms2 +libturbojpeg +libpng +librsvg +libtiff +libxml +libXpm'''.split() + ## Options DRY_RUN=False + +def check_output_maybe(*args,**kwargs): + if(DRY_RUN): + print("Calling: {}{}".format(args,kwargs)) + else: + return check_output(*args,**kwargs) + +## DLL Capture +def gather_deps(arch, directory): + os.mkdir(arch) + os.chdir(arch) + + for dep in full_dll_dependency(directory): + check_output_maybe(["cp /{}/bin/{}*.dll .".format(directory, dep)], + shell=True) + + ## And package them up + ## os.chdir(arch) + print("Zipping: {}".format(arch)) + check_output_maybe("zip -9r ../emacs-{}-{}{}-deps.zip *" + .format(EMACS_MAJOR_VERSION, DATE, arch), + shell=True) + os.chdir("../") + +## Return all Emacs dependencies +def full_dll_dependency(directory): + deps = [dll_dependency(dep, directory) for dep in DLL_REQ] + return set(sum(deps, []) + DLL_REQ) + +## Dependencies for a given DLL +def dll_dependency(dll, directory): + output = check_output(["/mingw64/bin/ntldd", "--recursive", + "/{}/bin/{}*.dll".format(directory, dll)]).decode("utf-8") + ## munge output + return ntldd_munge(output) + +def ntldd_munge(out): + deps = out.splitlines() + rtn = [] + for dep in deps: + ## Output looks something like this + + ## KERNEL32.dll => C:\Windows\SYSTEM32\KERNEL32.dll (0x0000000002a30000) + ## libwinpthread-1.dll => C:\msys64\mingw64\bin\libwinpthread-1.dll (0x0000000000090000) + + ## if it's the former, we want it, if its the later we don't + splt = dep.split() + if len(splt) > 2 and "msys64" in splt[2]: + print("Adding dep", splt[0]) + rtn.append(splt[0].split(".")[0]) + + return rtn + +#### Source Capture + ## Packages to fiddle with ## Source for gcc-libs is part of gcc SKIP_SRC_PKGS=["mingw-w64-gcc-libs"] @@ -62,12 +129,6 @@ ARCH_PKGS=[] SRC_REPO="https://sourceforge.net/projects/msys2/files/REPOS/MINGW/Sources" -def check_output_maybe(*args,**kwargs): - if(DRY_RUN): - print("Calling: {}{}".format(args,kwargs)) - else: - return check_output(*args,**kwargs) - def immediate_deps(pkg): package_info = check_output(["pacman", "-Si", pkg]).decode("utf-8").split("\n") @@ -87,6 +148,7 @@ def immediate_deps(pkg): return dependencies +## Extract all the msys2 packages that are dependencies of our direct dependencies def extract_deps(): print( "Extracting deps" ) @@ -105,44 +167,6 @@ def extract_deps(): return sorted(pkgs) -def gather_deps(deps, arch, directory): - - os.mkdir(arch) - os.chdir(arch) - - ## Replace the architecture with the correct one - deps = [re.sub(r"x86_64",arch,x) for x in deps] - - ## find all files the transitive dependencies - deps_files = check_output( - ["pacman", "-Ql"] + deps - ).decode("utf-8").split("\n") - - ## Produces output like - ## mingw-w64-x86_64-zlib /mingw64/lib/libminizip.a - - ## drop the package name - tmp = deps_files.copy() - deps_files=[] - for d in tmp: - slt = d.split() - if(not slt==[]): - deps_files.append(slt[1]) - - ## sort uniq - deps_files = sorted(list(set(deps_files))) - ## copy all files into local - print("Copying dependencies: {}".format(arch)) - check_output_maybe(["rsync", "-R"] + deps_files + ["."]) - - ## And package them up - os.chdir(directory) - print("Zipping: {}".format(arch)) - check_output_maybe("zip -9r ../../emacs-{}-{}{}-deps.zip *" - .format(EMACS_MAJOR_VERSION, DATE, arch), - shell=True) - os.chdir("../../") - def download_source(tarball): print("Acquiring {}...".format(tarball)) @@ -160,6 +184,7 @@ def download_source(tarball): ) print("Downloading {}... done".format(tarball)) +## Fetch all the source code def gather_source(deps): @@ -206,7 +231,7 @@ def gather_source(deps): to_download.append(tarball) ## Download in parallel or it is just too slow - p = mp.Pool(16) + p = mp.Pool(1) p.map(download_source,to_download) print("Zipping") @@ -255,7 +280,7 @@ parser.add_argument("-l", help="list dependencies only", args = parser.parse_args() do_all=not (args.c or args.r or args.f or args.t) -deps=extract_deps() + DRY_RUN=args.d @@ -270,12 +295,13 @@ else: DATE="" if( do_all or args.t ): - gather_deps(deps,"i686","mingw32") + gather_deps("i686","mingw32") if( do_all or args.f ): - gather_deps(deps,"x86_64","mingw64") + gather_deps("x86_64","mingw64") if( do_all or args.r ): + deps=extract_deps() gather_source(deps) if( args.c ): diff --git a/admin/nt/dist-build/build-zips.sh b/admin/nt/dist-build/build-zips.sh index 4a9a7b596e7..fbb98895384 100755 --- a/admin/nt/dist-build/build-zips.sh +++ b/admin/nt/dist-build/build-zips.sh @@ -64,10 +64,8 @@ function build_zip { make -j 4 $INSTALL_TARGET \ prefix=$HOME/emacs-build/install/emacs-$VERSION/$ARCH cd $HOME/emacs-build/install/emacs-$VERSION/$ARCH - cp $HOME/emacs-build/deps/libXpm/$ARCH/libXpm-noX4.dll bin zip -r -9 emacs-$OF_VERSION-$ARCH-no-deps.zip * mv emacs-$OF_VERSION-$ARCH-no-deps.zip $HOME/emacs-upload - rm bin/libXpm-noX4.dll if [ -z $SNAPSHOT ]; then @@ -78,7 +76,7 @@ function build_zip { fi echo [build] Using $DEPS_FILE - unzip $DEPS_FILE + unzip -d bin $DEPS_FILE zip -r -9 emacs-$OF_VERSION-$ARCH.zip * mv emacs-$OF_VERSION-$ARCH.zip ~/emacs-upload @@ -208,7 +206,7 @@ then else BRANCH=$REQUIRED_BRANCH echo [build] Building from Branch $BRANCH - VERSION=$VERSION-$BRANCH + VERSION=$VERSION-${BRANCH/\//_} OF_VERSION="$VERSION-`date +%Y-%m-%d`" ## Use snapshot dependencies SNAPSHOT=1 -- 2.39.2