From a8f54f96fc0b91dee8a0ea6daf9dfd55b46a9178 Mon Sep 17 00:00:00 2001 From: tech4me Date: Fri, 31 Aug 2018 20:08:03 +0200 Subject: [PATCH] travis: running mingw build on travis ci This commit also fixed a broken cmake dependency with unicorn --- .travis.yml | 13 ++++ .travis/common/post-upload.sh | 3 + .travis/linux-mingw/build.sh | 3 + .travis/linux-mingw/deps.sh | 3 + .travis/linux-mingw/docker.sh | 59 ++++++++++++++++++ .travis/linux-mingw/scan_dll.py | 106 ++++++++++++++++++++++++++++++++ .travis/linux-mingw/upload.sh | 13 ++++ .travis/linux/docker.sh | 4 +- .travis/macos/build.sh | 4 +- CMakeLists.txt | 17 +++-- CMakeModules/MinGWCross.cmake | 54 ++++++++++++++++ appveyor.yml | 10 --- 12 files changed, 273 insertions(+), 16 deletions(-) create mode 100755 .travis/linux-mingw/build.sh create mode 100755 .travis/linux-mingw/deps.sh create mode 100755 .travis/linux-mingw/docker.sh create mode 100644 .travis/linux-mingw/scan_dll.py create mode 100755 .travis/linux-mingw/upload.sh create mode 100644 CMakeModules/MinGWCross.cmake diff --git a/.travis.yml b/.travis.yml index dee34a8e3..4d363cbc9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,19 @@ matrix: script: "./.travis/macos/build.sh" after_success: "./.travis/macos/upload.sh" cache: ccache + - os: linux + env: NAME="MinGW build" + sudo: required + dist: trusty + services: docker + addons: + apt: + packages: + - p7zip-full + install: "./.travis/linux-mingw/deps.sh" + script: "./.travis/linux-mingw/build.sh" + after_success: "./.travis/linux-mingw/upload.sh" + cache: ccache deploy: provider: releases diff --git a/.travis/common/post-upload.sh b/.travis/common/post-upload.sh index 90deaaec8..28735a9cf 100755 --- a/.travis/common/post-upload.sh +++ b/.travis/common/post-upload.sh @@ -11,6 +11,9 @@ if [ -z $TRAVIS_TAG ]; then RELEASE_NAME=head else RELEASE_NAME=$(echo $TRAVIS_TAG | cut -d- -f1) + if [ "$NAME" = "MinGW build" ]; then + RELEASE_NAME="${RELEASE_NAME}-mingw" + fi fi mv "$REV_NAME" $RELEASE_NAME diff --git a/.travis/linux-mingw/build.sh b/.travis/linux-mingw/build.sh new file mode 100755 index 000000000..be03cc0f3 --- /dev/null +++ b/.travis/linux-mingw/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash -ex +mkdir "$HOME/.ccache" || true +docker run --env-file .travis/common/travis-ci.env -v $(pwd):/yuzu -v "$HOME/.ccache":/root/.ccache ubuntu:18.04 /bin/bash -ex /yuzu/.travis/linux-mingw/docker.sh diff --git a/.travis/linux-mingw/deps.sh b/.travis/linux-mingw/deps.sh new file mode 100755 index 000000000..540bb934a --- /dev/null +++ b/.travis/linux-mingw/deps.sh @@ -0,0 +1,3 @@ +#!/bin/sh -ex + +docker pull ubuntu:18.04 diff --git a/.travis/linux-mingw/docker.sh b/.travis/linux-mingw/docker.sh new file mode 100755 index 000000000..d15c3f6e8 --- /dev/null +++ b/.travis/linux-mingw/docker.sh @@ -0,0 +1,59 @@ +#!/bin/bash -ex + +cd /yuzu +MINGW_PACKAGES="sdl2-mingw-w64 qt5base-mingw-w64 qt5tools-mingw-w64 libsamplerate-mingw-w64 qt5multimedia-mingw-w64" +apt-get update +apt-get install -y gpg wget git python3-pip python ccache g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64 mingw-w64-tools cmake +echo 'deb http://ppa.launchpad.net/tobydox/mingw-w64/ubuntu bionic main ' > /etc/apt/sources.list.d/extras.list +apt-key adv --keyserver keyserver.ubuntu.com --recv '72931B477E22FEFD47F8DECE02FE5F12ADDE29B2' +apt-get update +apt-get install -y ${MINGW_PACKAGES} + +# fix a problem in current MinGW headers +wget -q https://raw.githubusercontent.com/Alexpux/mingw-w64/d0d7f784833bbb0b2d279310ddc6afb52fe47a46/mingw-w64-headers/crt/errno.h -O /usr/x86_64-w64-mingw32/include/errno.h +# override Travis CI unreasonable ccache size +echo 'max_size = 3.0G' > "$HOME/.ccache/ccache.conf" + +# Dirty hack to trick unicorn makefile into believing we are in a MINGW system +mv /bin/uname /bin/uname1 && echo -e '#!/bin/sh\necho MINGW64' >> /bin/uname +chmod +x /bin/uname + +# Dirty hack to trick unicorn makefile into believing we have cmd +echo '' >> /bin/cmd +chmod +x /bin/cmd + +mkdir build && cd build +cmake .. -DCMAKE_TOOLCHAIN_FILE="$(pwd)/../CMakeModules/MinGWCross.cmake" -DUSE_CCACHE=ON -DYUZU_USE_BUNDLED_UNICORN=ON -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON -DCMAKE_BUILD_TYPE=Release +make -j4 + +# Clean up the dirty hacks +rm /bin/uname && mv /bin/uname1 /bin/uname +rm /bin/cmd + +ccache -s + +echo "Tests skipped" +#ctest -VV -C Release + +echo 'Prepare binaries...' +cd .. +mkdir package + +QT_PLATFORM_DLL_PATH='/usr/x86_64-w64-mingw32/lib/qt5/plugins/platforms/' +find build/ -name "yuzu*.exe" -exec cp {} 'package' \; + +# copy Qt plugins +mkdir package/platforms +cp "${QT_PLATFORM_DLL_PATH}/qwindows.dll" package/platforms/ +cp -rv "${QT_PLATFORM_DLL_PATH}/../mediaservice/" package/ +cp -rv "${QT_PLATFORM_DLL_PATH}/../imageformats/" package/ +rm -f package/mediaservice/*d.dll + +for i in package/*.exe; do + # we need to process pdb here, however, cv2pdb + # does not work here, so we just simply strip all the debug symbols + x86_64-w64-mingw32-strip "${i}" +done + +pip3 install pefile +python3 .travis/linux-mingw/scan_dll.py package/*.exe "package/" diff --git a/.travis/linux-mingw/scan_dll.py b/.travis/linux-mingw/scan_dll.py new file mode 100644 index 000000000..163183f2e --- /dev/null +++ b/.travis/linux-mingw/scan_dll.py @@ -0,0 +1,106 @@ +import pefile +import sys +import re +import os +import queue +import shutil + +# constant definitions +KNOWN_SYS_DLLS = ['WINMM.DLL', 'MSVCRT.DLL', 'VERSION.DLL', 'MPR.DLL', + 'DWMAPI.DLL', 'UXTHEME.DLL', 'DNSAPI.DLL', 'IPHLPAPI.DLL'] +# below is for Ubuntu 18.04 with specified PPA enabled, if you are using +# other distro or different repositories, change the following accordingly +DLL_PATH = [ + '/usr/x86_64-w64-mingw32/bin/', + '/usr/x86_64-w64-mingw32/lib/', + '/usr/lib/gcc/x86_64-w64-mingw32/7.3-posix/' +] + +missing = [] + + +def parse_imports(file_name): + results = [] + pe = pefile.PE(file_name, fast_load=True) + pe.parse_data_directories() + + for entry in pe.DIRECTORY_ENTRY_IMPORT: + current = entry.dll.decode() + current_u = current.upper() # b/c Windows is often case insensitive + # here we filter out system dlls + # dll w/ names like *32.dll are likely to be system dlls + if current_u.upper() not in KNOWN_SYS_DLLS and not re.match(string=current_u, pattern=r'.*32\.DLL'): + results.append(current) + + return results + + +def parse_imports_recursive(file_name, path_list=[]): + q = queue.Queue() # create a FIFO queue + # file_name can be a string or a list for the convience + if isinstance(file_name, str): + q.put(file_name) + elif isinstance(file_name, list): + for i in file_name: + q.put(i) + full_list = [] + while q.qsize(): + current = q.get_nowait() + print('> %s' % current) + deps = parse_imports(current) + # if this dll does not have any import, ignore it + if not deps: + continue + for dep in deps: + # the dependency already included in the list, skip + if dep in full_list: + continue + # find the requested dll in the provided paths + full_path = find_dll(dep) + if not full_path: + missing.append(dep) + continue + full_list.append(dep) + q.put(full_path) + path_list.append(full_path) + return full_list + + +def find_dll(name): + for path in DLL_PATH: + for root, _, files in os.walk(path): + for f in files: + if name.lower() == f.lower(): + return os.path.join(root, f) + + +def deploy(name, dst, dry_run=False): + dlls_path = [] + parse_imports_recursive(name, dlls_path) + for dll_entry in dlls_path: + if not dry_run: + shutil.copy(dll_entry, dst) + else: + print('[Dry-Run] Copy %s to %s' % (dll_entry, dst)) + print('Deploy completed.') + return dlls_path + + +def main(): + if len(sys.argv) < 3: + print('Usage: %s [files to examine ...] [target deploy directory]') + return 1 + to_deploy = sys.argv[1:-1] + tgt_dir = sys.argv[-1] + if not os.path.isdir(tgt_dir): + print('%s is not a directory.' % tgt_dir) + return 1 + print('Scanning dependencies...') + deploy(to_deploy, tgt_dir) + if missing: + print('Following DLLs are not found: %s' % ('\n'.join(missing))) + return 0 + + +if __name__ == '__main__': + main() diff --git a/.travis/linux-mingw/upload.sh b/.travis/linux-mingw/upload.sh new file mode 100755 index 000000000..66e896bc4 --- /dev/null +++ b/.travis/linux-mingw/upload.sh @@ -0,0 +1,13 @@ +#!/bin/bash -ex + +. .travis/common/pre-upload.sh + +REV_NAME="yuzu-windows-mingw-${GITDATE}-${GITREV}" +ARCHIVE_NAME="${REV_NAME}.tar.gz" +COMPRESSION_FLAGS="-czvf" + +mkdir "$REV_NAME" +# get around the permission issues +cp -r package/* "$REV_NAME" + +. .travis/common/post-upload.sh diff --git a/.travis/linux/docker.sh b/.travis/linux/docker.sh index 459d6bc75..892d2480a 100755 --- a/.travis/linux/docker.sh +++ b/.travis/linux/docker.sh @@ -6,7 +6,9 @@ apt-get install --no-install-recommends -y build-essential git libqt5opengl5-dev cd /yuzu mkdir build && cd build -cmake .. -DYUZU_BUILD_UNICORN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=/usr/lib/ccache/gcc -DCMAKE_CXX_COMPILER=/usr/lib/ccache/g++ -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON -G Ninja +cmake .. -DYUZU_USE_BUNDLED_UNICORN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=/usr/lib/ccache/gcc -DCMAKE_CXX_COMPILER=/usr/lib/ccache/g++ -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON -G Ninja ninja +ccache -s + ctest -VV -C Release diff --git a/.travis/macos/build.sh b/.travis/macos/build.sh index b76a153be..b881fa190 100755 --- a/.travis/macos/build.sh +++ b/.travis/macos/build.sh @@ -9,7 +9,9 @@ export PATH="/usr/local/opt/ccache/libexec:$PATH" mkdir build && cd build cmake --version -cmake .. -DYUZU_BUILD_UNICORN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON +cmake .. -DYUZU_USE_BUNDLED_UNICORN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON make -j4 +ccache -s + ctest -VV -C Release diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f32ecfba..203c3cf56 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -269,10 +269,18 @@ if (YUZU_USE_BUNDLED_UNICORN) find_package(PythonInterp 2.7 REQUIRED) - add_custom_command(OUTPUT ${LIBUNICORN_LIBRARY} - COMMAND ${CMAKE_COMMAND} -E env UNICORN_ARCHS="aarch64" PYTHON="${PYTHON_EXECUTABLE}" /bin/sh make.sh macos-universal-no - WORKING_DIRECTORY ${UNICORN_PREFIX} - ) + if (MINGW) + add_custom_command(OUTPUT ${LIBUNICORN_LIBRARY} + COMMAND ${CMAKE_COMMAND} -E env UNICORN_ARCHS="aarch64" PYTHON="${PYTHON_EXECUTABLE}" /bin/sh make.sh cross-win64 + WORKING_DIRECTORY ${UNICORN_PREFIX} + ) + else() + add_custom_command(OUTPUT ${LIBUNICORN_LIBRARY} + COMMAND ${CMAKE_COMMAND} -E env UNICORN_ARCHS="aarch64" PYTHON="${PYTHON_EXECUTABLE}" /bin/sh make.sh macos-universal-no + WORKING_DIRECTORY ${UNICORN_PREFIX} + ) + endif() + # ALL makes this custom target build every time # but it won't actually build if LIBUNICORN_LIBRARY is up to date add_custom_target(unicorn-build ALL @@ -286,6 +294,7 @@ endif() if (UNICORN_FOUND) add_library(unicorn INTERFACE) + add_dependencies(unicorn unicorn-build) target_link_libraries(unicorn INTERFACE "${LIBUNICORN_LIBRARY}") target_include_directories(unicorn INTERFACE "${LIBUNICORN_INCLUDE_DIR}") else() diff --git a/CMakeModules/MinGWCross.cmake b/CMakeModules/MinGWCross.cmake new file mode 100644 index 000000000..25750f521 --- /dev/null +++ b/CMakeModules/MinGWCross.cmake @@ -0,0 +1,54 @@ +SET(MINGW_PREFIX /usr/x86_64-w64-mingw32/) +SET(CMAKE_SYSTEM_NAME Windows) +SET(CMAKE_SYSTEM_PROCESSOR x86_64) +# Actually a hack, w/o this will cause some strange errors +SET(CMAKE_HOST_WIN32 TRUE) + + +SET(CMAKE_FIND_ROOT_PATH ${MINGW_PREFIX}) +SET(SDL2_PATH ${MINGW_PREFIX}) +SET(MINGW_TOOL_PREFIX ${CMAKE_SYSTEM_PROCESSOR}-w64-mingw32-) + +# Specify the cross compiler +SET(CMAKE_C_COMPILER ${MINGW_TOOL_PREFIX}gcc-posix) +SET(CMAKE_CXX_COMPILER ${MINGW_TOOL_PREFIX}g++-posix) +SET(CMAKE_RC_COMPILER ${MINGW_TOOL_PREFIX}windres) + +# Mingw tools +SET(STRIP ${MINGW_TOOL_PREFIX}strip) +SET(WINDRES ${MINGW_TOOL_PREFIX}windres) +SET(ENV{PKG_CONFIG} ${MINGW_TOOL_PREFIX}pkg-config) + +# ccache wrapper +OPTION(USE_CCACHE "Use ccache for compilation" OFF) +IF(USE_CCACHE) + FIND_PROGRAM(CCACHE ccache) + IF (CCACHE) + MESSAGE(STATUS "Using ccache found in PATH") + SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE}) + SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE}) + ELSE(CCACHE) + MESSAGE(WARNING "USE_CCACHE enabled, but no ccache found") + ENDIF(CCACHE) +ENDIF(USE_CCACHE) + +# Search for programs in the build host directories +SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + + +# Echo modified cmake vars to screen for debugging purposes +IF(NOT DEFINED ENV{MINGW_DEBUG_INFO}) + MESSAGE("") + MESSAGE("Custom cmake vars: (blank = system default)") + MESSAGE("-----------------------------------------") + MESSAGE("* CMAKE_C_COMPILER : ${CMAKE_C_COMPILER}") + MESSAGE("* CMAKE_CXX_COMPILER : ${CMAKE_CXX_COMPILER}") + MESSAGE("* CMAKE_RC_COMPILER : ${CMAKE_RC_COMPILER}") + MESSAGE("* WINDRES : ${WINDRES}") + MESSAGE("* ENV{PKG_CONFIG} : $ENV{PKG_CONFIG}") + MESSAGE("* STRIP : ${STRIP}") + MESSAGE("* USE_CCACHE : ${USE_CCACHE}") + MESSAGE("") + # So that the debug info only appears once + SET(ENV{MINGW_DEBUG_INFO} SHOWN) +ENDIF() diff --git a/appveyor.yml b/appveyor.yml index d68ae87b1..a6a39a545 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -167,13 +167,3 @@ artifacts: - path: $(BUILD_UPDATE) name: update -deploy: - provider: GitHub - release: $(appveyor_repo_tag_name) - auth_token: - secure: QqePPnXbkzmXct5c8hZ2X5AbsthbI6cS1Sr+VBzcD8oUOIjfWJJKXVAQGUbQAbb0 - artifact: update,build - draft: false - prerelease: false - on: - appveyor_repo_tag: true