From c1f281eedff2cdced98b3f131087efc86f01c50e Mon Sep 17 00:00:00 2001 From: cm8 Date: Tue, 19 Jun 2018 17:56:53 +0200 Subject: [PATCH 1/6] initial mips platform support, import from issue 48 The target mips32r2_dyn is tested to fully compile, but untested to run. Target mips32r2_static reaches kiwix-lib and fails linking pthread. For further details see the comments in #48. Some known problems: * hardcoded values of icudt58l in some dependent packages unnoticable during compile, but will affect runtimes * the libdir name problem x86_64-linux-gnu <> mips-linux-gnu * pthread linking problem when compiling kiwix-lib statically may be a meson issue, since it somewhat claims to properly handle pthread linking in cross-compile situations; there are some url links in #48 laying out a proper workaround. --- kiwixbuild/dependencies/icu4c.py | 7 +- ...un_tools_w_qemu_if_endianess_differs.patch | 42 ++++++++ kiwixbuild/platforms/__init__.py | 1 + kiwixbuild/platforms/mips32r2.py | 95 +++++++++++++++++++ 4 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 kiwixbuild/patches/icu4c_run_tools_w_qemu_if_endianess_differs.patch create mode 100644 kiwixbuild/platforms/mips32r2.py diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index cc1587a18..ef29e528d 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -20,7 +20,8 @@ class Source(SvnClone): "icu4c_custom_data.patch", "icu4c_noxlocale.patch", "icu4c_rpath.patch", - "icu4c_build_config.patch"] + "icu4c_build_config.patch", + "icu4c_run_tools_w_qemu_if_endianess_differs.patch"] class Builder(MakeBuilder): @@ -41,8 +42,10 @@ def configure_option(self): icu_native_builder = get_target_step( 'icu4c', 'native_static' if platformInfo.static else 'native_dyn') - options += " --with-cross-build={} --disable-tools".format( + options += " --with-cross-build={}".format( icu_native_builder.build_path) + if platformInfo.build == 'iOS': + options += " --disable-tools" if platformInfo.build == 'android': options += " --with-data-packaging=archive" return options diff --git a/kiwixbuild/patches/icu4c_run_tools_w_qemu_if_endianess_differs.patch b/kiwixbuild/patches/icu4c_run_tools_w_qemu_if_endianess_differs.patch new file mode 100644 index 000000000..6a3c35f77 --- /dev/null +++ b/kiwixbuild/patches/icu4c_run_tools_w_qemu_if_endianess_differs.patch @@ -0,0 +1,42 @@ +diff -ur icu4c/source/data/Makefile.in icu4c.patched/source/data/Makefile.in +--- icu4c/source/data/Makefile.in 2018-06-17 03:28:46.646446362 +0200 ++++ icu4c.patched/source/data/Makefile.in 2018-06-17 03:30:44.072860064 +0200 +@@ -577,9 +577,9 @@ + # There must be a nicer way to do this. + + $(CFU_FILES): $(ALL_CFU_SOURCE) $(TOOLBINDIR)/gencfu$(TOOLEXEEXT) $(DAT_FILES) +- $(INVOKE) echo ALL_CFU_SOURCE: $(ALL_CFU_SOURCE) +- $(INVOKE) echo CFU_FILES: $(CFU_FILES) +- $(INVOKE) echo CFU_FILES_SHORT: $(CFU_FILES_SHORT) ++ echo ALL_CFU_SOURCE: $(ALL_CFU_SOURCE) ++ echo CFU_FILES: $(CFU_FILES) ++ echo CFU_FILES_SHORT: $(CFU_FILES_SHORT) + $(INVOKE) $(TOOLBINDIR)/gencfu -c -i $(BUILDDIR) -r $(word 1,$(ALL_CFU_SOURCE)) -w $(word 2,$(ALL_CFU_SOURCE)) -o $@ + + +diff -ur icu4c/source/Makefile.in icu4c.patched/source/Makefile.in +--- icu4c/source/Makefile.in 2018-06-17 03:28:45.222465606 +0200 ++++ icu4c.patched/source/Makefile.in 2018-06-17 03:29:33.757809798 +0200 +@@ -243,9 +243,20 @@ + @(echo "CROSS_ICU_VERSION=$(VERSION)" ;\ + echo "TOOLEXEEXT=$(EXEEXT)" \ + ) > $@ +- @(echo 'TOOLBINDIR=$$(cross_buildroot)/bin' ;\ ++ @(echo '' ;\ ++ echo 'ifneq ($$(ICUDATA_PLATFORM_NAME),$(ICUDATA_PLATFORM_NAME))' ;\ ++ echo 'cross_buildroot=$$(top_builddir)' ;\ ++ echo 'endif' ;\ ++ echo '' ;\ ++ echo 'TOOLBINDIR=$$(cross_buildroot)/bin' ;\ + echo 'TOOLLIBDIR=$$(cross_buildroot)/lib' ;\ + echo "INVOKE=$(LDLIBRARYPATH_ENVVAR)=$(LIBRARY_PATH_PREFIX)"'$$(TOOLLIBDIR):$$(cross_buildroot)/stubdata:$$(cross_buildroot)/tools/ctestfw:$$$$'"$(LDLIBRARYPATH_ENVVAR)" ;\ + echo "PKGDATA_INVOKE=$(LDLIBRARYPATH_ENVVAR)=$(LIBRARY_PATH_PREFIX)"'$$(cross_buildroot)/stubdata:$$(cross_buildroot)/tools/ctestfw:$$(TOOLLIBDIR):$$$$'"$(LDLIBRARYPATH_ENVVAR) " ;\ ++ echo '' ;\ ++ echo 'ifneq ($$(ICUDATA_PLATFORM_NAME),$(ICUDATA_PLATFORM_NAME))' ;\ ++ echo 'export QEMU_SET_ENV:=$$(strip $$(subst $$$$'"$(LDLIBRARYPATH_ENVVAR)"',,$$(INVOKE)))$$(strip $$(subst LD_LIBRARY_PATH=,,$$(QEMU_SET_ENV)))' ;\ ++ echo '$$(eval $$(shell grep host_cpu $$(top_builddir)/config/Makefile.inc))' ;\ ++ echo 'INVOKE=qemu-$$(host_cpu)-static' ;\ ++ echo 'endif' ;\ + echo ) >> $@ + + config/icucross.inc: $(top_builddir)/icudefs.mk $(top_builddir)/Makefile @platform_make_fragment@ diff --git a/kiwixbuild/platforms/__init__.py b/kiwixbuild/platforms/__init__.py index 4b63719b6..d7022420b 100644 --- a/kiwixbuild/platforms/__init__.py +++ b/kiwixbuild/platforms/__init__.py @@ -6,6 +6,7 @@ armhf, i586, ios, + mips32r2, native, neutral, win32 diff --git a/kiwixbuild/platforms/mips32r2.py b/kiwixbuild/platforms/mips32r2.py new file mode 100644 index 000000000..98ddd156f --- /dev/null +++ b/kiwixbuild/platforms/mips32r2.py @@ -0,0 +1,95 @@ +from .base import PlatformInfo + +from kiwixbuild.utils import pj +from kiwixbuild._global import get_target_step + + +class MIPS32R2PlatformInfo(PlatformInfo): + build = 'mips' + arch_full = 'mips-linux-gnu' + compatible_hosts = ['fedora', 'debian'] + + def get_cross_config(self): + return { + 'binaries': self.binaries, + 'exec_wrapper_def': '', + 'root_path': self.root_path, + 'extra_libs': [], + 'extra_cflags': [], + 'host_machine': { + 'system': 'linux', + 'lsystem': 'linux', + 'cpu_family': 'mips32r2', + 'cpu': '24kc', + 'endian': 'big', + 'abi': '' + } + } + + @property + def root_path(self): + return '/usr/mips-linux-gnu' + + @property + def binaries(self): + binaries = ((k,'{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + ) + return {k:pj('/usr', 'bin', v) + for k,v in binaries} + + @property + def exec_wrapper_def(self): + try: + which('qemu-mips-static') + except subprocess.CalledProcessError: + return "" + else: + return "exec_wrapper = 'qemu-mips-static'" + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def set_env(self, env): + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + # soft float is another abi and thus unsupported by current maintained (apt-get'able) debian/ubuntu tcs, + # additionally these are (gnu) libc only; if uclibc is needed, see https://buildroot.org/downloads/ and + # make a toolchain and rootfs, then checkout and use kiwixbuild --target_platform native_* within chroot env + # (it needs qemu-mips-static from host copied to rootfs/usr/bin, in order to chroot into using a non-mips-host) + #env['CFLAGS'] = " -march=mips32r2 -mtune=24kc -msoft-float "+env['CFLAGS'] + #env['CXXFLAGS'] = " -march=mips32r2 -mtune=24kc -msoft-float "+env['CXXFLAGS'] + env['CFLAGS'] = " -march=mips32r2 -mtune=24kc "+env['CFLAGS'] + env['CXXFLAGS'] = " -march=mips32r2 -mtune=24kc "+env['CXXFLAGS'] + env['QEMU_LD_PREFIX'] = pj(self.root_path) + env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( + ':'.join([ + pj(self.root_path, "lib"), + env['LD_LIBRARY_PATH'] + ])) + + def set_compiler(self, env): + env['CC'] = self.binaries['CC'] + env['CXX'] = self.binaries['CXX'] + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + +class MIPS32R2Dyn(MIPS32R2PlatformInfo): + name = 'mips32r2_dyn' + static = False + +class MIPS32R2Static(MIPS32R2PlatformInfo): + name = 'mips32r2_static' + static = True From 20241878918b8867ba96ce0cac0789452a6b9fcf Mon Sep 17 00:00:00 2001 From: cm8 Date: Thu, 21 Jun 2018 11:09:22 +0200 Subject: [PATCH 2/6] mips uclibc target mips32r2_uclibc_dyn added ATM the uclibc toolchain buildable by freetz (see https://github.com/freetz/freetz) is used. When configuring freetz make sure FREETZ_LIB_libuClibc__WITH_WCHAR=y FREETZ_BUILD_TOOLCHAIN=y are set, so uClibc++ as part of the toolchain is built with wchar_t support. Eventually root_path definition in mips32r2.py (hardcoded for now) needs to be adjusted. Some (all?) prebuilt, downloadable tcs of the freetz project do not have wchar support in uClibc++ (but uclibc does). KNOWN PROBLEMS: xapian-core currently does not compile with uClibc++ --- kiwixbuild/platforms/mips32r2.py | 91 +++++++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/kiwixbuild/platforms/mips32r2.py b/kiwixbuild/platforms/mips32r2.py index 98ddd156f..05e50cd39 100644 --- a/kiwixbuild/platforms/mips32r2.py +++ b/kiwixbuild/platforms/mips32r2.py @@ -5,7 +5,7 @@ class MIPS32R2PlatformInfo(PlatformInfo): - build = 'mips' + build = 'mips32r2' arch_full = 'mips-linux-gnu' compatible_hosts = ['fedora', 'debian'] @@ -93,3 +93,92 @@ class MIPS32R2Dyn(MIPS32R2PlatformInfo): class MIPS32R2Static(MIPS32R2PlatformInfo): name = 'mips32r2_static' static = True + + + + +class MIPS32R2_UCLIBCPlatformInfo(PlatformInfo): + build = 'mips32r2_uclibc' + arch_full = 'mips-linux-uclibc' + compatible_hosts = ['fedora', 'debian'] + + def get_cross_config(self): + return { + 'binaries': self.binaries, + 'exec_wrapper_def': '', + 'root_path': self.root_path, + 'extra_libs': [ '-lm' ], + 'extra_cflags': [], + 'host_machine': { + 'system': 'linux', + 'lsystem': 'linux', + 'cpu_family': 'mips32r2', + 'cpu': '24kc', + 'endian': 'big', + 'abi': '' + } + } + + @property + def root_path(self): + return '/dev/shm/freetz/toolchain/build/mips_gcc-5.5.0_uClibc-0.9.33.2-nptl/mips-linux-uclibc' + + @property + def binaries(self): + binaries = ((k,'{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++-uc'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + ) + return {k:pj(self.root_path, 'usr', 'bin', v) + for k,v in binaries} + + @property + def exec_wrapper_def(self): + try: + which('qemu-mips-static') + except subprocess.CalledProcessError: + return "" + else: + return "exec_wrapper = 'qemu-mips-static'" + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def set_env(self, env): + # altering PATH for mips-linux-uclibc-g++-uc wrapper to find real binary + env['PATH'] = ':'.join([pj(self.root_path, 'usr', 'bin'), env['PATH']]) + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['CFLAGS'] = " -march=mips32r2 -mtune=24kc -msoft-float -Os -pipe -Wa,--trap "+env['CFLAGS'] + env['CXXFLAGS'] = " -march=mips32r2 -mtune=24kc -msoft-float -Os -pipe -Wa,--trap "+env['CXXFLAGS'] + env['QEMU_LD_PREFIX'] = pj(self.root_path) + env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( + ':'.join([ + pj(self.root_path, "lib"), + env['LD_LIBRARY_PATH'] + ])) + + def set_compiler(self, env): + env['CC'] = self.binaries['CC'] + env['CXX'] = self.binaries['CXX'] + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + +class MIPS32R2_UCLIBCDyn(MIPS32R2_UCLIBCPlatformInfo): + name = 'mips32r2_uclibc_dyn' + static = False + +class MIPS32R2_UCLIBCStatic(MIPS32R2_UCLIBCPlatformInfo): + name = 'mips32r2_uclibc_static' + static = True From 4cbbf2325b33b2d0002019d0cdb8982760582567 Mon Sep 17 00:00:00 2001 From: cm8 Date: Sat, 23 Jun 2018 11:18:08 +0200 Subject: [PATCH 3/6] cleanup mips32r2.py and use python class inheritance for flavors added mips32r2_uclibc_gclibcxx_dyn, (preferred atm, tested on prod hw) mips32r2_uclibc_gclibcxx_static targets (shared) and renamed previous uclibc ones to mips32r2_uclibc_uclibcxx_dyn, mips32r2_uclibc_uclibcxx_static for clarity (i.e. non-shared, pure ones) reworked builder classes to use inheritance of properties and methods (instead of copying boiler plate code) mips32r2_uclibc_gclibcxx_dyn target compiles and tested to run on production targets such as avm routers modified with a freetz env. See cm8/freetz@41d97c378987786111ef78c30aa8de597e53f7e2 for one of many possible projects to build a working toolchain with. In short - git clone - umask 0022 - make menuconfig (choose expert, disable toolchain download and let the toolchain/make scripts built a gcc-5.x one, do not forget to set FREETZ_LIB_libuClibc__WITH_WCHAR=y) - read the commit message for further info on long double math peculiarities - tested here with 0.9.33.2 Remember that swap will need to be running on the box, or else kiwix-serve is likely to quit with "invalid lzma stream in cluster" errors (if memory is too low). If the box lacks library support such as libstdc++.so*, you can copy them from the target toolchain libdir over to BUILD_mips32r2_uclibc_gclibcxx_dyn/INSTALL/lib if there are unsatisfied dependencies at runtime. (Which libraries need to be supplemented this way depends on your firmware and/or freetz configuration). Another issue is the execution in non-standard installation directories. LD_LIBRARY_PATH env needs to point to "our" lib directory. If you tar INSTALL/ dir, transport the result to an embedded device, untar and wan't to run from there, it is best to wrap all the elf binaries with a shell script that correctly sets LD_LIBRARY_PATH. This step has been automated, but needs testing, see kiwixbuild/patches/fixenv-run-in-nonstd-installdir.sh for details. This fixenv script is copied to INSTALL/bin during the build and it should be run on the box, if kiwix-tools is to reside in nonstd location (i.e. if files are not installed or installable to /bin, /lib or their usr/ pendants). Feel free to improve on automation of the necessary setup steps to make mips port build and deployment easier. --- .../fixenv-run-in-nonstd-installdir.sh | 39 +++++ kiwixbuild/platforms/mips32r2.py | 150 ++++++++---------- 2 files changed, 106 insertions(+), 83 deletions(-) create mode 100644 kiwixbuild/patches/fixenv-run-in-nonstd-installdir.sh diff --git a/kiwixbuild/patches/fixenv-run-in-nonstd-installdir.sh b/kiwixbuild/patches/fixenv-run-in-nonstd-installdir.sh new file mode 100644 index 000000000..ebd97f50d --- /dev/null +++ b/kiwixbuild/patches/fixenv-run-in-nonstd-installdir.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +if ! test -f kiwix-serve +then + echo Needs to run in the same directory that hosts kiwix-serve binary. + exit 1 +fi + +find -type f | \ +while read +do + if hexdump -C $REPLY | head -1 | grep -q ELF + then + mv -iv "$REPLY" "$REPLY.real" + ln -s run-nonroot "$REPLY" + fi +done + +cat < run-nonroot +#!/bin/sh + +BASENAME=`basename $0` +SCRIPT=`realpath $0` +SCRIPTPATH=`dirname $SCRIPT` + +if test x"$BASENAME" != xrun-nonroot +then + for fullarch in "" mips-linux-gnu + do for libdir in usr/lib lib + do LD_LIBRARY_PATH="${SCRIPTPATH%/bin}/$libdir/$fullarch:$LD_LIBRARY_PATH" + done + done + + export LD_LIBRARY_PATH + exec ${SCRIPTPATH}/$BASENAME.real "$@" +fi +EOF + +chmod +x run-nonroot diff --git a/kiwixbuild/platforms/mips32r2.py b/kiwixbuild/platforms/mips32r2.py index 05e50cd39..b99c849d0 100644 --- a/kiwixbuild/platforms/mips32r2.py +++ b/kiwixbuild/platforms/mips32r2.py @@ -2,10 +2,13 @@ from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step +from os import makedirs, chmod +from shutil import copy +from .base import _SCRIPT_DIR class MIPS32R2PlatformInfo(PlatformInfo): - build = 'mips32r2' + build = 'mips32r2_glibc_glibcxx' arch_full = 'mips-linux-gnu' compatible_hosts = ['fedora', 'debian'] @@ -28,11 +31,24 @@ def get_cross_config(self): @property def root_path(self): + # apt-get'able debian/ubuntu hard-float glibc glibcxx toolchain return '/usr/mips-linux-gnu' + @property + def tcbindir(self): + return pj(self.root_path, 'bin') + + @property + def tcbinprefix(self): + return '../../bin/' + self.arch_full + '-' + + @property + def tclibdir(self): + return pj(self.root_path, 'lib') + @property def binaries(self): - binaries = ((k,'{}-{}'.format(self.arch_full, v)) + binaries = ((k,'{}{}'.format(self.tcbinprefix, v)) for k, v in (('CC', 'gcc'), ('CXX', 'g++'), ('AR', 'ar'), @@ -41,7 +57,7 @@ def binaries(self): ('RANLIB', 'ranlib'), ('LD', 'ld')) ) - return {k:pj('/usr', 'bin', v) + return {k:pj(self.tcbindir, v) for k,v in binaries} @property @@ -57,23 +73,14 @@ def exec_wrapper_def(self): def configure_option(self): return '--host={}'.format(self.arch_full) - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - def set_env(self, env): - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - # soft float is another abi and thus unsupported by current maintained (apt-get'able) debian/ubuntu tcs, - # additionally these are (gnu) libc only; if uclibc is needed, see https://buildroot.org/downloads/ and - # make a toolchain and rootfs, then checkout and use kiwixbuild --target_platform native_* within chroot env - # (it needs qemu-mips-static from host copied to rootfs/usr/bin, in order to chroot into using a non-mips-host) - #env['CFLAGS'] = " -march=mips32r2 -mtune=24kc -msoft-float "+env['CFLAGS'] - #env['CXXFLAGS'] = " -march=mips32r2 -mtune=24kc -msoft-float "+env['CXXFLAGS'] - env['CFLAGS'] = " -march=mips32r2 -mtune=24kc "+env['CFLAGS'] + env['PKG_CONFIG_LIBDIR'] = pj(self.tclibdir, 'pkgconfig') + env['CFLAGS'] = " -march=mips32r2 -mtune=24kc "+env['CFLAGS'] env['CXXFLAGS'] = " -march=mips32r2 -mtune=24kc "+env['CXXFLAGS'] env['QEMU_LD_PREFIX'] = pj(self.root_path) env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( ':'.join([ - pj(self.root_path, "lib"), + pj(self.tclibdir), env['LD_LIBRARY_PATH'] ])) @@ -97,88 +104,65 @@ class MIPS32R2Static(MIPS32R2PlatformInfo): -class MIPS32R2_UCLIBCPlatformInfo(PlatformInfo): - build = 'mips32r2_uclibc' +class MIPS32R2_UC_GCXXPlatformInfo(MIPS32R2PlatformInfo): + build = 'mips32r2_uclibc_gclibcxx' # "shared, heterogenous" (gnu c++ lib "on top of" uClibc) arch_full = 'mips-linux-uclibc' - compatible_hosts = ['fedora', 'debian'] - - def get_cross_config(self): - return { - 'binaries': self.binaries, - 'exec_wrapper_def': '', - 'root_path': self.root_path, - 'extra_libs': [ '-lm' ], - 'extra_cflags': [], - 'host_machine': { - 'system': 'linux', - 'lsystem': 'linux', - 'cpu_family': 'mips32r2', - 'cpu': '24kc', - 'endian': 'big', - 'abi': '' - } - } @property def root_path(self): return '/dev/shm/freetz/toolchain/build/mips_gcc-5.5.0_uClibc-0.9.33.2-nptl/mips-linux-uclibc' @property - def binaries(self): - binaries = ((k,'{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++-uc'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('LD', 'ld')) - ) - return {k:pj(self.root_path, 'usr', 'bin', v) - for k,v in binaries} - - @property - def exec_wrapper_def(self): - try: - which('qemu-mips-static') - except subprocess.CalledProcessError: - return "" - else: - return "exec_wrapper = 'qemu-mips-static'" - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] + def tcbinprefix(self): + return self.arch_full + '-' def set_env(self, env): - # altering PATH for mips-linux-uclibc-g++-uc wrapper to find real binary - env['PATH'] = ':'.join([pj(self.root_path, 'usr', 'bin'), env['PATH']]) - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['CFLAGS'] = " -march=mips32r2 -mtune=24kc -msoft-float -Os -pipe -Wa,--trap "+env['CFLAGS'] - env['CXXFLAGS'] = " -march=mips32r2 -mtune=24kc -msoft-float -Os -pipe -Wa,--trap "+env['CXXFLAGS'] - env['QEMU_LD_PREFIX'] = pj(self.root_path) - env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( - ':'.join([ - pj(self.root_path, "lib"), - env['LD_LIBRARY_PATH'] - ])) + super().set_env(env) + env['CFLAGS'] = " -msoft-float -Os -pipe -Wa,--trap "+env['CFLAGS'] + env['CXXFLAGS'] = " -msoft-float -Os -pipe -Wa,--trap "+env['CXXFLAGS'] - def set_compiler(self, env): - env['CC'] = self.binaries['CC'] - env['CXX'] = self.binaries['CXX'] +class MIPS32R2_UC_GCXXDyn(MIPS32R2_UC_GCXXPlatformInfo): + name = 'mips32r2_uclibc_gclibcxx_dyn' + static = False def finalize_setup(self): super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + dest = pj(_SCRIPT_DIR, '..', '..', 'BUILD_'+self.name, 'INSTALL', 'bin') + makedirs(dest, mode=0o755, exist_ok=True) + dest = pj(dest, 'fixenv-run-in-nonstd-installdir.sh') + copy(pj(_SCRIPT_DIR, '..', 'patches', 'fixenv-run-in-nonstd-installdir.sh'), dest) + chmod(dest, 0o755) + +class MIPS32R2_UC_GCXXStatic(MIPS32R2_UC_GCXXPlatformInfo): + name = 'mips32r2_uclibc_gclibcxx_static' + static = True + + + + +class MIPS32R2_UC_UCXXPlatformInfo(MIPS32R2_UC_GCXXPlatformInfo): + build = 'mips32r2_uclibc_uclibcxx' # "pure, homogenous" + + def get_cross_config(self): + conf = super().get_cross_config() + conf['extra_libs'] += [ '-lm' ] + return conf + + @property + def binaries(self): + bins = super().binaries + bins['CXX'] += '-uc' + return bins + + def set_env(self, env): + super().set_env(env) + # for mips-linux-uclibc-g++-uc wrapper to find real binary + env['PATH'] = ':'.join([pj(self.tcbindir), env['PATH']]) -class MIPS32R2_UCLIBCDyn(MIPS32R2_UCLIBCPlatformInfo): - name = 'mips32r2_uclibc_dyn' +class MIPS32R2_UC_UCXXDyn(MIPS32R2_UC_UCXXPlatformInfo): + name = 'mips32r2_uclibc_uclibcxx_dyn' static = False -class MIPS32R2_UCLIBCStatic(MIPS32R2_UCLIBCPlatformInfo): - name = 'mips32r2_uclibc_static' +class MIPS32R2_UC_UCXXStatic(MIPS32R2_UC_UCXXPlatformInfo): + name = 'mips32r2_uclibc_uclibcxx_static' static = True From 27516167534cf263c3fb482cc32aa10329166d99 Mon Sep 17 00:00:00 2001 From: cm8 Date: Sun, 24 Jun 2018 04:25:08 +0200 Subject: [PATCH 4/6] fix libdir name issue, copy tc libs for production targets The following applies to mips32r2_uclibc_glibcxx_dyn target, only: - libstdc++.so* tc libs are copied to INSTALL/lib/ - i.e. there is an example showing how to copy other libs from the toolchain to supplement the installation files (in case they are found to be missing on a target machine) - if target already supplies libstdc++.so*, copied ones will be preferred for kiwix-tools binaries (when run from a non-std installation directory on the target), drawback in this case is extra space occupied by the lib, but the gain is less hazzle on target boxes that lack libstdc++.so* - comment or modify the lines in mips32r2.py accordingly to the setup of your mips target --- kiwixbuild/buildenv.py | 2 + ...-installdir.sh => fixenv-nonstd-libdir.sh} | 2 +- kiwixbuild/platforms/mips32r2.py | 53 +++++++++++++------ 3 files changed, 39 insertions(+), 18 deletions(-) rename kiwixbuild/patches/{fixenv-run-in-nonstd-installdir.sh => fixenv-nonstd-libdir.sh} (92%) diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 156809035..fdca6484f 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -100,6 +100,8 @@ def _is_debianlike(self): return os.path.isfile('/etc/debian_version') def _detect_libdir(self): + if getattr(self.platformInfo, 'build', '').startswith('mips'): + return 'lib/' + self.platformInfo.arch_full if self._is_debianlike(): try: pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'], diff --git a/kiwixbuild/patches/fixenv-run-in-nonstd-installdir.sh b/kiwixbuild/patches/fixenv-nonstd-libdir.sh similarity index 92% rename from kiwixbuild/patches/fixenv-run-in-nonstd-installdir.sh rename to kiwixbuild/patches/fixenv-nonstd-libdir.sh index ebd97f50d..5e9845200 100644 --- a/kiwixbuild/patches/fixenv-run-in-nonstd-installdir.sh +++ b/kiwixbuild/patches/fixenv-nonstd-libdir.sh @@ -25,7 +25,7 @@ SCRIPTPATH=`dirname $SCRIPT` if test x"$BASENAME" != xrun-nonroot then - for fullarch in "" mips-linux-gnu + for fullarch in "" ${ARCH_FULL:-mips-linux-gnu} do for libdir in usr/lib lib do LD_LIBRARY_PATH="${SCRIPTPATH%/bin}/$libdir/$fullarch:$LD_LIBRARY_PATH" done diff --git a/kiwixbuild/platforms/mips32r2.py b/kiwixbuild/platforms/mips32r2.py index b99c849d0..0d123d9e0 100644 --- a/kiwixbuild/platforms/mips32r2.py +++ b/kiwixbuild/platforms/mips32r2.py @@ -1,10 +1,20 @@ -from .base import PlatformInfo +from .base import PlatformInfo, _SCRIPT_DIR from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step -from os import makedirs, chmod -from shutil import copy -from .base import _SCRIPT_DIR +from re import sub as sed + +from glob import glob +from os import makedirs, chmod, readlink, symlink +from os.path import basename, islink +from shutil import copy2 + +def copy(src, dst): + if islink(src): + linkto = readlink(src) + symlink(linkto, dst) + else: + copy2(src,dst) class MIPS32R2PlatformInfo(PlatformInfo): @@ -94,18 +104,18 @@ def finalize_setup(self): self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') class MIPS32R2Dyn(MIPS32R2PlatformInfo): - name = 'mips32r2_dyn' + name = MIPS32R2PlatformInfo.build + '_dyn' static = False class MIPS32R2Static(MIPS32R2PlatformInfo): - name = 'mips32r2_static' + name = MIPS32R2PlatformInfo.build + '_static' static = True class MIPS32R2_UC_GCXXPlatformInfo(MIPS32R2PlatformInfo): - build = 'mips32r2_uclibc_gclibcxx' # "shared, heterogenous" (gnu c++ lib "on top of" uClibc) + build = 'mips32r2_uclibc_gclibcxx' # "shared, heterogeneous" arch_full = 'mips-linux-uclibc' @property @@ -122,26 +132,35 @@ def set_env(self, env): env['CXXFLAGS'] = " -msoft-float -Os -pipe -Wa,--trap "+env['CXXFLAGS'] class MIPS32R2_UC_GCXXDyn(MIPS32R2_UC_GCXXPlatformInfo): - name = 'mips32r2_uclibc_gclibcxx_dyn' + name = MIPS32R2_UC_GCXXPlatformInfo.build + '_dyn' static = False def finalize_setup(self): super().finalize_setup() - dest = pj(_SCRIPT_DIR, '..', '..', 'BUILD_'+self.name, 'INSTALL', 'bin') - makedirs(dest, mode=0o755, exist_ok=True) - dest = pj(dest, 'fixenv-run-in-nonstd-installdir.sh') - copy(pj(_SCRIPT_DIR, '..', 'patches', 'fixenv-run-in-nonstd-installdir.sh'), dest) - chmod(dest, 0o755) + d = pj(_SCRIPT_DIR, '..', '..', 'BUILD_'+self.name, 'INSTALL') + + makedirs(pj(d, 'bin'), mode=0o755, exist_ok=True) + with open(pj(_SCRIPT_DIR, '..', 'patches', 'fixenv-nonstd-libdir.sh'), "r") as sources: + lines = sources.readlines() + with open(pj(d, 'bin', 'fixenv-nonstd-libdir'), "w") as sources: + for line in lines: + sources.write(sed(r'\$\{ARCH_FULL[^}]*\}', self.arch_full, line)) + chmod(pj(d, 'bin', 'fixenv-nonstd-libdir'), 0o755) + + d = pj(d, 'lib', self.arch_full) + makedirs(d, mode=0o755, exist_ok=True) + for f in glob(pj(self.tclibdir, 'libstdc++.so*')): + copy(f, pj(d, basename(f))) class MIPS32R2_UC_GCXXStatic(MIPS32R2_UC_GCXXPlatformInfo): - name = 'mips32r2_uclibc_gclibcxx_static' + name = MIPS32R2_UC_GCXXPlatformInfo.build + '_static' static = True class MIPS32R2_UC_UCXXPlatformInfo(MIPS32R2_UC_GCXXPlatformInfo): - build = 'mips32r2_uclibc_uclibcxx' # "pure, homogenous" + build = 'mips32r2_uclibc_uclibcxx' # "pure, homogeneous" def get_cross_config(self): conf = super().get_cross_config() @@ -160,9 +179,9 @@ def set_env(self, env): env['PATH'] = ':'.join([pj(self.tcbindir), env['PATH']]) class MIPS32R2_UC_UCXXDyn(MIPS32R2_UC_UCXXPlatformInfo): - name = 'mips32r2_uclibc_uclibcxx_dyn' + name = MIPS32R2_UC_UCXXPlatformInfo.build + '_dyn' static = False class MIPS32R2_UC_UCXXStatic(MIPS32R2_UC_UCXXPlatformInfo): - name = 'mips32r2_uclibc_uclibcxx_static' + name = MIPS32R2_UC_UCXXPlatformInfo.build + '_static' static = True From 615e994d2277c92a16645d41cf0c19cc2d09d821 Mon Sep 17 00:00:00 2001 From: cm8 Date: Sun, 24 Jun 2018 06:17:19 +0200 Subject: [PATCH 5/6] avoid heredoc expanding variables in fixenv-nonst-libdir script necessary to output run-nonroot script correctly --- kiwixbuild/patches/fixenv-nonstd-libdir.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwixbuild/patches/fixenv-nonstd-libdir.sh b/kiwixbuild/patches/fixenv-nonstd-libdir.sh index 5e9845200..477f03597 100644 --- a/kiwixbuild/patches/fixenv-nonstd-libdir.sh +++ b/kiwixbuild/patches/fixenv-nonstd-libdir.sh @@ -16,7 +16,7 @@ do fi done -cat < run-nonroot +cat <<'EOF' > run-nonroot #!/bin/sh BASENAME=`basename $0` From 26e6b819243ddf1451cbd07b6664632e23b07013 Mon Sep 17 00:00:00 2001 From: cm8 Date: Sun, 24 Jun 2018 08:48:38 +0200 Subject: [PATCH 6/6] rework fixenv-nonstd-libdir script the script won't create another file (run-nonroot previously) and instead carry out environment setup itself for the binary links pointing at it - it's more readable and smaller this way rework mips32r2.py file copy def - may be useful in other files, eventually transport this functionality to base.py (!?) later on, after this branch is merged to master --- kiwixbuild/patches/fixenv-nonstd-libdir | 27 +++++++++++++++ kiwixbuild/patches/fixenv-nonstd-libdir.sh | 39 ---------------------- kiwixbuild/platforms/mips32r2.py | 28 ++++++++++------ 3 files changed, 45 insertions(+), 49 deletions(-) create mode 100644 kiwixbuild/patches/fixenv-nonstd-libdir delete mode 100644 kiwixbuild/patches/fixenv-nonstd-libdir.sh diff --git a/kiwixbuild/patches/fixenv-nonstd-libdir b/kiwixbuild/patches/fixenv-nonstd-libdir new file mode 100644 index 000000000..0db8731cc --- /dev/null +++ b/kiwixbuild/patches/fixenv-nonstd-libdir @@ -0,0 +1,27 @@ +#!/bin/sh + +BASENAME=`basename $0` +SCRIPT=`realpath $0` +SCRIPTPATH=`dirname $SCRIPT` + +if ! test -L "$0" +then + cd "$SCRIPTPATH" && find -type f | grep -v ".real$" | \ + while read b + do + if hexdump -C "$b" | head -1 | grep -q ELF + then + mv -iv "$b" "$b.real" + ln -s "$BASENAME" "$b" + fi + done +else + for fullarch in "" mips-linux-uclibc + do for libdir in usr/lib lib + do LD_LIBRARY_PATH="${SCRIPTPATH%/bin}/$libdir/$fullarch:$LD_LIBRARY_PATH" + done + done + + export LD_LIBRARY_PATH + exec ${SCRIPTPATH}/$BASENAME.real "$@" +fi diff --git a/kiwixbuild/patches/fixenv-nonstd-libdir.sh b/kiwixbuild/patches/fixenv-nonstd-libdir.sh deleted file mode 100644 index 477f03597..000000000 --- a/kiwixbuild/patches/fixenv-nonstd-libdir.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh - -if ! test -f kiwix-serve -then - echo Needs to run in the same directory that hosts kiwix-serve binary. - exit 1 -fi - -find -type f | \ -while read -do - if hexdump -C $REPLY | head -1 | grep -q ELF - then - mv -iv "$REPLY" "$REPLY.real" - ln -s run-nonroot "$REPLY" - fi -done - -cat <<'EOF' > run-nonroot -#!/bin/sh - -BASENAME=`basename $0` -SCRIPT=`realpath $0` -SCRIPTPATH=`dirname $SCRIPT` - -if test x"$BASENAME" != xrun-nonroot -then - for fullarch in "" ${ARCH_FULL:-mips-linux-gnu} - do for libdir in usr/lib lib - do LD_LIBRARY_PATH="${SCRIPTPATH%/bin}/$libdir/$fullarch:$LD_LIBRARY_PATH" - done - done - - export LD_LIBRARY_PATH - exec ${SCRIPTPATH}/$BASENAME.real "$@" -fi -EOF - -chmod +x run-nonroot diff --git a/kiwixbuild/platforms/mips32r2.py b/kiwixbuild/platforms/mips32r2.py index 0d123d9e0..2b2797ec0 100644 --- a/kiwixbuild/platforms/mips32r2.py +++ b/kiwixbuild/platforms/mips32r2.py @@ -2,19 +2,28 @@ from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step -from re import sub as sed from glob import glob from os import makedirs, chmod, readlink, symlink from os.path import basename, islink +from re import sub as subst from shutil import copy2 -def copy(src, dst): +def copy(src, dst, search=None, repl=None, mode=None): if islink(src): linkto = readlink(src) symlink(linkto, dst) else: - copy2(src,dst) + if search is None: + copy2(src, dst) + else: + with open(src, "r") as sources: + lines = sources.readlines() + with open(dst, "w") as sources: + for line in lines: + sources.write(subst(search, repl, line)) + if mode is not None: + chmod(dst, mode) class MIPS32R2PlatformInfo(PlatformInfo): @@ -115,7 +124,7 @@ class MIPS32R2Static(MIPS32R2PlatformInfo): class MIPS32R2_UC_GCXXPlatformInfo(MIPS32R2PlatformInfo): - build = 'mips32r2_uclibc_gclibcxx' # "shared, heterogeneous" + build = 'mips32r2_uclibc_glibcxx' # "shared, heterogeneous" arch_full = 'mips-linux-uclibc' @property @@ -140,12 +149,11 @@ def finalize_setup(self): d = pj(_SCRIPT_DIR, '..', '..', 'BUILD_'+self.name, 'INSTALL') makedirs(pj(d, 'bin'), mode=0o755, exist_ok=True) - with open(pj(_SCRIPT_DIR, '..', 'patches', 'fixenv-nonstd-libdir.sh'), "r") as sources: - lines = sources.readlines() - with open(pj(d, 'bin', 'fixenv-nonstd-libdir'), "w") as sources: - for line in lines: - sources.write(sed(r'\$\{ARCH_FULL[^}]*\}', self.arch_full, line)) - chmod(pj(d, 'bin', 'fixenv-nonstd-libdir'), 0o755) + copy(pj(_SCRIPT_DIR, '..', 'patches', 'fixenv-nonstd-libdir'), + pj(d, 'bin', 'fixenv-nonstd-libdir'), + search=r'\$\{ARCH_FULL[^}]*\}', + repl=self.arch_full, + mode=0o755) d = pj(d, 'lib', self.arch_full) makedirs(d, mode=0o755, exist_ok=True)