diff --git a/.gitattributes b/.gitattributes index b4035da..7783af3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -21,6 +21,7 @@ *.rtf diff=astextplain *.RTF diff=astextplain -# Ignore build scripts in language stats +# Ignore build scripts and test cases in language stats build/* linguist-vendored configure.py linguist-vendored=true +test/* linguist-vendored diff --git a/README.md b/README.md index 55bb4da..617c131 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,9 @@ The functions do not need to deal with alignment, this is done by rpmalloc inter Memory mapping requests are always done in multiples of the memory page size. You can specify a custom page size when initializing rpmalloc with __rpmalloc_initialize_config__, or pass 0 to let rpmalloc determine the system memory page size using OS APIs. The page size MUST be a power of two in [512, 16384] range. # Memory guards -If you define the __ENABLE_GUARDS__ to 1, all memory allocations will be padded with extra guard areas before and after the memory block (while still honoring the requested alignment). These dead zones will be filled with a pattern and checked when the block is freed. If the patterns are not intact, an assert is fired. +If you define the __ENABLE_GUARDS__ to 1, all memory allocations will be padded with extra guard areas before and after the memory block (while still honoring the requested alignment). These dead zones will be filled with a pattern and checked when the block is freed. If the patterns are not intact the callback set in initialization config is called, or if not set an assert is fired. + +Note that the end of the memory block in this case is defined by the total usable size of the block as returned by `rpmalloc_usable_size`, which can be larger than the size passed to allocation request due to size class buckets. # Memory fragmentation There is no memory fragmentation by the allocator in the sense that it will not leave unallocated and unusable "holes" in the memory pages by calls to allocate and free blocks of different sizes. This is due to the fact that the memory pages allocated for each size class is split up in perfectly aligned blocks which are not reused for a request of a different size. The block freed by a call to `rpfree` will always be immediately available for an allocation request within the same size class. diff --git a/build/ninja/android.py b/build/ninja/android.py index 968e740..82fba80 100644 --- a/build/ninja/android.py +++ b/build/ninja/android.py @@ -86,7 +86,7 @@ class Android(object): self.hostarchname = 'linux-x86_64' else: self.hostarchname = 'linux-x86' - elif self.host.is_macosx(): + elif self.host.is_macos(): self.hostarchname = 'darwin-x86_64' def build_toolchain(self): diff --git a/build/ninja/clang.py b/build/ninja/clang.py index 6e2bb68..bc8f2ef 100644 --- a/build/ninja/clang.py +++ b/build/ninja/clang.py @@ -9,11 +9,11 @@ import toolchain class ClangToolchain(toolchain.Toolchain): - def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables): + def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables, subninja): #Local variable defaults self.toolchain = '' self.sdkpath = '' - self.includepaths = includepaths + self.includepaths = [] self.libpaths = libpaths self.ccompiler = 'clang' self.cxxcompiler = 'clang++' @@ -27,16 +27,16 @@ class ClangToolchain(toolchain.Toolchain): self.sysroot = '' if self.target.is_ios(): self.deploymenttarget = '9.0' - if self.target.is_macosx(): + if self.target.is_macos(): self.deploymenttarget = '10.7' #Command definitions - self.cccmd = '$toolchain$cc -MMD -MT $out -MF $out.d -I. $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags -c $in -o $out' - self.cxxcmd = '$toolchain$cxx -MMD -MT $out -MF $out.d -I. $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags $cmoreflags -c $in -o $out' + self.cccmd = '$toolchain$cc -MMD -MT $out -MF $out.d $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags -c $in -o $out' + self.cxxcmd = '$toolchain$cxx -MMD -MT $out -MF $out.d $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags $cmoreflags -c $in -o $out' self.ccdeps = 'gcc' self.ccdepfile = '$out.d' self.arcmd = self.rmcmd('$out') + ' && $toolchain$ar crsD $ararchflags $arflags $out $in' - self.linkcmd = '$toolchain$link $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags -o $out $in $libs $archlibs $oslibs $frameworks' + self.linkcmd = '$toolchain$cc $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags -o $out $in $libs $archlibs $oslibs $frameworks' #Base flags self.cflags = ['-D' + project.upper() + '_COMPILE=1', @@ -45,12 +45,23 @@ class ClangToolchain(toolchain.Toolchain): '-fno-trapping-math', '-ffast-math'] self.cwarnflags = ['-W', '-Werror', '-pedantic', '-Wall', '-Weverything', '-Wno-padded', '-Wno-documentation-unknown-command'] + self.cmoreflags = [] self.mflags = [] self.arflags = [] self.linkflags = [] self.oslibs = [] self.frameworks = [] + self.initialize_subninja(subninja) + self.initialize_archs(archs) + self.initialize_configs(configs) + self.initialize_project(project) + self.initialize_toolchain() + self.initialize_depends(dependlibs) + + self.parse_default_variables(variables) + self.read_build_prefs() + if self.target.is_linux() or self.target.is_bsd() or self.target.is_raspberrypi(): self.cflags += ['-D_GNU_SOURCE=1'] self.linkflags += ['-pthread'] @@ -60,14 +71,7 @@ class ClangToolchain(toolchain.Toolchain): if self.target.is_bsd(): self.oslibs += ['execinfo'] - self.initialize_archs(archs) - self.initialize_configs(configs) - self.initialize_project(project) - self.initialize_toolchain() - self.initialize_depends(dependlibs) - - self.parse_default_variables(variables) - self.read_build_prefs() + self.includepaths = self.prefix_includepaths((includepaths or []) + ['.']) if self.is_monolithic(): self.cflags += ['-DBUILD_MONOLITHIC=1'] @@ -75,6 +79,16 @@ class ClangToolchain(toolchain.Toolchain): self.cflags += ['--coverage'] self.linkflags += ['--coverage'] + if not 'nowarning' in variables or not variables['nowarning']: + self.cflags += self.cwarnflags + self.cxxflags = list(self.cflags) + + self.cflags += ['-std=c11'] + if self.target.is_macos() or self.target.is_ios(): + self.cxxflags += ['-std=c++14', '-stdlib=libc++'] + else: + self.cxxflags += ['-std=gnu++14'] + #Overrides self.objext = '.o' @@ -85,7 +99,7 @@ class ClangToolchain(toolchain.Toolchain): self.builders['lib'] = self.builder_lib self.builders['sharedlib'] = self.builder_sharedlib self.builders['bin'] = self.builder_bin - if self.target.is_macosx() or self.target.is_ios(): + if self.target.is_macos() or self.target.is_ios(): self.builders['m'] = self.builder_cm self.builders['multilib'] = self.builder_apple_multilib self.builders['multisharedlib'] = self.builder_apple_multisharedlib @@ -102,21 +116,6 @@ class ClangToolchain(toolchain.Toolchain): #Setup target platform self.build_toolchain() - self.cflags += ['-std=c11'] - if self.target.is_macosx() or self.target.is_ios(): - self.cxxflags += ['-std=c++11', '-stdlib=libc++'] - if self.target.is_linux(): - self.cxxflags += ['-std=gnu++11'] - - self.cmoreflags = [] - self.cexternflags = [] - self.cxxexternflags = [] - self.cexternflags += self.cflags + ['-w'] - self.cxxexternflags += self.cxxflags + ['-w'] - - self.cflags += self.cwarnflags - self.cxxflags += self.cwarnflags - def name(self): return 'clang' @@ -134,10 +133,10 @@ class ClangToolchain(toolchain.Toolchain): iosprefs = prefs['ios'] if 'deploymenttarget' in iosprefs: self.deploymenttarget = iosprefs['deploymenttarget'] - if self.target.is_macosx() and 'macosx' in prefs: - macosxprefs = prefs['macosx'] - if 'deploymenttarget' in macosxprefs: - self.deploymenttarget = macosxprefs['deploymenttarget'] + if self.target.is_macos() and 'macos' in prefs: + macosprefs = prefs['macos'] + if 'deploymenttarget' in macosprefs: + self.deploymenttarget = macosprefs['deploymenttarget'] if self.target.is_pnacl() and 'pnacl' in prefs: pnaclprefs = prefs['pnacl'] if 'sdkpath' in pnaclprefs: @@ -152,7 +151,7 @@ class ClangToolchain(toolchain.Toolchain): writer.variable('cxx', self.cxxcompiler) writer.variable('ar', self.archiver) writer.variable('link', self.linker) - if self.target.is_macosx() or self.target.is_ios(): + if self.target.is_macos() or self.target.is_ios(): writer.variable('lipo', self.lipo) if self.target.is_pnacl(): writer.variable('finalize', self.finalizer) @@ -161,7 +160,7 @@ class ClangToolchain(toolchain.Toolchain): writer.variable('moreincludepaths', '') writer.variable('cflags', self.cflags) writer.variable('cxxflags', self.cxxflags) - if self.target.is_macosx() or self.target.is_ios(): + if self.target.is_macos() or self.target.is_ios(): writer.variable('mflags', self.mflags) writer.variable('carchflags', '') writer.variable('cconfigflags', '') @@ -184,7 +183,7 @@ class ClangToolchain(toolchain.Toolchain): super(ClangToolchain, self).write_rules(writer) writer.rule('cc', command = self.cccmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CC $in') writer.rule('cxx', command = self.cxxcmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CXX $in') - if self.target.is_macosx() or self.target.is_ios(): + if self.target.is_macos() or self.target.is_ios(): writer.rule('cm', command = self.cmcmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CM $in') writer.rule( 'lipo', command = self.lipocmd, description = 'LIPO $out' ) writer.rule('ar', command = self.arcmd, description = 'LIB $out') @@ -201,21 +200,15 @@ class ClangToolchain(toolchain.Toolchain): self.build_windows_toolchain() elif self.target.is_android(): self.build_android_toolchain() - elif self.target.is_macosx() or self.target.is_ios(): + elif self.target.is_macos() or self.target.is_ios(): self.build_xcode_toolchain() elif self.target.is_pnacl(): self.build_pnacl_toolchain() - else: - self.build_default_toolchain() if self.toolchain != '' and not self.toolchain.endswith('/') and not self.toolchain.endswith('\\'): self.toolchain += os.sep - def build_default_toolchain(self): - self.cxxflags = list(self.cflags) - def build_windows_toolchain(self): self.cflags += ['-U__STRICT_ANSI__', '-Wno-reserved-id-macro'] - self.cxxflags = list(self.cflags) self.oslibs = ['kernel32', 'user32', 'shell32', 'advapi32'] def build_android_toolchain(self): @@ -225,7 +218,6 @@ class ClangToolchain(toolchain.Toolchain): self.linkcmd += ' -shared -Wl,-soname,$liblinkname --sysroot=$sysroot' self.cflags += ['-fpic', '-ffunction-sections', '-funwind-tables', '-fstack-protector', '-fomit-frame-pointer', '-no-canonical-prefixes', '-Wa,--noexecstack'] - self.cxxflags = list(self.cflags) self.linkflags += ['-no-canonical-prefixes', '-Wl,--no-undefined', '-Wl,-z,noexecstack', '-Wl,-z,relro', '-Wl,-z,now'] @@ -237,16 +229,18 @@ class ClangToolchain(toolchain.Toolchain): self.toolchain = os.path.join('$ndk', 'toolchains', 'llvm', 'prebuilt', self.android.hostarchname, 'bin', '') def build_xcode_toolchain(self): - if self.target.is_macosx(): + if self.target.is_macos(): sdk = 'macosx' deploytarget = 'MACOSX_DEPLOYMENT_TARGET=' + self.deploymenttarget self.cflags += ['-fasm-blocks', '-mmacosx-version-min=' + self.deploymenttarget, '-isysroot', '$sysroot'] + self.cxxflags += ['-fasm-blocks', '-mmacosx-version-min=' + self.deploymenttarget, '-isysroot', '$sysroot'] self.arflags += ['-static', '-no_warning_for_no_symbols'] self.linkflags += ['-isysroot', '$sysroot'] elif self.target.is_ios(): sdk = 'iphoneos' deploytarget = 'IPHONEOS_DEPLOYMENT_TARGET=' + self.deploymenttarget self.cflags += ['-fasm-blocks', '-miphoneos-version-min=' + self.deploymenttarget, '-isysroot', '$sysroot'] + self.cxxflags += ['-fasm-blocks', '-miphoneos-version-min=' + self.deploymenttarget, '-isysroot', '$sysroot'] self.arflags += ['-static', '-no_warning_for_no_symbols'] self.linkflags += ['-isysroot', '$sysroot'] self.cflags += ['-fembed-bitcode-marker'] @@ -254,25 +248,22 @@ class ClangToolchain(toolchain.Toolchain): platformpath = subprocess.check_output(['xcrun', '--sdk', sdk, '--show-sdk-platform-path']).strip() localpath = platformpath + "/Developer/usr/bin:/Applications/Xcode.app/Contents/Developer/usr/bin:/usr/bin:/bin:/usr/sbin:/sbin" - if self.target.is_macosx(): - self.sysroot = subprocess.check_output(['xcrun', '--sdk', 'macosx', '--show-sdk-path']).strip() - elif self.target.is_ios(): - self.sysroot = subprocess.check_output(['xcrun', '--sdk', 'iphoneos', '--show-sdk-path']).strip() + self.sysroot = subprocess.check_output(['xcrun', '--sdk', sdk, '--show-sdk-path']).strip() self.ccompiler = "PATH=" + localpath + " " + subprocess.check_output(['xcrun', '--sdk', sdk, '-f', 'clang']).strip() self.archiver = "PATH=" + localpath + " " + subprocess.check_output(['xcrun', '--sdk', sdk, '-f', 'libtool']).strip() self.linker = deploytarget + " " + self.ccompiler self.lipo = "PATH=" + localpath + " " + subprocess.check_output(['xcrun', '--sdk', sdk, '-f', 'lipo']).strip() - self.mflags += self.cflags + ['-fobjc-arc', '-fno-objc-exceptions', '-x', 'objective-c'] - self.cxxflags = self.cflags + ['-x', 'c++'] + self.mflags += list(self.cflags) + ['-fobjc-arc', '-fno-objc-exceptions', '-x', 'objective-c'] self.cflags += ['-x', 'c'] + self.cxxflags += ['-x', 'c++'] self.cmcmd = self.cccmd.replace('$cflags', '$mflags') self.arcmd = self.rmcmd('$out') + ' && $ar $ararchflags $arflags $in -o $out' self.lipocmd = '$lipo $in -create -output $out' - if self.target.is_macosx(): + if self.target.is_macos(): self.frameworks = ['Cocoa', 'CoreFoundation'] if self.target.is_ios(): self.frameworks = ['CoreGraphics', 'UIKit', 'Foundation'] @@ -292,7 +283,6 @@ class ClangToolchain(toolchain.Toolchain): self.linker = self.ccompiler self.finalizer = os.path.join('bin', 'pnacl-finalize' + shsuffix) self.nmfer = os.path.join('tools', 'create_nmf.py') - self.cxxflags = list(self.cflags) self.finalizecmd = '$toolchain$finalize -o $out $in' self.nmfcmd = self.python + ' ' + os.path.join('$sdkpath', '$nmf') + ' -o $out $in' @@ -303,14 +293,17 @@ class ClangToolchain(toolchain.Toolchain): def make_includepaths(self, includepaths): if not includepaths is None: - return ['-I' + self.path_escape(path) for path in list(includepaths)] + return ['-I' + path for path in list(includepaths)] return [] + def make_libpath(self, path): + return self.path_escape(path) + def make_libpaths(self, libpaths): if not libpaths is None: if self.target.is_windows(): return ['-Xlinker /LIBPATH:' + self.path_escape(path) for path in libpaths] - return ['-L' + self.path_escape(path) for path in libpaths] + return ['-L' + self.make_libpath(path) for path in libpaths] return [] def make_targetarchflags(self, arch, targettype): @@ -335,7 +328,7 @@ class ClangToolchain(toolchain.Toolchain): elif arch == 'mips64': flags += ['-target', 'mips64el-none-linux-android'] flags += ['-gcc-toolchain', self.android.make_gcc_toolchain_path(arch)] - elif self.target.is_macosx() or self.target.is_ios(): + elif self.target.is_macos() or self.target.is_ios(): if arch == 'x86': flags += [' -arch x86'] elif arch == 'x86-64': @@ -355,8 +348,6 @@ class ClangToolchain(toolchain.Toolchain): flags = [] if targettype == 'sharedlib': flags += ['-DBUILD_DYNAMIC_LINK=1'] - if self.target.is_linux(): - flags += ['-fPIC'] flags += self.make_targetarchflags(arch, targettype) return flags @@ -391,7 +382,7 @@ class ClangToolchain(toolchain.Toolchain): flags += ['-Xlinker', '/MACHINE:X86'] elif arch == 'x86-64': flags += ['-Xlinker', '/MACHINE:X64'] - if self.target.is_macosx() and 'support_lua' in variables and variables['support_lua']: + if self.target.is_macos() and 'support_lua' in variables and variables['support_lua']: flags += ['-pagezero_size', '10000', '-image_base', '100000000'] return flags @@ -402,12 +393,6 @@ class ClangToolchain(toolchain.Toolchain): flags += ['-Xlinker', '/DLL'] elif targettype == 'bin': flags += ['-Xlinker', '/SUBSYSTEM:CONSOLE'] - else: - if targettype == 'sharedlib': - if self.target.is_macosx() or self.target.is_ios(): - flags += ['-dynamiclib'] - else: - flags += ['-shared'] return flags def make_linkarchlibs(self, arch, targettype): @@ -432,18 +417,18 @@ class ClangToolchain(toolchain.Toolchain): def make_configlibpaths(self, config, arch, extralibpaths): libpaths = [self.libpath, os.path.join(self.libpath, config)] - if not self.target.is_macosx() and not self.target.is_ios(): + if not self.target.is_macos() and not self.target.is_ios(): libpaths += [os.path.join(self.libpath, arch)] libpaths += [os.path.join(self.libpath, config, arch)] if extralibpaths != None: libpaths += [os.path.join(libpath, self.libpath) for libpath in extralibpaths] libpaths += [os.path.join(libpath, self.libpath, config) for libpath in extralibpaths] - if not self.target.is_macosx() and not self.target.is_ios(): + if not self.target.is_macos() and not self.target.is_ios(): libpaths += [os.path.join(libpath, self.libpath, arch) for libpath in extralibpaths] libpaths += [os.path.join(libpath, self.libpath, config, arch) for libpath in extralibpaths] return self.make_libpaths(libpaths) - def cc_variables(self, config, arch, targettype, variables, externalsources): + def cc_variables(self, config, arch, targettype, variables): localvariables = [] if 'includepaths' in variables: moreincludepaths = self.make_includepaths(variables['includepaths']) @@ -457,8 +442,6 @@ class ClangToolchain(toolchain.Toolchain): localvariables += [('cconfigflags', cconfigflags)] if self.target.is_android(): localvariables += [('sysroot', self.android.make_sysroot_path(arch))] - if externalsources: - localvariables += [('cflags', self.cexternflags), ('cxxflags', self.cxxexternflags)] if 'defines' in variables: localvariables += [('cmoreflags', ['-D' + define for define in variables['defines']])] return localvariables @@ -477,8 +460,6 @@ class ClangToolchain(toolchain.Toolchain): def link_variables(self, config, arch, targettype, variables): localvariables = [] - if not variables: - variables = {} linkarchflags = self.make_linkarchflags(arch, targettype, variables) if linkarchflags != []: localvariables += [('linkarchflags', linkarchflags)] @@ -495,7 +476,7 @@ class ClangToolchain(toolchain.Toolchain): localframeworks += list(variables['frameworks']) if len(localframeworks) > 0: localvariables += [('frameworks', self.make_frameworks(list(localframeworks)))] - + libpaths = [] if 'libpaths' in variables: libpaths = variables['libpaths'] @@ -511,40 +492,39 @@ class ClangToolchain(toolchain.Toolchain): return localvariables - def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables, externalsources): - return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources)) + def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables): + return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables)) - def builder_cxx(self, writer, config, arch, targettype, infile, outfile, variables, externalsources): - return writer.build(outfile, 'cxx', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources)) + def builder_cxx(self, writer, config, arch, targettype, infile, outfile, variables): + return writer.build(outfile, 'cxx', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables)) - def builder_cm(self, writer, config, arch, targettype, infile, outfile, variables, externalsources): - return writer.build(outfile, 'cm', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources)) + def builder_cm(self, writer, config, arch, targettype, infile, outfile, variables): + return writer.build(outfile, 'cm', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables)) - def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables): return writer.build(outfile, 'ar', infiles, implicit = self.implicit_deps(config, variables), variables = self.ar_variables(config, arch, targettype, variables)) - def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables): return writer.build(outfile, 'so', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables)) - def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables): return writer.build(outfile, 'link', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables)) #Apple universal targets - def builder_apple_multilib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_apple_multilib(self, writer, config, arch, targettype, infiles, outfile, variables): localvariables = [('arflags', '-static -no_warning_for_no_symbols')] if variables != None: localvariables = variables + localvariables return writer.build(os.path.join(outfile, self.buildtarget), 'ar', infiles, variables = localvariables); - def builder_apple_multisharedlib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): - #return writer.build(outfile, 'so', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables)) - return writer.build(os.path.join(outfile, self.buildtarget), 'lipo', infiles, variables = variables) + def builder_apple_multisharedlib(self, writer, config, arch, targettype, infiles, outfile, variables): + return writer.build(outfile, 'so', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables)) - def builder_apple_multibin(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_apple_multibin(self, writer, config, arch, targettype, infiles, outfile, variables): return writer.build(os.path.join(outfile, self.buildtarget), 'lipo', infiles, variables = variables) #PNaCl finalizer - def builder_pnacl_multibin(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_pnacl_multibin(self, writer, config, arch, targettype, infiles, outfile, variables): binfile = os.path.splitext(self.buildtarget)[0] pexe = writer.build(os.path.join(outfile, binfile + '.pexe'), 'finalize', infiles) nmf = writer.build(os.path.join(outfile, binfile + '.nmf'), 'nmf', pexe + infiles) diff --git a/build/ninja/codesign.py b/build/ninja/codesign.py index ad8fe35..fff4528 100644 --- a/build/ninja/codesign.py +++ b/build/ninja/codesign.py @@ -14,7 +14,7 @@ parser.add_argument('file', type=str, help = 'Bundle/package to sign') parser.add_argument('--target', type=str, help = 'Target', - choices = ['macosx', 'ios', 'android'], + choices = ['macos', 'ios', 'android'], default = '') parser.add_argument('--bundle', type=str, help = 'Bundle identifier (OSX/iOS)', @@ -65,13 +65,13 @@ options = parser.parse_args() androidprefs = {} iosprefs = {} -macosxprefs = {} +macosprefs = {} def parse_prefs( prefsfile ): global androidprefs global iosprefs - global macosxprefs + global macosprefs if not os.path.isfile( prefsfile ): return file = open( prefsfile, 'r' ) @@ -81,8 +81,8 @@ def parse_prefs( prefsfile ): androidprefs = prefs['android'] if 'ios' in prefs: iosprefs = prefs['ios'] - if 'macosx' in prefs: - macosxprefs = prefs['macosx'] + if 'macos' in prefs: + macosprefs = prefs['macos'] def codesign_ios(): @@ -127,7 +127,7 @@ def codesign_ios(): if os.path.isfile( os.path.join( options.file, '_CodeSignature', 'CodeResources' ) ): os.remove( os.path.join( options.file, '_CodeSignature', 'CodeResources' ) ) - os.system( '/usr/bin/codesign --force --sign ' + iosprefs['signature'] + ' --entitlements ' + plistpath + ' ' + options.file ) + os.system( '/usr/bin/codesign --force --sign "' + iosprefs['signature'] + '" --entitlements ' + plistpath + ' ' + options.file ) if os.path.isfile( os.path.join( options.file, '_CodeSignature', 'CodeResources' ) ): os.utime( os.path.join( options.file, '_CodeSignature', 'CodeResources' ), None ) @@ -135,13 +135,13 @@ def codesign_ios(): os.utime( options.file, None ) -def codesign_macosx(): - if not 'organisation' in macosxprefs: - macosxprefs['organisation'] = options.organisation - if not 'bundleidentifier' in macosxprefs: - macosxprefs['bundleidentifier'] = options.bundle - if not 'provisioning' in macosxprefs: - macosxprefs['provisioning'] = options.provisioning +def codesign_macos(): + if not 'organisation' in macosprefs: + macosprefs['organisation'] = options.organisation + if not 'bundleidentifier' in macosprefs: + macosprefs['bundleidentifier'] = options.bundle + if not 'provisioning' in macosprefs: + macosprefs['provisioning'] = options.provisioning codesign_allocate = subprocess.check_output( [ 'xcrun', '--sdk', 'macosx', '-f', 'codesign_allocate' ] ).strip() sdkdir = subprocess.check_output( [ 'xcrun', '--sdk', 'macosx', '--show-sdk-path' ] ).strip() @@ -150,8 +150,8 @@ def codesign_macosx(): if os.path.isfile( os.path.join( options.file, 'Contents', '_CodeSignature', 'CodeResources' ) ): os.remove( os.path.join( options.file, 'Contents', '_CodeSignature', 'CodeResources' ) ) - if 'signature' in macosxprefs: - os.system( 'export CODESIGN_ALLOCATE=' + codesign_allocate + '; /usr/bin/codesign --force --sign ' + macosxprefs['signature'] + ' ' + options.file ) + if 'signature' in macosprefs: + os.system( 'export CODESIGN_ALLOCATE=' + codesign_allocate + '; /usr/bin/codesign --force --sign ' + macosprefs['signature'] + ' ' + options.file ) if os.path.isfile( os.path.join( options.file, 'Contents', '_CodeSignature', 'CodeResources' ) ): os.utime( os.path.join( options.file, 'Contents', '_CodeSignature', 'CodeResources' ), None ) @@ -215,7 +215,7 @@ parse_prefs( options.prefs ) if options.target == 'ios': codesign_ios() -elif options.target == 'macosx': - codesign_macosx() +elif options.target == 'macos': + codesign_macos() elif options.target == 'android': codesign_android() diff --git a/build/ninja/gcc.py b/build/ninja/gcc.py index fdd5253..a30f133 100644 --- a/build/ninja/gcc.py +++ b/build/ninja/gcc.py @@ -8,24 +8,24 @@ import toolchain class GCCToolchain(toolchain.Toolchain): - def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables): + def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables, subninja): #Local variable defaults self.toolchain = '' - self.includepaths = includepaths + self.includepaths = [] self.libpaths = libpaths self.ccompiler = 'gcc' - self.cxxompiler = 'g++' + self.cxxcompiler = 'g++' self.archiver = 'ar' self.linker = 'gcc' self.cxxlinker = 'g++' #Command definitions - self.cccmd = '$toolchain$cc -MMD -MT $out -MF $out.d -I. $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags -c $in -o $out' - self.cxxcmd = '$toolchain$cxx -MMD -MT $out -MF $out.d -I. $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags $cmoreflags -c $in -o $out' + self.cccmd = '$toolchain$cc -MMD -MT $out -MF $out.d $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags -c $in -o $out' + self.cxxcmd = '$toolchain$cxx -MMD -MT $out -MF $out.d $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags $cmoreflags -c $in -o $out' self.ccdeps = 'gcc' self.ccdepfile = '$out.d' self.arcmd = self.rmcmd('$out') + ' && $toolchain$ar crsD $ararchflags $arflags $out $in' - self.linkcmd = '$toolchain$link $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags -o $out $in $libs $archlibs $oslibs' + self.linkcmd = '$toolchain$cc $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags -o $out $in $libs $archlibs $oslibs' #Base flags self.cflags = ['-D' + project.upper() + '_COMPILE=1', @@ -33,17 +33,13 @@ class GCCToolchain(toolchain.Toolchain): '-fno-math-errno','-ffinite-math-only', '-funsafe-math-optimizations', '-fno-trapping-math', '-ffast-math'] self.cwarnflags = ['-Wextra', '-Wall', '-Werror'] + self.cmoreflags = [] self.mflags = [] self.arflags = [] self.linkflags = [] self.oslibs = [] - if self.target.is_linux() or self.target.is_bsd() or self.target.is_raspberrypi(): - self.linkflags += ['-pthread'] - self.oslibs += ['m'] - if self.target.is_linux() or self.target.is_raspberrypi(): - self.oslibs += ['dl'] - + self.initialize_subninja(subninja) self.initialize_archs(archs) self.initialize_configs(configs) self.initialize_project(project) @@ -53,8 +49,32 @@ class GCCToolchain(toolchain.Toolchain): self.parse_default_variables(variables) self.read_build_prefs() + if self.target.is_linux() or self.target.is_bsd() or self.target.is_raspberrypi(): + self.cflags += ['-D_GNU_SOURCE=1'] + self.linkflags += ['-pthread'] + self.oslibs += ['m'] + if self.target.is_linux() or self.target.is_raspberrypi(): + self.oslibs += ['dl'] + if self.target.is_bsd(): + self.oslibs += ['execinfo'] + + self.includepaths = self.prefix_includepaths((includepaths or []) + ['.']) + if self.is_monolithic(): self.cflags += ['-DBUILD_MONOLITHIC=1'] + if self.use_coverage(): + self.cflags += ['--coverage'] + self.linkflags += ['--coverage'] + + if not 'nowarning' in variables or not variables['nowarning']: + self.cflags += self.cwarnflags + self.cxxflags = list(self.cflags) + + self.cflags += ['-std=c11'] + if self.target.is_macos() or self.target.is_ios(): + self.cxxflags += ['-std=c++14', '-stdlib=libc++'] + else: + self.cxxflags += ['-std=gnu++14'] #Overrides self.objext = '.o' @@ -62,7 +82,6 @@ class GCCToolchain(toolchain.Toolchain): #Builders self.builders['c'] = self.builder_cc self.builders['cc'] = self.builder_cxx - self.builders['cpp'] = self.builder_cxx self.builders['lib'] = self.builder_lib self.builders['multilib'] = self.builder_multicopy self.builders['sharedlib'] = self.builder_sharedlib @@ -73,15 +92,6 @@ class GCCToolchain(toolchain.Toolchain): #Setup target platform self.build_target_toolchain(self.target) - self.cmoreflags = [] - self.cexternflags = [] - self.cxxexternflags = [] - self.cexternflags += self.cflags - self.cxxexternflags += self.cxxflags - - self.cflags += self.cwarnflags - self.cxxflags += self.cwarnflags - def name(self): return 'gcc' @@ -98,7 +108,7 @@ class GCCToolchain(toolchain.Toolchain): super(GCCToolchain, self).write_variables(writer) writer.variable('toolchain', self.toolchain) writer.variable('cc', self.ccompiler) - writer.variable('cxx', self.cxxompiler) + writer.variable('cxx', self.cxxcompiler) writer.variable('ar', self.archiver) writer.variable('link', self.linker) writer.variable('includepaths', self.make_includepaths(self.includepaths)) @@ -133,31 +143,31 @@ class GCCToolchain(toolchain.Toolchain): def build_target_toolchain(self, target): if target.is_windows(): self.build_windows_toolchain() - else: - self.build_default_toolchain() if self.toolchain != '' and not self.toolchain.endswith('/') and not self.toolchain.endswith('\\'): self.toolchain += os.sep - def build_default_toolchain(self): - self.cxxflags = self.cflags + ['-std=c++11', '-D_GNU_SOURCE=1'] - self.cflags += ['-std=c11', '-D_GNU_SOURCE=1'] - if self.target.is_macosx() or self.target.is_ios(): - self.cxxflags += ['-stdlib=libc++'] - def build_windows_toolchain(self): - self.cxxflags = self.cflags - self.cxxflags += ['-U__STRICT_ANSI__', '-std=c++11'] - self.cflags += ['-U__STRICT_ANSI__', '-std=c11'] + self.cflags += ['-U__STRICT_ANSI__'] self.oslibs = ['kernel32', 'user32', 'shell32', 'advapi32'] + def make_includepath(self, path): + if os.path.isabs(path) or self.subninja == '': + return self.path_escape(path) + if path == '.': + return self.path_escape(self.subninja) + return self.path_escape(os.path.join(self.subninja, path)) + def make_includepaths(self, includepaths): if not includepaths is None: - return ['-I' + self.path_escape(path) for path in list(includepaths)] + return ['-I' + self.make_includepath(path) for path in list(includepaths)] return [] + def make_libpath(self, path): + return self.path_escape(path) + def make_libpaths(self, libpaths): if not libpaths is None: - return ['-L' + self.path_escape(path) for path in libpaths] + return ['-L' + self.make_libpath(path) for path in libpaths] return [] def make_targetarchflags(self, arch, targettype): @@ -172,8 +182,6 @@ class GCCToolchain(toolchain.Toolchain): flags = [] if targettype == 'sharedlib': flags += ['-DBUILD_DYNAMIC_LINK=1'] - if self.target.is_linux(): - flags += ['-fPIC'] flags += self.make_targetarchflags(arch, targettype) return flags @@ -204,17 +212,6 @@ class GCCToolchain(toolchain.Toolchain): def make_linkconfigflags(self, config, targettype): flags = [] - if self.target.is_windows(): - if targettype == 'sharedlib': - flags += ['-Xlinker', '/DLL'] - elif targettype == 'bin': - flags += ['-Xlinker', '/SUBSYSTEM:CONSOLE'] - else: - if targettype == 'sharedlib': - if self.target.is_macosx() or self.target.is_ios(): - flags += ['-dynamiclib'] - else: - flags += ['-shared'] return flags def make_libs(self, libs): @@ -236,7 +233,7 @@ class GCCToolchain(toolchain.Toolchain): libpaths += [os.path.join(libpath, self.libpath, config, arch) for libpath in extralibpaths] return self.make_libpaths(libpaths) - def cc_variables(self, config, arch, targettype, variables, externalsources): + def cc_variables(self, config, arch, targettype, variables): localvariables = [] if 'includepaths' in variables: moreincludepaths = self.make_includepaths(variables['includepaths']) @@ -248,8 +245,6 @@ class GCCToolchain(toolchain.Toolchain): cconfigflags = self.make_cconfigflags(config, targettype) if cconfigflags != []: localvariables += [('cconfigflags', cconfigflags)] - if externalsources: - localvariables += [('cflags', self.cexternflags), ('cxxflags', self.cxxexternflags)] if 'defines' in variables: localvariables += [('cmoreflags', ['-D' + define for define in variables['defines']])] return localvariables @@ -280,23 +275,25 @@ class GCCToolchain(toolchain.Toolchain): if 'libpaths' in variables: libpaths = variables['libpaths'] localvariables += [('configlibpaths', self.make_configlibpaths(config, arch, libpaths))] + if 'runtime' in variables and variables['runtime'] == 'c++': localvariables += [('link', self.cxxlinker)] + return localvariables - def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables, externalsources): - return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources)) + def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables): + return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables)) - def builder_cxx(self, writer, config, arch, targettype, infile, outfile, variables, externalsources): - return writer.build(outfile, 'cxx', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources)) + def builder_cxx(self, writer, config, arch, targettype, infile, outfile, variables): + return writer.build(outfile, 'cxx', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables)) - def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables): return writer.build(outfile, 'ar', infiles, implicit = self.implicit_deps(config, variables), variables = self.ar_variables(config, arch, targettype, variables)) - def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables): return writer.build(outfile, 'so', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables)) - def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables): return writer.build(outfile, 'link', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables)) def create(host, target, toolchain): diff --git a/build/ninja/generator.py b/build/ninja/generator.py index 725f490..ce504b2 100644 --- a/build/ninja/generator.py +++ b/build/ninja/generator.py @@ -25,7 +25,7 @@ class Generator(object): choices = toolchain.supported_toolchains()) parser.add_argument('-c', '--config', action = 'append', help = 'Build configuration', - choices = ['debug', 'release'], + choices = ['debug', 'release', 'profile', 'deploy'], default = []) parser.add_argument('-a', '--arch', action = 'append', help = 'Add architecture', @@ -40,11 +40,15 @@ class Generator(object): parser.add_argument('--coverage', action='store_true', help = 'Build with code coverage', default = False) + parser.add_argument('--subninja', action='store', + help = 'Build as subproject (exclude rules and pools) with the given subpath', + default = '') options = parser.parse_args() self.project = project self.target = platform.Platform(options.target) self.host = platform.Platform(options.host) + self.subninja = options.subninja archs = options.arch configs = options.config if includepaths is None: @@ -72,23 +76,20 @@ class Generator(object): config_str = ' '.join([key + '=' + pipes.quote(configure_env[key]) for key in configure_env]) writer.variable('configure_env', config_str + '$ ') + if variables is None: + variables = {} + if not isinstance(variables, dict): + variables = dict(variables) + if options.monolithic: - if variables is None: - variables = {} - if isinstance(variables, dict): - variables['monolithic'] = True - else: - variables += [('monolithic', True)] + variables['monolithic'] = True if options.coverage: - if variables is None: - variables = {} - if isinstance(variables, dict): - variables['coverage'] = True - else: - variables += [('coverage', True)] + variables['coverage'] = True + if self.subninja != '': + variables['internal_deps'] = True self.toolchain = toolchain.make_toolchain(self.host, self.target, options.toolchain) - self.toolchain.initialize(project, archs, configs, includepaths, dependlibs, libpaths, variables) + self.toolchain.initialize(project, archs, configs, includepaths, dependlibs, libpaths, variables, self.subninja) self.writer.variable('configure_toolchain', self.toolchain.name()) self.writer.variable('configure_archs', archs) @@ -96,7 +97,8 @@ class Generator(object): self.writer.newline() self.toolchain.write_variables(self.writer) - self.toolchain.write_rules(self.writer) + if self.subninja == '': + self.toolchain.write_rules(self.writer) def target(self): return self.target @@ -110,19 +112,23 @@ class Generator(object): def writer(self): return self.writer - def lib(self, module, sources, libname, basepath = None, configs = None, includepaths = None, variables = None, externalsources = False): - return self.toolchain.lib(self.writer, module, sources, libname, basepath, configs, includepaths, variables, None, externalsources) + def is_subninja(self): + return self.subninja != '' - def sharedlib(self, module, sources, libname, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, libs = None, frameworks = None, variables = None, externalsources = False): - return self.toolchain.sharedlib(self.writer, module, sources, libname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, None, externalsources) + def lib(self, module, sources, libname = None, basepath = None, configs = None, includepaths = None, variables = None): + return self.toolchain.lib(self.writer, module, sources, libname, basepath, configs, includepaths, variables) - def bin(self, module, sources, binname, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, libs = None, frameworks = None, variables = None, outpath = None, externalsources = False): - return self.toolchain.bin(self.writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, outpath, externalsources) + def sharedlib(self, module, sources, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, dependlibs = None, libs = None, frameworks = None, variables = None): + return self.toolchain.sharedlib(self.writer, module, sources, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables) - def app(self, module, sources, binname, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, libs = None, frameworks = None, variables = None, resources = None): - return self.toolchain.app(self.writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, resources) + def bin(self, module, sources, binname, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, dependlibs = None, libs = None, frameworks = None, variables = None): + return self.toolchain.bin(self.writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables) + + def app(self, module, sources, binname, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, dependlibs = None, libs = None, frameworks = None, variables = None, resources = None): + return self.toolchain.app(self.writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables, resources) def test_includepaths(self): + #TODO: This is ugly if self.project == "foundation": return ['test'] return ['test', os.path.join('..', 'foundation_lib', 'test')] diff --git a/build/ninja/msvc.py b/build/ninja/msvc.py index 79eb9b8..1421319 100644 --- a/build/ninja/msvc.py +++ b/build/ninja/msvc.py @@ -9,11 +9,11 @@ import toolchain class MSVCToolchain(toolchain.Toolchain): - def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables): + def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables, subninja): #Local variable defaults self.sdkpath = '' self.toolchain = '' - self.includepaths = includepaths + self.includepaths = [] self.libpaths = libpaths self.ccompiler = 'cl' self.archiver = 'lib' @@ -21,30 +21,20 @@ class MSVCToolchain(toolchain.Toolchain): self.dller = 'dll' #Command definitions - self.cccmd = '$toolchain$cc /showIncludes /I. $includepaths $moreincludepaths $cflags $carchflags $cconfigflags /c $in /Fo$out /Fd$pdbpath /FS /nologo' - self.cxxcmd = '$toolchain$cc /showIncludes /I. $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags /c $in /Fo$out /Fd$pdbpath /FS /nologo /EHsc' + self.cccmd = '$toolchain$cc /showIncludes /I. $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags /c $in /Fo$out /Fd$pdbpath /FS /nologo' self.ccdepfile = None self.ccdeps = 'msvc' self.arcmd = '$toolchain$ar $arflags $ararchflags $arconfigflags /NOLOGO /OUT:$out $in' self.linkcmd = '$toolchain$link $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags /DEBUG /NOLOGO /SUBSYSTEM:CONSOLE /DYNAMICBASE /NXCOMPAT /MANIFEST /MANIFESTUAC:\"level=\'asInvoker\' uiAccess=\'false\'\" /TLBID:1 /PDB:$pdbpath /OUT:$out $in $libs $archlibs $oslibs' self.dllcmd = self.linkcmd + ' /DLL' - #Base flags - self.cflags = ['/D', '"' + project.upper() + '_COMPILE=1"', '/Zi', '/Oi', '/Oy-', '/GS-', '/Gy-', '/Qpar-', '/fp:fast', '/fp:except-', '/Zc:forScope', '/Zc:wchar_t', '/GR-', '/openmp-'] - self.cxxflags = self.cflags - self.cwarnflags = ['/W3', '/WX'] + self.cflags = ['/D', '"' + project.upper() + '_COMPILE=1"', '/D', '"_UNICODE"', '/D', '"UNICODE"', '/Zi', '/W3', '/WX', '/Oi', '/Oy-', '/GS-', '/Gy-', '/Qpar-', '/fp:fast', '/fp:except-', '/Zc:forScope', '/Zc:wchar_t', '/GR-', '/openmp-'] + self.cmoreflags = [] self.arflags = ['/ignore:4221'] #Ignore empty object file warning] self.linkflags = ['/DEBUG'] self.oslibs = ['kernel32', 'user32', 'shell32', 'advapi32'] - self.cexternflags = ['/W0'] - self.cxxexternflags = ['/W0', '/EHsc'] - self.cexternflags += self.cflags - self.cxxexternflags += self.cxxflags - - self.cflags += self.cwarnflags - self.cxxflags += self.cwarnflags - + self.initialize_subninja(subninja) self.initialize_archs(archs) self.initialize_configs(configs) self.initialize_project(project) @@ -54,6 +44,8 @@ class MSVCToolchain(toolchain.Toolchain): self.parse_default_variables(variables) self.read_build_prefs() + self.includepaths = self.prefix_includepaths((includepaths or []) + ['.']) + if self.is_monolithic(): self.cflags += ['/D', '"BUILD_MONOLITHIC=1"'] @@ -62,8 +54,6 @@ class MSVCToolchain(toolchain.Toolchain): #Builders self.builders['c'] = self.builder_cc - self.builders['cc'] = self.builder_cxx - self.builders['cpp'] = self.builder_cxx self.builders['lib'] = self.builder_lib self.builders['multilib'] = self.builder_multicopy self.builders['sharedlib'] = self.builder_sharedlib @@ -99,7 +89,7 @@ class MSVCToolchain(toolchain.Toolchain): writer.variable('cflags', self.cflags) writer.variable('carchflags', '') writer.variable('cconfigflags', '') - writer.variable('cxxflags', self.cxxflags) + writer.variable('cmoreflags', self.cmoreflags) writer.variable('arflags', self.arflags) writer.variable('ararchflags', '') writer.variable('arconfigflags', '') @@ -116,7 +106,6 @@ class MSVCToolchain(toolchain.Toolchain): def write_rules(self, writer): super(MSVCToolchain, self).write_rules(writer) writer.rule('cc', command = self.cccmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CC $in') - writer.rule('cxx', command = self.cxxcmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CXX $in') writer.rule('ar', command = self.arcmd, description = 'LIB $out') writer.rule('link', command = self.linkcmd, description = 'LINK $out') writer.rule('dll', command = self.dllcmd, description = 'DLL $out') @@ -213,9 +202,12 @@ class MSVCToolchain(toolchain.Toolchain): return ['/I' + self.path_escape(path) for path in list(includepaths)] return [] + def make_libpath(self, path): + return self.path_escape(path) + def make_libpaths(self, libpaths): if not libpaths is None: - return ['/LIBPATH:' + self.path_escape(path) for path in list(libpaths)] + return ['/LIBPATH:' + self.make_libpath(path) for path in libpaths] return [] def make_arch_toolchain_path(self, arch): @@ -324,7 +316,7 @@ class MSVCToolchain(toolchain.Toolchain): libpaths += [os.path.join( self.sdkpath, 'lib', self.sdkversionpath, 'ucrt', 'x64')] return self.make_libpaths(libpaths) - def cc_variables(self, config, arch, targettype, variables, externalsources): + def cc_variables(self, config, arch, targettype, variables): localvariables = [('toolchain', self.make_arch_toolchain_path(arch))] if 'includepaths' in variables: moreincludepaths = self.make_includepaths(variables['includepaths']) @@ -338,8 +330,11 @@ class MSVCToolchain(toolchain.Toolchain): cconfigflags = self.make_cconfigflags(config, targettype) if cconfigflags != []: localvariables += [('cconfigflags', cconfigflags)] - if externalsources: - localvariables += [('cflags', self.cexternflags), ('cxxflags', self.cxxexternflags)] + if 'defines' in variables: + definelist = [] + for define in variables['defines']: + definelist += ['/D', '"' + define + '"'] + localvariables += [('cmoreflags', definelist)] return localvariables def ar_variables(self, config, arch, targettype, variables): @@ -372,19 +367,16 @@ class MSVCToolchain(toolchain.Toolchain): localvariables += [('configlibpaths', self.make_configlibpaths(config, arch, libpaths))] return localvariables - def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables, externalsources): - return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources)) + def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables): + return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables)) - def builder_cxx(self, writer, config, arch, targettype, infile, outfile, variables, externalsources): - return writer.build(outfile, 'cxx', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources)) - - def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables): return writer.build(outfile, 'ar', infiles, implicit = self.implicit_deps(config, variables), variables = self.ar_variables(config, arch, targettype, variables)) - def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables): return writer.build(outfile, 'dll', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables)) - def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources): + def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables): return writer.build(outfile, 'link', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables)) def create(host, target, toolchain): diff --git a/build/ninja/platform.py b/build/ninja/platform.py index 9280148..f831f3a 100644 --- a/build/ninja/platform.py +++ b/build/ninja/platform.py @@ -5,7 +5,7 @@ import sys def supported_platforms(): - return [ 'windows', 'linux', 'macosx', 'bsd', 'ios', 'android', 'raspberrypi', 'pnacl', 'tizen' ] + return [ 'windows', 'linux', 'macos', 'bsd', 'ios', 'android', 'raspberrypi', 'pnacl', 'tizen' ] class Platform(object): def __init__(self, platform): @@ -15,9 +15,9 @@ class Platform(object): if self.platform.startswith('linux'): self.platform = 'linux' elif self.platform.startswith('darwin'): - self.platform = 'macosx' + self.platform = 'macos' elif self.platform.startswith('macos'): - self.platform = 'macosx' + self.platform = 'macos' elif self.platform.startswith('win'): self.platform = 'windows' elif 'bsd' in self.platform: @@ -42,8 +42,8 @@ class Platform(object): def is_windows(self): return self.platform == 'windows' - def is_macosx(self): - return self.platform == 'macosx' + def is_macos(self): + return self.platform == 'macos' def is_bsd(self): return self.platform == 'bsd' @@ -62,6 +62,3 @@ class Platform(object): def is_tizen(self): return self.platform == 'tizen' - - def get(self): - return self.platform diff --git a/build/ninja/plist.py b/build/ninja/plist.py index 2398e29..55c5fec 100644 --- a/build/ninja/plist.py +++ b/build/ninja/plist.py @@ -56,9 +56,9 @@ if not options.exename: if not options.prodname: options.prodname = 'unknown' if not options.target: - options.target = 'macosx' + options.target = 'macos' if not options.deploymenttarget: - if options.target == 'macosx': + if options.target == 'macos': options.deploymenttarget = '10.7' else: options.deploymenttarget = '6.0' diff --git a/build/ninja/toolchain.py b/build/ninja/toolchain.py index 8e750a4..85afd4b 100644 --- a/build/ninja/toolchain.py +++ b/build/ninja/toolchain.py @@ -10,6 +10,7 @@ import random import string import json import zlib +import version import android import xcode @@ -42,11 +43,13 @@ class Toolchain(object): self.host = host self.target = target self.toolchain = toolchain + self.subninja = '' #Set default values self.build_monolithic = False self.build_coverage = False self.support_lua = False + self.internal_deps = False self.python = 'python' self.objext = '.o' if target.is_windows(): @@ -70,10 +73,7 @@ class Toolchain(object): else: self.libprefix = 'lib' self.staticlibext = '.a' - if target.is_macosx() or target.is_ios(): - self.dynamiclibext = '.dylib' - else: - self.dynamiclibext = '.so' + self.dynamiclibext = '.so' self.binprefix = '' self.binext = '' @@ -105,7 +105,7 @@ class Toolchain(object): #Target functionality if target.is_android(): self.android = android.make_target(self, host, target) - if target.is_macosx() or target.is_ios(): + if target.is_macos() or target.is_ios(): self.xcode = xcode.make_target(self, host, target) #Builders @@ -114,8 +114,12 @@ class Toolchain(object): #Paths created self.paths_created = {} + def initialize_subninja(self, path): + self.subninja = path + def initialize_project(self, project): self.project = project + version.generate_version(self.project, self.project) def initialize_archs(self, archs): self.archs = list(archs) @@ -133,7 +137,7 @@ class Toolchain(object): self.archs = ['x86'] else: self.archs = [localarch] - elif self.target.is_macosx(): + elif self.target.is_macos(): self.archs = ['x86-64'] elif self.target.is_ios(): self.archs = ['arm7', 'arm64'] @@ -152,7 +156,7 @@ class Toolchain(object): self.initialize_default_configs() def initialize_default_configs(self): - self.configs = ['debug', 'release'] + self.configs = ['debug', 'release', 'profile', 'deploy'] def initialize_toolchain(self): if self.android != None: @@ -161,9 +165,29 @@ class Toolchain(object): self.xcode.initialize_toolchain() def initialize_depends(self, dependlibs): - #TODO: Improve localization of dependend libs - self.depend_includepaths = [os.path.join('..', lib + '_lib') for lib in dependlibs] - self.depend_libpaths = [os.path.join('..', lib + '_lib') for lib in dependlibs] + for lib in dependlibs: + includepath = '' + libpath = '' + testpaths = [ + os.path.join('..', lib), + os.path.join('..', lib + '_lib') + ] + for testpath in testpaths: + if os.path.isfile(os.path.join(testpath, lib, lib + '.h')): + if self.subninja != '': + basepath, _ = os.path.split(self.subninja) + _, libpath = os.path.split(testpath) + testpath = os.path.join(basepath, libpath) + includepath = testpath + libpath = testpath + break + if includepath == '': + print("Unable to locate dependent lib: " + lib) + sys.exit(-1) + else: + self.depend_includepaths += [includepath] + if self.subninja == '': + self.depend_libpaths += [libpath] def build_toolchain(self): if self.android != None: @@ -185,6 +209,8 @@ class Toolchain(object): self.build_coverage = get_boolean_flag(val) elif key == 'support_lua': self.support_lua = get_boolean_flag(val) + elif key == 'internal_deps': + self.internal_deps = get_boolean_flag(val) if self.xcode != None: self.xcode.parse_default_variables(variables) @@ -256,6 +282,8 @@ class Toolchain(object): def mkdir(self, writer, path, implicit = None, order_only = None): if path in self.paths_created: return self.paths_created[path] + if self.subninja != '': + return cmd = writer.build(path, 'mkdir', None, implicit = implicit, order_only = order_only) self.paths_created[path] = cmd return cmd @@ -263,7 +291,7 @@ class Toolchain(object): def copy(self, writer, src, dst, implicit = None, order_only = None): return writer.build(dst, 'copy', src, implicit = implicit, order_only = order_only) - def builder_multicopy(self, writer, config, archs, targettype, infiles, outpath, variables, externalsources): + def builder_multicopy(self, writer, config, archs, targettype, infiles, outpath, variables): output = [] rootdir = self.mkdir(writer, outpath) for file in infiles: @@ -295,6 +323,16 @@ class Toolchain(object): def paths_forward_slash(self, paths): return [path.replace('\\', '/') for path in paths] + def prefix_includepath(self, path): + if os.path.isabs(path) or self.subninja == '': + return path + if path == '.': + return self.subninja + return os.path.join(self.subninja, path) + + def prefix_includepaths(self, includepaths): + return [self.prefix_includepath(path) for path in includepaths] + def list_per_config(self, config_dicts, config): if config_dicts is None: return [] @@ -310,29 +348,45 @@ class Toolchain(object): return self.list_per_config(variables['implicit_deps'], config) return None - def compile_file(self, writer, config, arch, targettype, infile, outfile, variables, externalsources): + def make_implicit_deps(self, outpath, arch, config, dependlibs): + deps = {} + deps[config] = [] + for lib in dependlibs: + if self.target.is_macos() or self.target.is_ios(): + finalpath = os.path.join(self.libpath, config, self.libprefix + lib + self.staticlibext) + else: + finalpath = os.path.join(self.libpath, config, arch, self.libprefix + lib + self.staticlibext) + deps[config] += [finalpath] + return [deps] + + def compile_file(self, writer, config, arch, targettype, infile, outfile, variables): extension = os.path.splitext(infile)[1][1:] if extension in self.builders: - return self.builders[extension](writer, config, arch, targettype, infile, outfile, variables, externalsources) + return self.builders[extension](writer, config, arch, targettype, infile, outfile, variables) return [] - def compile_node(self, writer, nodetype, config, arch, infiles, outfile, variables, externalsources): + def compile_node(self, writer, nodetype, config, arch, infiles, outfile, variables): if nodetype in self.builders: - return self.builders[nodetype](writer, config, arch, nodetype, infiles, outfile, variables, externalsources) + return self.builders[nodetype](writer, config, arch, nodetype, infiles, outfile, variables) return [] - def build_sources(self, writer, nodetype, multitype, module, sources, binfile, basepath, outpath, configs, includepaths, libpaths, libs, implicit_deps, variables, frameworks, externalsources): + def build_sources(self, writer, nodetype, multitype, module, sources, binfile, basepath, outpath, configs, includepaths, libpaths, dependlibs, libs, implicit_deps, variables, frameworks): if module != '': - decoratedmodule = module + make_pathhash(module, nodetype) + decoratedmodule = module + make_pathhash(self.subninja + module + binfile, nodetype) else: - decoratedmodule = basepath + make_pathhash(basepath, nodetype) + decoratedmodule = basepath + make_pathhash(self.subninja + basepath + binfile, nodetype) built = {} if includepaths is None: includepaths = [] if libpaths is None: libpaths = [] sourcevariables = (variables or {}).copy() - sourcevariables.update({'includepaths': self.depend_includepaths + list(includepaths)}) + sourcevariables.update({ + 'includepaths': self.depend_includepaths + self.prefix_includepaths(list(includepaths))}) + if not libs and dependlibs != None: + libs = [] + if dependlibs != None: + libs += (dependlibs or []) nodevariables = (variables or {}).copy() nodevariables.update({ 'libs': libs, @@ -350,47 +404,59 @@ class Toolchain(object): modulepath = os.path.join(buildpath, basepath, decoratedmodule) sourcevariables['modulepath'] = modulepath nodevariables['modulepath'] = modulepath + #Make per-arch-and-config list of final implicit deps, including dependent libs + if self.internal_deps and dependlibs != None: + dep_implicit_deps = [] + if implicit_deps: + dep_implicit_deps += implicit_deps + dep_implicit_deps += self.make_implicit_deps(outpath, arch, config, dependlibs) + nodevariables['implicit_deps'] = dep_implicit_deps #Compile all sources for name in sources: if os.path.isabs(name): infile = name - outfile = os.path.join(modulepath, os.path.splitext(os.path.basename(name))[0] + make_pathhash(binfile + infile, nodetype) + self.objext) + outfile = os.path.join(modulepath, os.path.splitext(os.path.basename(name))[0] + make_pathhash(infile, nodetype) + self.objext) else: infile = os.path.join(basepath, module, name) - outfile = os.path.join(modulepath, os.path.splitext(name)[0] + make_pathhash(binfile + infile, nodetype) + self.objext) - objs += self.compile_file(writer, config, arch, nodetype, infile, outfile, sourcevariables, externalsources) + outfile = os.path.join(modulepath, os.path.splitext(name)[0] + make_pathhash(infile, nodetype) + self.objext) + if self.subninja != '': + infile = os.path.join(self.subninja, infile) + objs += self.compile_file(writer, config, arch, nodetype, infile, outfile, sourcevariables) #Build arch node (per-config-and-arch binary) archoutpath = os.path.join(modulepath, binfile) - archnodes += self.compile_node(writer, nodetype, config, arch, objs, archoutpath, nodevariables, externalsources) + archnodes += self.compile_node(writer, nodetype, config, arch, objs, archoutpath, nodevariables) #Build final config node (per-config binary) - finalpath = os.path.join(outpath, config) - built[config] += self.compile_node(writer, multitype, config, self.archs, archnodes, finalpath, None, externalsources) + built[config] += self.compile_node(writer, multitype, config, self.archs, archnodes, os.path.join(outpath, config), None) writer.newline() return built - def lib(self, writer, module, sources, libname, basepath, configs, includepaths, variables, outpath, externalsources): + def lib(self, writer, module, sources, libname, basepath, configs, includepaths, variables, outpath = None): built = {} if basepath == None: basepath = '' if configs is None: configs = list(self.configs) + if libname is None: + libname = module libfile = self.libprefix + libname + self.staticlibext if outpath is None: outpath = self.libpath - return self.build_sources(writer, 'lib', 'multilib', module, sources, libfile, basepath, outpath, configs, includepaths, None, None, None, variables, None, externalsources) + return self.build_sources(writer, 'lib', 'multilib', module, sources, libfile, basepath, outpath, configs, includepaths, None, None, None, None, variables, None) - def sharedlib(self, writer, module, sources, libname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, outpath, externalsources): + def sharedlib(self, writer, module, sources, libname, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables, outpath = None): built = {} if basepath == None: basepath = '' if configs is None: configs = list(self.configs) + if libname is None: + libname = module libfile = self.libprefix + libname + self.dynamiclibext if outpath is None: outpath = self.binpath - return self.build_sources(writer, 'sharedlib', 'multisharedlib', module, sources, libfile, basepath, outpath, configs, includepaths, libpaths, libs, implicit_deps, variables, frameworks, externalsources) + return self.build_sources(writer, 'sharedlib', 'multisharedlib', module, sources, libfile, basepath, outpath, configs, includepaths, libpaths, dependlibs, libs, implicit_deps, variables, frameworks) - def bin(self, writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, outpath, externalsources): + def bin(self, writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables, outpath = None): built = {} if basepath == None: basepath = '' @@ -399,12 +465,12 @@ class Toolchain(object): binfile = self.binprefix + binname + self.binext if outpath is None: outpath = self.binpath - return self.build_sources(writer, 'bin', 'multibin', module, sources, binfile, basepath, outpath, configs, includepaths, libpaths, libs, implicit_deps, variables, frameworks, externalsources) + return self.build_sources(writer, 'bin', 'multibin', module, sources, binfile, basepath, outpath, configs, includepaths, libpaths, dependlibs, libs, implicit_deps, variables, frameworks) - def app(self, writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, resources): + def app(self, writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables, resources): builtbin = [] # Filter out platforms that do not have app concept - if not (self.target.is_macosx() or self.target.is_ios() or self.target.is_android() or self.target.is_tizen()): + if not (self.target.is_macos() or self.target.is_ios() or self.target.is_android() or self.target.is_tizen()): return builtbin if basepath is None: basepath = '' @@ -413,8 +479,8 @@ class Toolchain(object): if configs is None: configs = list(self.configs) for config in configs: - archbins = self.bin(writer, module, sources, binname, basepath, [config], includepaths, libpaths, implicit_deps, libs, frameworks, variables, '$buildpath') - if self.target.is_macosx() or self.target.is_ios(): + archbins = self.bin(writer, module, sources, binname, basepath, [config], includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables, '$buildpath') + if self.target.is_macos() or self.target.is_ios(): binpath = os.path.join(self.binpath, config, binname + '.app') builtbin += self.xcode.app(self, writer, module, archbins, self.binpath, binname, basepath, config, None, resources, True) if self.target.is_android(): diff --git a/build/ninja/version.py b/build/ninja/version.py index 6d1931f..983094d 100644 --- a/build/ninja/version.py +++ b/build/ninja/version.py @@ -70,7 +70,7 @@ def write_version_string(output_path, str): def generate_version(libname, output_path): generated = generate_version_string(libname) if generated == None: - return + return previous = read_version_string(output_path) if generated != previous: diff --git a/build/ninja/xcode.py b/build/ninja/xcode.py index 3abc957..8e158d9 100644 --- a/build/ninja/xcode.py +++ b/build/ninja/xcode.py @@ -21,13 +21,13 @@ class XCode(object): self.organisation = '' self.bundleidentifier = '' self.provisioning = '' - if self.target.is_macosx(): + if self.target.is_macos(): self.deploymenttarget = '10.7' elif self.target.is_ios(): self.deploymenttarget = '8.0' def build_toolchain(self): - if self.target.is_macosx(): + if self.target.is_macos(): sdk = 'macosx' deploytarget = 'MACOSX_DEPLOYMENT_TARGET=' + self.deploymenttarget elif self.target.is_ios(): @@ -43,7 +43,7 @@ class XCode(object): self.dsymutil = "PATH=" + localpath + " " + subprocess.check_output(['xcrun', '--sdk', sdk, '-f', 'dsymutil']).strip() self.plistcmd = 'build/ninja/plist.py --exename $exename --prodname $prodname --bundle $bundleidentifier --target $target --deploymenttarget $deploymenttarget --output $outpath $in' - if self.target.is_macosx(): + if self.target.is_macos(): self.xcassetscmd = 'mkdir -p $outpath && $xcassets --output-format human-readable-text --output-partial-info-plist $outplist' \ ' --app-icon AppIcon --launch-image LaunchImage --platform macosx --minimum-deployment-target ' + self.deploymenttarget + \ ' --target-device mac --compress-pngs --compile $outpath $in >/dev/null' @@ -88,16 +88,16 @@ class XCode(object): self.bundleidentifier = iosprefs['bundleidentifier'] if 'provisioning' in iosprefs: self.provisioning = iosprefs['provisioning'] - elif self.target.is_macosx() and 'macosx' in prefs: - macosxprefs = prefs['macosx'] - if 'deploymenttarget' in macosxprefs: - self.deploymenttarget = macosxprefs['deploymenttarget'] - if 'organisation' in macosxprefs: - self.organisation = macosxprefs['organisation'] - if 'bundleidentifier' in macosxprefs: - self.bundleidentifier = macosxprefs['bundleidentifier'] - if 'provisioning' in macosxprefs: - self.provisioning = macosxprefs['provisioning'] + elif self.target.is_macos() and 'macos' in prefs: + macosprefs = prefs['macos'] + if 'deploymenttarget' in macosprefs: + self.deploymenttarget = macosprefs['deploymenttarget'] + if 'organisation' in macosprefs: + self.organisation = macosprefs['organisation'] + if 'bundleidentifier' in macosprefs: + self.bundleidentifier = macosprefs['bundleidentifier'] + if 'provisioning' in macosprefs: + self.provisioning = macosprefs['provisioning'] def write_variables(self, writer): writer.variable('plist', self.plist) @@ -154,14 +154,14 @@ class XCode(object): #First build everything except plist inputs for resource in resources: if resource.endswith('.xcassets'): - if self.target.is_macosx(): + if self.target.is_macos(): assetsvars = [('outpath', os.path.join(os.getcwd(), apppath, 'Contents', 'Resources'))] else: assetsvars = [('outpath', apppath)] outplist = os.path.join(os.getcwd(), builddir, os.path.splitext(os.path.basename(resource))[0] + '-xcassets.plist') assetsvars += [('outplist', outplist)] outfiles = [outplist] - if self.target.is_macosx(): + if self.target.is_macos(): outfiles += [os.path.join(os.getcwd(), apppath, 'Contents', 'Resources', 'AppIcon.icns')] elif self.target.is_ios(): pass #TODO: Need to list all icon and launch image files here @@ -169,7 +169,7 @@ class XCode(object): has_resources = True elif resource.endswith('.xib'): xibmodule = binname.replace('-', '_').replace('.', '_') - if self.target.is_macosx(): + if self.target.is_macos(): nibpath = os.path.join(apppath, 'Contents', 'Resources', os.path.splitext(os.path.basename(resource))[0] + '.nib') else: nibpath = os.path.join(apppath, os.path.splitext(os.path.basename(resource))[0] + '.nib') @@ -186,11 +186,11 @@ class XCode(object): #Extra output files/directories outfiles = [] - if has_resources and self.target.is_macosx(): + if has_resources and self.target.is_macos(): outfiles += [os.path.join(apppath, 'Contents', 'Resources')] #Now build input plists appending partial plists created by previous resources - if self.target.is_macosx(): + if self.target.is_macos(): plistpath = os.path.join(apppath, 'Contents', 'Info.plist') pkginfopath = os.path.join(apppath, 'Contents', 'PkgInfo') else: @@ -210,7 +210,7 @@ class XCode(object): if self.provisioning != '': codesignvars += [('provisioning', self.provisioning)] writer.build([os.path.join(apppath, '_CodeSignature', 'CodeResources'), os.path.join(apppath, '_CodeSignature'), apppath], 'codesign', builtbin, implicit = builtres + [os.path.join('build', 'ninja', 'codesign.py')], variables = codesignvars) - elif self.target.is_macosx(): + elif self.target.is_macos(): if self.provisioning != '': codesignvars += [('provisioning', self.provisioning)] writer.build([os.path.join(apppath, 'Contents', '_CodeSignature', 'CodeResources'), os.path.join(apppath, 'Contents', '_CodeSignature'), os.path.join(apppath, 'Contents'), apppath], 'codesign', builtbin, implicit = builtres + [os.path.join('build', 'ninja', 'codesign.py')], variables = codesignvars) diff --git a/configure.py b/configure.py index 1daf2df..ac138e9 100755 --- a/configure.py +++ b/configure.py @@ -19,3 +19,9 @@ rpmallocwrap_lib = generator.lib(module = 'rpmalloc', libname = 'rpmallocwrap', if not target.is_windows(): rpmalloc_so = generator.sharedlib(module = 'rpmalloc', libname = 'rpmalloc', sources = ['rpmalloc.c']) rpmallocwrap_so = generator.sharedlib(module = 'rpmalloc', libname = 'rpmallocwrap', sources = ['rpmalloc.c', 'malloc.c', 'new.cc'], variables = {'runtime': 'c++', 'defines': ['ENABLE_PRELOAD=1']}) + +rpmallocguards_lib = generator.lib(module = 'rpmalloc', libname = 'rpmallocguards', sources = ['rpmalloc.c'], variables = {'defines': ['ENABLE_GUARDS=1']}) + +if not target.is_ios() and not target.is_android(): + generator.bin(module = 'test', sources = ['thread.c', 'main.c'], binname = 'rpmalloc-test', implicit_deps = [rpmalloc_lib], libs = ['rpmalloc'], includepaths = ['rpmalloc', 'test']) + generator.bin(module = 'test', sources = ['thread.c', 'main.c'], binname = 'rpmalloc-test-guards', implicit_deps = [rpmallocguards_lib], libs = ['rpmallocguards'], includepaths = ['rpmalloc', 'test'], variables = {'defines': ['ENABLE_GUARDS=1']}) diff --git a/rpmalloc/rpmalloc.c b/rpmalloc/rpmalloc.c index 69f9da9..43fff8b 100644 --- a/rpmalloc/rpmalloc.c +++ b/rpmalloc/rpmalloc.c @@ -118,7 +118,7 @@ # include #else # undef assert -# define assert(x) +# define assert(x) do {} while(0) #endif #if ENABLE_GUARDS @@ -1423,6 +1423,7 @@ _memory_adjust_size_class(size_t iclass) { //! Initialize the allocator and setup global data int rpmalloc_initialize(void) { + memset(&_memory_config, 0, sizeof(rpmalloc_config_t)); return rpmalloc_initialize_config(0); } @@ -1572,6 +1573,8 @@ rpmalloc_finalize(void) { atomic_thread_fence_release(); + set_thread_heap(0); + #if defined(__APPLE__) && ENABLE_PRELOAD pthread_key_delete(_memory_thread_heap); #endif @@ -1791,12 +1794,24 @@ _memory_validate_integrity(void* p) { uint32_t* deadzone = block_start; //If these asserts fire, you have written to memory before the block start for (int i = 0; i < 4; ++i) { - assert(deadzone[i] == MAGIC_GUARD); + if (deadzone[i] == MAGIC_GUARD) + continue; + if (_memory_config.memory_overwrite) + _memory_config.memory_overwrite(p); + else + assert(deadzone[i] == MAGIC_GUARD && "Memory overwrite before block start"); + return; } deadzone = (uint32_t*)pointer_offset(block_start, block_size - 16); //If these asserts fire, you have written to memory after the block end for (int i = 0; i < 4; ++i) { - assert(deadzone[i] == MAGIC_GUARD); + if (deadzone[i] == MAGIC_GUARD) + continue; + if (_memory_config.memory_overwrite) + _memory_config.memory_overwrite(p); + else + assert(deadzone[i] == MAGIC_GUARD && "Memory overwrite after block end"); + return; } } #endif @@ -1944,7 +1959,14 @@ rpposix_memalign(void **memptr, size_t alignment, size_t size) { size_t rpmalloc_usable_size(void* ptr) { - return ptr ? _memory_usable_size(ptr) : 0; + size_t size = 0; + if (ptr) { + size = _memory_usable_size(ptr); +#if ENABLE_GUARDS + size -= 32; +#endif + } + return size; } void diff --git a/rpmalloc/rpmalloc.h b/rpmalloc/rpmalloc.h index 88ed1dc..52ab2d3 100644 --- a/rpmalloc/rpmalloc.h +++ b/rpmalloc/rpmalloc.h @@ -79,6 +79,8 @@ typedef struct rpmalloc_config_t { // size. If set to 0, rpmalloc will use system calls to determine the page size. The page // size MUST be a power of two in [512,16384] range (2^9 to 2^14). size_t page_size; + //! Debug callback if memory guards are enabled. Called if a memory overwrite is detected + void (*memory_overwrite)(void* address); } rpmalloc_config_t; extern int diff --git a/test/main.c b/test/main.c new file mode 100644 index 0000000..a1b571d --- /dev/null +++ b/test/main.c @@ -0,0 +1,573 @@ + +#if defined(_WIN32) && !defined(_CRT_SECURE_NO_WARNINGS) +# define _CRT_SECURE_NO_WARNINGS +#endif + +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +# define PRIsize "Iu" +#else +# define PRIsize "zu" +#endif + +#define pointer_offset(ptr, ofs) (void*)((char*)(ptr) + (ptrdiff_t)(ofs)) +#define pointer_diff(first, second) (ptrdiff_t)((const char*)(first) - (const char*)(second)) + +#ifndef ENABLE_GUARDS +# define ENABLE_GUARDS 0 +#endif + +static int +test_alloc(void) { + unsigned int iloop = 0; + unsigned int ipass = 0; + unsigned int icheck = 0; + unsigned int id = 0; + void* addr[8142]; + char data[20000]; + unsigned int datasize[7] = { 473, 39, 195, 24, 73, 376, 245 }; + + rpmalloc_initialize(); + + for (id = 0; id < 20000; ++id) + data[id] = (char)(id % 139 + id % 17); + + for (iloop = 0; iloop < 64; ++iloop) { + for (ipass = 0; ipass < 8142; ++ipass) { + addr[ipass] = rpmalloc(500); + if (addr[ipass] == 0) + return -1; + + memcpy(addr[ipass], data, 500); + + for (icheck = 0; icheck < ipass; ++icheck) { + if (addr[icheck] == addr[ipass]) + return -1; + if (addr[icheck] < addr[ipass]) { + if (pointer_offset(addr[icheck], 500) > addr[ipass]) + return -1; + } + else if (addr[icheck] > addr[ipass]) { + if (pointer_offset(addr[ipass], 500) > addr[icheck]) + return -1; + } + } + } + + for (ipass = 0; ipass < 8142; ++ipass) { + if (memcmp(addr[ipass], data, 500)) + return -1; + } + + for (ipass = 0; ipass < 8142; ++ipass) + rpfree(addr[ipass]); + } + + for (iloop = 0; iloop < 64; ++iloop) { + for (ipass = 0; ipass < 1024; ++ipass) { + unsigned int cursize = datasize[ipass%7] + ipass; + + addr[ipass] = rpmalloc(cursize); + if (addr[ipass] == 0) + return -1; + + memcpy(addr[ipass], data, cursize); + + for (icheck = 0; icheck < ipass; ++icheck) { + if (addr[icheck] == addr[ipass]) + return -1; + if (addr[icheck] < addr[ipass]) { + if (pointer_offset(addr[icheck], rpmalloc_usable_size(addr[icheck])) > addr[ipass]) + return -1; + } + else if (addr[icheck] > addr[ipass]) { + if (pointer_offset(addr[ipass], rpmalloc_usable_size(addr[ipass])) > addr[icheck]) + return -1; + } + } + } + + for (ipass = 0; ipass < 1024; ++ipass) { + unsigned int cursize = datasize[ipass%7] + ipass; + if (memcmp(addr[ipass], data, cursize)) + return -1; + } + + for (ipass = 0; ipass < 1024; ++ipass) + rpfree(addr[ipass]); + } + + for (iloop = 0; iloop < 128; ++iloop) { + for (ipass = 0; ipass < 1024; ++ipass) { + addr[ipass] = rpmalloc(500); + if (addr[ipass] == 0) + return -1; + + memcpy(addr[ipass], data, 500); + + for (icheck = 0; icheck < ipass; ++icheck) { + if (addr[icheck] == addr[ipass]) + return -1; + if (addr[icheck] < addr[ipass]) { + if (pointer_offset(addr[icheck], 500) > addr[ipass]) + return -1; + } + else if (addr[icheck] > addr[ipass]) { + if (pointer_offset(addr[ipass], 500) > addr[icheck]) + return -1; + } + } + } + + for (ipass = 0; ipass < 1024; ++ipass) { + if (memcmp(addr[ipass], data, 500)) + return -1; + } + + for (ipass = 0; ipass < 1024; ++ipass) + rpfree(addr[ipass]); + } + + rpmalloc_finalize(); + + printf("Memory allocation tests passed\n"); + + return 0; +} + +typedef struct _allocator_thread_arg { + unsigned int loops; + unsigned int passes; //max 4096 + unsigned int datasize[32]; + unsigned int num_datasize; //max 32 + void** pointers; +} allocator_thread_arg_t; + +static void +allocator_thread(void* argp) { + allocator_thread_arg_t arg = *(allocator_thread_arg_t*)argp; + unsigned int iloop = 0; + unsigned int ipass = 0; + unsigned int icheck = 0; + unsigned int id = 0; + void* addr[4096]; + char data[8192]; + unsigned int cursize; + unsigned int iwait = 0; + int ret = 0; + + rpmalloc_thread_initialize(); + + for (id = 0; id < 8192; ++id) + data[id] = (char)id; + + thread_sleep(1); + + for (iloop = 0; iloop < arg.loops; ++iloop) { + for (ipass = 0; ipass < arg.passes; ++ipass) { + cursize = 4 + arg.datasize[(iloop + ipass + iwait) % arg.num_datasize] + (iloop % 1024); + + addr[ipass] = rpmalloc(4 + cursize); + if (addr[ipass] == 0) { + ret = -1; + goto end; + } + + *(uint32_t*)addr[ipass] = (uint32_t)cursize; + memcpy(pointer_offset(addr[ipass], 4), data, cursize); + + for (icheck = 0; icheck < ipass; ++icheck) { + if (addr[icheck] == addr[ipass]) { + ret = -1; + goto end; + } + if (addr[icheck] < addr[ipass]) { + if (pointer_offset(addr[icheck], *(uint32_t*)addr[icheck]) > addr[ipass]) { + if (pointer_offset(addr[icheck], *(uint32_t*)addr[icheck]) > addr[ipass]) { + ret = -1; + goto end; + } + } + } + else if (addr[icheck] > addr[ipass]) { + if (pointer_offset(addr[ipass], *(uint32_t*)addr[ipass]) > addr[ipass]) { + if (pointer_offset(addr[ipass], *(uint32_t*)addr[ipass]) > addr[icheck]) { + ret = -1; + goto end; + } + } + } + } + } + + for (ipass = 0; ipass < arg.passes; ++ipass) { + cursize = *(uint32_t*)addr[ipass]; + + if (memcmp(pointer_offset(addr[ipass], 4), data, cursize)) { + ret = -1; + goto end; + } + + rpfree(addr[ipass]); + } + } + + rpmalloc_thread_finalize(); + +end: + thread_exit((uintptr_t)ret); +} + +static void +crossallocator_thread(void* argp) { + allocator_thread_arg_t arg = *(allocator_thread_arg_t*)argp; + unsigned int iloop = 0; + unsigned int ipass = 0; + unsigned int cursize; + unsigned int iwait = 0; + int ret = 0; + + rpmalloc_thread_initialize(); + + thread_sleep(1); + + for (iloop = 0; iloop < arg.loops; ++iloop) { + for (ipass = 0; ipass < arg.passes; ++ipass) { + cursize = arg.datasize[(iloop + ipass + iwait) % arg.num_datasize ] + (iloop % 1024); + + void* addr = rpmalloc(cursize); + if (addr == 0) { + ret = -1; + goto end; + } + + arg.pointers[iloop * arg.passes + ipass] = addr; + } + } + + rpmalloc_thread_finalize(); + +end: + thread_exit((uintptr_t)ret); +} + +static void +initfini_thread(void* argp) { + allocator_thread_arg_t arg = *(allocator_thread_arg_t*)argp; + unsigned int iloop = 0; + unsigned int ipass = 0; + unsigned int icheck = 0; + unsigned int id = 0; + void* addr[4096]; + char data[8192]; + unsigned int cursize; + unsigned int iwait = 0; + int ret = 0; + + for (id = 0; id < 8192; ++id) + data[id] = (char)id; + + thread_yield(); + + for (iloop = 0; iloop < arg.loops; ++iloop) { + rpmalloc_thread_initialize(); + + for (ipass = 0; ipass < arg.passes; ++ipass) { + cursize = 4 + arg.datasize[(iloop + ipass + iwait) % arg.num_datasize] + (iloop % 1024); + + addr[ipass] = rpmalloc(4 + cursize); + if (addr[ipass] == 0) { + ret = -1; + goto end; + } + + *(uint32_t*)addr[ipass] = (uint32_t)cursize; + memcpy(pointer_offset(addr[ipass], 4), data, cursize); + + for (icheck = 0; icheck < ipass; ++icheck) { + if (addr[icheck] == addr[ipass]) { + ret = -1; + goto end; + } + if (addr[icheck] < addr[ipass]) { + if (pointer_offset(addr[icheck], *(uint32_t*)addr[icheck]) > addr[ipass]) { + if (pointer_offset(addr[icheck], *(uint32_t*)addr[icheck]) > addr[ipass]) { + ret = -1; + goto end; + } + } + } + else if (addr[icheck] > addr[ipass]) { + if (pointer_offset(addr[ipass], *(uint32_t*)addr[ipass]) > addr[ipass]) { + if (pointer_offset(addr[ipass], *(uint32_t*)addr[ipass]) > addr[icheck]) { + ret = -1; + goto end; + } + } + } + } + } + + for (ipass = 0; ipass < arg.passes; ++ipass) { + cursize = *(uint32_t*)addr[ipass]; + + if (memcmp(pointer_offset(addr[ipass], 4), data, cursize)) { + ret = -1; + goto end; + } + + rpfree(addr[ipass]); + } + + rpmalloc_thread_finalize(); + } + +end: + rpmalloc_thread_finalize(); + thread_exit((uintptr_t)ret); +} + +static int +test_threaded(void) { + uintptr_t thread[32]; + uintptr_t threadres[32]; + unsigned int i; + size_t num_alloc_threads; + allocator_thread_arg_t arg; + + rpmalloc_initialize(); + + num_alloc_threads = 3; + + arg.datasize[0] = 19; + arg.datasize[1] = 249; + arg.datasize[2] = 797; + arg.datasize[3] = 3; + arg.datasize[4] = 79; + arg.datasize[5] = 34; + arg.datasize[6] = 389; + arg.num_datasize = 7; + arg.loops = 4096; + arg.passes = 1024; + + thread_arg targ = { allocator_thread, &arg }; + for (i = 0; i < num_alloc_threads; ++i) + thread[i] = thread_run(&targ); + + thread_sleep(1000); + + for (i = 0; i < num_alloc_threads; ++i) + threadres[i] = thread_join(thread[i]); + + rpmalloc_finalize(); + + for (i = 0; i < num_alloc_threads; ++i) { + if (threadres[i]) + return -1; + } + + printf("Memory threaded tests passed\n"); + + return 0; +} + +static int +test_crossthread(void) { + uintptr_t thread; + allocator_thread_arg_t arg; + + rpmalloc_initialize(); + + arg.loops = 100; + arg.passes = 1024; + arg.pointers = rpmalloc(sizeof(void*) * arg.loops * arg.passes); + arg.datasize[0] = 19; + arg.datasize[1] = 249; + arg.datasize[2] = 797; + arg.datasize[3] = 3; + arg.datasize[4] = 79; + arg.datasize[5] = 34; + arg.datasize[6] = 389; + arg.num_datasize = 7; + + thread_arg targ = { crossallocator_thread, &arg }; + thread = thread_run(&targ); + + thread_sleep(1000); + + if (thread_join(thread) != 0) + return -1; + + //Off-thread deallocation + for (size_t iptr = 0; iptr < arg.loops * arg.passes; ++iptr) + rpfree(arg.pointers[iptr]); + + rpfree(arg.pointers); + + //Simulate thread exit + rpmalloc_thread_finalize(); + + rpmalloc_finalize(); + + printf("Memory cross thread free tests passed\n"); + + return 0; +} + +static int +test_threadspam(void) { + uintptr_t thread[64]; + uintptr_t threadres[64]; + unsigned int i, j; + size_t num_passes, num_alloc_threads; + allocator_thread_arg_t arg; + + rpmalloc_initialize(); + + num_passes = 100; + num_alloc_threads = 5; + + arg.loops = 500; + arg.passes = 10; + arg.datasize[0] = 19; + arg.datasize[1] = 249; + arg.datasize[2] = 797; + arg.datasize[3] = 3; + arg.datasize[4] = 79; + arg.datasize[5] = 34; + arg.datasize[6] = 389; + arg.num_datasize = 7; + + thread_arg targ = { initfini_thread, &arg }; + for (i = 0; i < num_alloc_threads; ++i) + thread[i] = thread_run(&targ); + + for (j = 0; j < num_passes; ++j) { + thread_sleep(10); + thread_fence(); + + for (i = 0; i < num_alloc_threads; ++i) { + threadres[i] = thread_join(thread[i]); + if (threadres[i]) + return -1; + thread[i] = thread_run(&targ); + } + } + + thread_sleep(1000); + + for (i = 0; i < num_alloc_threads; ++i) + threadres[i] = thread_join(thread[i]); + + rpmalloc_finalize(); + + for (i = 0; i < num_alloc_threads; ++i) { + if (threadres[i]) + return -1; + } + + printf("Memory thread spam tests passed\n"); + + return 0; +} + +#if ENABLE_GUARDS + +static int test_overwrite_detected; + +static void +test_overwrite_cb(void* addr) { + ++test_overwrite_detected; +} + +static int +test_overwrite(void) { + int ret = 0; + char* addr; + size_t istep, size; + + rpmalloc_config_t config; + memset(&config, 0, sizeof(config)); + config.memory_overwrite = test_overwrite_cb; + + rpmalloc_initialize_config(&config); + + for (istep = 0, size = 16; size < 16 * 1024 * 1024; size <<= 1, ++istep) { + test_overwrite_detected = 0; + addr = rpmalloc(size); + *(addr - 2) = 1; + rpfree(addr); + + if (!test_overwrite_detected) { + printf("Failed to detect memory overwrite before start of block in step %" PRIsize " size %" PRIsize "\n", istep, size); + ret = -1; + goto cleanup; + } + + test_overwrite_detected = 0; + addr = rpmalloc(size); + *(addr + rpmalloc_usable_size(addr) + 1) = 1; + rpfree(addr); + + if (!test_overwrite_detected) { + printf("Failed to detect memory overwrite after end of block in step %" PRIsize " size %" PRIsize "\n", istep, size); + ret = -1; + goto cleanup; + } + } + + printf("Memory overwrite tests passed\n"); + +cleanup: + rpmalloc_finalize(); + return ret; +} + +#else + +static int +test_overwrite(void) { + return 0; +} + +#endif + +int +test_run(int argc, char** argv) { + if (test_alloc()) + return -1; + if (test_threaded()) + return -1; + if (test_crossthread()) + return -1; + if (test_threadspam()) + return -1; + if (test_overwrite()) + return -1; + return 0; +} + +#if ( defined( __APPLE__ ) && __APPLE__ ) +# include +# if defined( __IPHONE__ ) || ( defined( TARGET_OS_IPHONE ) && TARGET_OS_IPHONE ) || ( defined( TARGET_IPHONE_SIMULATOR ) && TARGET_IPHONE_SIMULATOR ) +# define NO_MAIN 1 +# endif +#endif + +#if !defined(NO_MAIN) + +int +main(int argc, char** argv) { + return test_run(argc, argv); +} + +#endif diff --git a/test/test.h b/test/test.h new file mode 100644 index 0000000..b65fb46 --- /dev/null +++ b/test/test.h @@ -0,0 +1,5 @@ + +#pragma once + +extern int +test_run(int argc, char** argv); diff --git a/test/thread.c b/test/thread.c new file mode 100644 index 0000000..13e6c97 --- /dev/null +++ b/test/thread.c @@ -0,0 +1,106 @@ + +#include + +#ifdef _MSC_VER +# define ATTRIBUTE_NORETURN +#else +# define ATTRIBUTE_NORETURN __attribute__((noreturn)) +#endif + +#ifdef _WIN32 +# include +# include + +static unsigned __stdcall +thread_entry(void* argptr) { + thread_arg* arg = argptr; + arg->fn(arg->arg); + return 0; +} + +#else +# include +# include +# include + +static void* +thread_entry(void* argptr) { + thread_arg* arg = argptr; + arg->fn(arg->arg); + return 0; +} + +#endif + +uintptr_t +thread_run(thread_arg* arg) { +#ifdef _WIN32 + return _beginthreadex(0, 0, thread_entry, arg, 0, 0); +#else + pthread_t id = 0; + int err = pthread_create(&id, 0, thread_entry, arg); + if (err) + return 0; + return (uintptr_t)id; +#endif +} + +void ATTRIBUTE_NORETURN +thread_exit(uintptr_t value) { +#ifdef _WIN32 + _endthreadex((unsigned)value); +#else + pthread_exit((void*)value); +#endif +} + +uintptr_t +thread_join(uintptr_t handle) { + if (!handle) + return (uintptr_t)-1; + uintptr_t ret; +#ifdef _WIN32 + DWORD exit_code = 0; + WaitForSingleObject((HANDLE)handle, INFINITE); + GetExitCodeThread((HANDLE)handle, &exit_code); + CloseHandle((HANDLE)handle); + ret = exit_code; +#else + void* result = 0; + pthread_join((pthread_t)handle, &result); + ret = (uintptr_t)result; +#endif + return ret; +} + +void +thread_sleep(int milliseconds) { +#ifdef _WIN32 + SleepEx(milliseconds, 1); +#else + struct timespec ts; + ts.tv_sec = milliseconds / 1000; + ts.tv_nsec = (long)(milliseconds % 1000) * 1000000L; + nanosleep(&ts, 0); +#endif +} + +void +thread_yield(void) { +#ifdef _WIN32 + Sleep(0); + _ReadWriteBarrier(); +#else + sched_yield(); + __sync_synchronize(); +#endif +} + +void +thread_fence(void) { +#ifdef _WIN32 + _ReadWriteBarrier(); +#else + __sync_synchronize(); +#endif +} diff --git a/test/thread.h b/test/thread.h new file mode 100644 index 0000000..49ce1a5 --- /dev/null +++ b/test/thread.h @@ -0,0 +1,27 @@ + +#include + + +struct thread_arg { + void (*fn)(void*); + void* arg; +}; +typedef struct thread_arg thread_arg; + +extern uintptr_t +thread_run(thread_arg* arg); + +extern void +thread_exit(uintptr_t value); + +extern uintptr_t +thread_join(uintptr_t handle); + +extern void +thread_sleep(int milliseconds); + +extern void +thread_yield(void); + +extern void +thread_fence(void);