Mirrors/rpmalloc
0
0
Fork 0
mirror of https://github.com/mjansson/rpmalloc.git synced 2026-01-15 19:41:17 +01:00
Code Issues Packages Projects Releases Wiki Activity

Test cases (#45)

Browse Source
This commit is contained in:
Mattias Jansson
2018-01-23 10:06:06 +01:00
committed by GitHub
parent 504dedc22c
commit 7eb62ee892
20 changed files with 1074 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitattributes vendored
View File

@@ -21,6 +21,7 @@
*.rtf diff=astextplain
*.RTF diff=astextplain
# Ignore build scripts in language stats
# Ignore build scripts and test cases in language stats
build/* linguist-vendored
configure.py linguist-vendored=true
test/* linguist-vendored

4
README.md
View File

@@ -98,7 +98,9 @@ The functions do not need to deal with alignment, this is done by rpmalloc inter
Memory mapping requests are always done in multiples of the memory page size. You can specify a custom page size when initializing rpmalloc with __rpmalloc_initialize_config__, or pass 0 to let rpmalloc determine the system memory page size using OS APIs. The page size MUST be a power of two in [512, 16384] range.
# Memory guards
If you define the __ENABLE_GUARDS__ to 1, all memory allocations will be padded with extra guard areas before and after the memory block (while still honoring the requested alignment). These dead zones will be filled with a pattern and checked when the block is freed. If the patterns are not intact, an assert is fired.
If you define the __ENABLE_GUARDS__ to 1, all memory allocations will be padded with extra guard areas before and after the memory block (while still honoring the requested alignment). These dead zones will be filled with a pattern and checked when the block is freed. If the patterns are not intact the callback set in initialization config is called, or if not set an assert is fired.
Note that the end of the memory block in this case is defined by the total usable size of the block as returned by `rpmalloc_usable_size`, which can be larger than the size passed to allocation request due to size class buckets.
# Memory fragmentation
There is no memory fragmentation by the allocator in the sense that it will not leave unallocated and unusable "holes" in the memory pages by calls to allocate and free blocks of different sizes. This is due to the fact that the memory pages allocated for each size class is split up in perfectly aligned blocks which are not reused for a request of a different size. The block freed by a call to `rpfree` will always be immediately available for an allocation request within the same size class.

2
build/ninja/android.py
View File

@@ -86,7 +86,7 @@ class Android(object):
self.hostarchname = 'linux-x86_64'
else:
self.hostarchname = 'linux-x86'
elif self.host.is_macosx():
elif self.host.is_macos():
self.hostarchname = 'darwin-x86_64'
def build_toolchain(self):

158
build/ninja/clang.py
View File

@@ -9,11 +9,11 @@ import toolchain
class ClangToolchain(toolchain.Toolchain):
def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables):
def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables, subninja):
#Local variable defaults
self.toolchain = ''
self.sdkpath = ''
self.includepaths = includepaths
self.includepaths = []
self.libpaths = libpaths
self.ccompiler = 'clang'
self.cxxcompiler = 'clang++'
@@ -27,16 +27,16 @@ class ClangToolchain(toolchain.Toolchain):
self.sysroot = ''
if self.target.is_ios():
self.deploymenttarget = '9.0'
if self.target.is_macosx():
if self.target.is_macos():
self.deploymenttarget = '10.7'
#Command definitions
self.cccmd = '$toolchain$cc -MMD -MT $out -MF $out.d -I. $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags -c $in -o $out'
self.cxxcmd = '$toolchain$cxx -MMD -MT $out -MF $out.d -I. $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags $cmoreflags -c $in -o $out'
self.cccmd = '$toolchain$cc -MMD -MT $out -MF $out.d $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags -c $in -o $out'
self.cxxcmd = '$toolchain$cxx -MMD -MT $out -MF $out.d $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags $cmoreflags -c $in -o $out'
self.ccdeps = 'gcc'
self.ccdepfile = '$out.d'
self.arcmd = self.rmcmd('$out') + ' && $toolchain$ar crsD $ararchflags $arflags $out $in'
self.linkcmd = '$toolchain$link $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags -o $out $in $libs $archlibs $oslibs $frameworks'
self.linkcmd = '$toolchain$cc $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags -o $out $in $libs $archlibs $oslibs $frameworks'
#Base flags
self.cflags = ['-D' + project.upper() + '_COMPILE=1',
@@ -45,12 +45,23 @@ class ClangToolchain(toolchain.Toolchain):
'-fno-trapping-math', '-ffast-math']
self.cwarnflags = ['-W', '-Werror', '-pedantic', '-Wall', '-Weverything',
'-Wno-padded', '-Wno-documentation-unknown-command']
self.cmoreflags = []
self.mflags = []
self.arflags = []
self.linkflags = []
self.oslibs = []
self.frameworks = []
self.initialize_subninja(subninja)
self.initialize_archs(archs)
self.initialize_configs(configs)
self.initialize_project(project)
self.initialize_toolchain()
self.initialize_depends(dependlibs)
self.parse_default_variables(variables)
self.read_build_prefs()
if self.target.is_linux() or self.target.is_bsd() or self.target.is_raspberrypi():
self.cflags += ['-D_GNU_SOURCE=1']
self.linkflags += ['-pthread']
@@ -60,14 +71,7 @@ class ClangToolchain(toolchain.Toolchain):
if self.target.is_bsd():
self.oslibs += ['execinfo']
self.initialize_archs(archs)
self.initialize_configs(configs)
self.initialize_project(project)
self.initialize_toolchain()
self.initialize_depends(dependlibs)
self.parse_default_variables(variables)
self.read_build_prefs()
self.includepaths = self.prefix_includepaths((includepaths or []) + ['.'])
if self.is_monolithic():
self.cflags += ['-DBUILD_MONOLITHIC=1']
@@ -75,6 +79,16 @@ class ClangToolchain(toolchain.Toolchain):
self.cflags += ['--coverage']
self.linkflags += ['--coverage']
if not 'nowarning' in variables or not variables['nowarning']:
self.cflags += self.cwarnflags
self.cxxflags = list(self.cflags)
self.cflags += ['-std=c11']
if self.target.is_macos() or self.target.is_ios():
self.cxxflags += ['-std=c++14', '-stdlib=libc++']
else:
self.cxxflags += ['-std=gnu++14']
#Overrides
self.objext = '.o'
@@ -85,7 +99,7 @@ class ClangToolchain(toolchain.Toolchain):
self.builders['lib'] = self.builder_lib
self.builders['sharedlib'] = self.builder_sharedlib
self.builders['bin'] = self.builder_bin
if self.target.is_macosx() or self.target.is_ios():
if self.target.is_macos() or self.target.is_ios():
self.builders['m'] = self.builder_cm
self.builders['multilib'] = self.builder_apple_multilib
self.builders['multisharedlib'] = self.builder_apple_multisharedlib
@@ -102,21 +116,6 @@ class ClangToolchain(toolchain.Toolchain):
#Setup target platform
self.build_toolchain()
self.cflags += ['-std=c11']
if self.target.is_macosx() or self.target.is_ios():
self.cxxflags += ['-std=c++11', '-stdlib=libc++']
if self.target.is_linux():
self.cxxflags += ['-std=gnu++11']
self.cmoreflags = []
self.cexternflags = []
self.cxxexternflags = []
self.cexternflags += self.cflags + ['-w']
self.cxxexternflags += self.cxxflags + ['-w']
self.cflags += self.cwarnflags
self.cxxflags += self.cwarnflags
def name(self):
return 'clang'
@@ -134,10 +133,10 @@ class ClangToolchain(toolchain.Toolchain):
iosprefs = prefs['ios']
if 'deploymenttarget' in iosprefs:
self.deploymenttarget = iosprefs['deploymenttarget']
if self.target.is_macosx() and 'macosx' in prefs:
macosxprefs = prefs['macosx']
if 'deploymenttarget' in macosxprefs:
self.deploymenttarget = macosxprefs['deploymenttarget']
if self.target.is_macos() and 'macos' in prefs:
macosprefs = prefs['macos']
if 'deploymenttarget' in macosprefs:
self.deploymenttarget = macosprefs['deploymenttarget']
if self.target.is_pnacl() and 'pnacl' in prefs:
pnaclprefs = prefs['pnacl']
if 'sdkpath' in pnaclprefs:
@@ -152,7 +151,7 @@ class ClangToolchain(toolchain.Toolchain):
writer.variable('cxx', self.cxxcompiler)
writer.variable('ar', self.archiver)
writer.variable('link', self.linker)
if self.target.is_macosx() or self.target.is_ios():
if self.target.is_macos() or self.target.is_ios():
writer.variable('lipo', self.lipo)
if self.target.is_pnacl():
writer.variable('finalize', self.finalizer)
@@ -161,7 +160,7 @@ class ClangToolchain(toolchain.Toolchain):
writer.variable('moreincludepaths', '')
writer.variable('cflags', self.cflags)
writer.variable('cxxflags', self.cxxflags)
if self.target.is_macosx() or self.target.is_ios():
if self.target.is_macos() or self.target.is_ios():
writer.variable('mflags', self.mflags)
writer.variable('carchflags', '')
writer.variable('cconfigflags', '')
@@ -184,7 +183,7 @@ class ClangToolchain(toolchain.Toolchain):
super(ClangToolchain, self).write_rules(writer)
writer.rule('cc', command = self.cccmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CC $in')
writer.rule('cxx', command = self.cxxcmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CXX $in')
if self.target.is_macosx() or self.target.is_ios():
if self.target.is_macos() or self.target.is_ios():
writer.rule('cm', command = self.cmcmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CM $in')
writer.rule( 'lipo', command = self.lipocmd, description = 'LIPO $out' )
writer.rule('ar', command = self.arcmd, description = 'LIB $out')
@@ -201,21 +200,15 @@ class ClangToolchain(toolchain.Toolchain):
self.build_windows_toolchain()
elif self.target.is_android():
self.build_android_toolchain()
elif self.target.is_macosx() or self.target.is_ios():
elif self.target.is_macos() or self.target.is_ios():
self.build_xcode_toolchain()
elif self.target.is_pnacl():
self.build_pnacl_toolchain()
else:
self.build_default_toolchain()
if self.toolchain != '' and not self.toolchain.endswith('/') and not self.toolchain.endswith('\\'):
self.toolchain += os.sep
def build_default_toolchain(self):
self.cxxflags = list(self.cflags)
def build_windows_toolchain(self):
self.cflags += ['-U__STRICT_ANSI__', '-Wno-reserved-id-macro']
self.cxxflags = list(self.cflags)
self.oslibs = ['kernel32', 'user32', 'shell32', 'advapi32']
def build_android_toolchain(self):
@@ -225,7 +218,6 @@ class ClangToolchain(toolchain.Toolchain):
self.linkcmd += ' -shared -Wl,-soname,$liblinkname --sysroot=$sysroot'
self.cflags += ['-fpic', '-ffunction-sections', '-funwind-tables', '-fstack-protector', '-fomit-frame-pointer',
'-no-canonical-prefixes', '-Wa,--noexecstack']
self.cxxflags = list(self.cflags)
self.linkflags += ['-no-canonical-prefixes', '-Wl,--no-undefined', '-Wl,-z,noexecstack', '-Wl,-z,relro', '-Wl,-z,now']
@@ -237,16 +229,18 @@ class ClangToolchain(toolchain.Toolchain):
self.toolchain = os.path.join('$ndk', 'toolchains', 'llvm', 'prebuilt', self.android.hostarchname, 'bin', '')
def build_xcode_toolchain(self):
if self.target.is_macosx():
if self.target.is_macos():
sdk = 'macosx'
deploytarget = 'MACOSX_DEPLOYMENT_TARGET=' + self.deploymenttarget
self.cflags += ['-fasm-blocks', '-mmacosx-version-min=' + self.deploymenttarget, '-isysroot', '$sysroot']
self.cxxflags += ['-fasm-blocks', '-mmacosx-version-min=' + self.deploymenttarget, '-isysroot', '$sysroot']
self.arflags += ['-static', '-no_warning_for_no_symbols']
self.linkflags += ['-isysroot', '$sysroot']
elif self.target.is_ios():
sdk = 'iphoneos'
deploytarget = 'IPHONEOS_DEPLOYMENT_TARGET=' + self.deploymenttarget
self.cflags += ['-fasm-blocks', '-miphoneos-version-min=' + self.deploymenttarget, '-isysroot', '$sysroot']
self.cxxflags += ['-fasm-blocks', '-miphoneos-version-min=' + self.deploymenttarget, '-isysroot', '$sysroot']
self.arflags += ['-static', '-no_warning_for_no_symbols']
self.linkflags += ['-isysroot', '$sysroot']
self.cflags += ['-fembed-bitcode-marker']
@@ -254,25 +248,22 @@ class ClangToolchain(toolchain.Toolchain):
platformpath = subprocess.check_output(['xcrun', '--sdk', sdk, '--show-sdk-platform-path']).strip()
localpath = platformpath + "/Developer/usr/bin:/Applications/Xcode.app/Contents/Developer/usr/bin:/usr/bin:/bin:/usr/sbin:/sbin"
if self.target.is_macosx():
self.sysroot = subprocess.check_output(['xcrun', '--sdk', 'macosx', '--show-sdk-path']).strip()
elif self.target.is_ios():
self.sysroot = subprocess.check_output(['xcrun', '--sdk', 'iphoneos', '--show-sdk-path']).strip()
self.sysroot = subprocess.check_output(['xcrun', '--sdk', sdk, '--show-sdk-path']).strip()
self.ccompiler = "PATH=" + localpath + " " + subprocess.check_output(['xcrun', '--sdk', sdk, '-f', 'clang']).strip()
self.archiver = "PATH=" + localpath + " " + subprocess.check_output(['xcrun', '--sdk', sdk, '-f', 'libtool']).strip()
self.linker = deploytarget + " " + self.ccompiler
self.lipo = "PATH=" + localpath + " " + subprocess.check_output(['xcrun', '--sdk', sdk, '-f', 'lipo']).strip()
self.mflags += self.cflags + ['-fobjc-arc', '-fno-objc-exceptions', '-x', 'objective-c']
self.cxxflags = self.cflags + ['-x', 'c++']
self.mflags += list(self.cflags) + ['-fobjc-arc', '-fno-objc-exceptions', '-x', 'objective-c']
self.cflags += ['-x', 'c']
self.cxxflags += ['-x', 'c++']
self.cmcmd = self.cccmd.replace('$cflags', '$mflags')
self.arcmd = self.rmcmd('$out') + ' && $ar $ararchflags $arflags $in -o $out'
self.lipocmd = '$lipo $in -create -output $out'
if self.target.is_macosx():
if self.target.is_macos():
self.frameworks = ['Cocoa', 'CoreFoundation']
if self.target.is_ios():
self.frameworks = ['CoreGraphics', 'UIKit', 'Foundation']
@@ -292,7 +283,6 @@ class ClangToolchain(toolchain.Toolchain):
self.linker = self.ccompiler
self.finalizer = os.path.join('bin', 'pnacl-finalize' + shsuffix)
self.nmfer = os.path.join('tools', 'create_nmf.py')
self.cxxflags = list(self.cflags)
self.finalizecmd = '$toolchain$finalize -o $out $in'
self.nmfcmd = self.python + ' ' + os.path.join('$sdkpath', '$nmf') + ' -o $out $in'
@@ -303,14 +293,17 @@ class ClangToolchain(toolchain.Toolchain):
def make_includepaths(self, includepaths):
if not includepaths is None:
return ['-I' + self.path_escape(path) for path in list(includepaths)]
return ['-I' + path for path in list(includepaths)]
return []
def make_libpath(self, path):
return self.path_escape(path)
def make_libpaths(self, libpaths):
if not libpaths is None:
if self.target.is_windows():
return ['-Xlinker /LIBPATH:' + self.path_escape(path) for path in libpaths]
return ['-L' + self.path_escape(path) for path in libpaths]
return ['-L' + self.make_libpath(path) for path in libpaths]
return []
def make_targetarchflags(self, arch, targettype):
@@ -335,7 +328,7 @@ class ClangToolchain(toolchain.Toolchain):
elif arch == 'mips64':
flags += ['-target', 'mips64el-none-linux-android']
flags += ['-gcc-toolchain', self.android.make_gcc_toolchain_path(arch)]
elif self.target.is_macosx() or self.target.is_ios():
elif self.target.is_macos() or self.target.is_ios():
if arch == 'x86':
flags += [' -arch x86']
elif arch == 'x86-64':
@@ -355,8 +348,6 @@ class ClangToolchain(toolchain.Toolchain):
flags = []
if targettype == 'sharedlib':
flags += ['-DBUILD_DYNAMIC_LINK=1']
if self.target.is_linux():
flags += ['-fPIC']
flags += self.make_targetarchflags(arch, targettype)
return flags
@@ -391,7 +382,7 @@ class ClangToolchain(toolchain.Toolchain):
flags += ['-Xlinker', '/MACHINE:X86']
elif arch == 'x86-64':
flags += ['-Xlinker', '/MACHINE:X64']
if self.target.is_macosx() and 'support_lua' in variables and variables['support_lua']:
if self.target.is_macos() and 'support_lua' in variables and variables['support_lua']:
flags += ['-pagezero_size', '10000', '-image_base', '100000000']
return flags
@@ -402,12 +393,6 @@ class ClangToolchain(toolchain.Toolchain):
flags += ['-Xlinker', '/DLL']
elif targettype == 'bin':
flags += ['-Xlinker', '/SUBSYSTEM:CONSOLE']
else:
if targettype == 'sharedlib':
if self.target.is_macosx() or self.target.is_ios():
flags += ['-dynamiclib']
else:
flags += ['-shared']
return flags
def make_linkarchlibs(self, arch, targettype):
@@ -432,18 +417,18 @@ class ClangToolchain(toolchain.Toolchain):
def make_configlibpaths(self, config, arch, extralibpaths):
libpaths = [self.libpath, os.path.join(self.libpath, config)]
if not self.target.is_macosx() and not self.target.is_ios():
if not self.target.is_macos() and not self.target.is_ios():
libpaths += [os.path.join(self.libpath, arch)]
libpaths += [os.path.join(self.libpath, config, arch)]
if extralibpaths != None:
libpaths += [os.path.join(libpath, self.libpath) for libpath in extralibpaths]
libpaths += [os.path.join(libpath, self.libpath, config) for libpath in extralibpaths]
if not self.target.is_macosx() and not self.target.is_ios():
if not self.target.is_macos() and not self.target.is_ios():
libpaths += [os.path.join(libpath, self.libpath, arch) for libpath in extralibpaths]
libpaths += [os.path.join(libpath, self.libpath, config, arch) for libpath in extralibpaths]
return self.make_libpaths(libpaths)
def cc_variables(self, config, arch, targettype, variables, externalsources):
def cc_variables(self, config, arch, targettype, variables):
localvariables = []
if 'includepaths' in variables:
moreincludepaths = self.make_includepaths(variables['includepaths'])
@@ -457,8 +442,6 @@ class ClangToolchain(toolchain.Toolchain):
localvariables += [('cconfigflags', cconfigflags)]
if self.target.is_android():
localvariables += [('sysroot', self.android.make_sysroot_path(arch))]
if externalsources:
localvariables += [('cflags', self.cexternflags), ('cxxflags', self.cxxexternflags)]
if 'defines' in variables:
localvariables += [('cmoreflags', ['-D' + define for define in variables['defines']])]
return localvariables
@@ -477,8 +460,6 @@ class ClangToolchain(toolchain.Toolchain):
def link_variables(self, config, arch, targettype, variables):
localvariables = []
if not variables:
variables = {}
linkarchflags = self.make_linkarchflags(arch, targettype, variables)
if linkarchflags != []:
localvariables += [('linkarchflags', linkarchflags)]
@@ -495,7 +476,7 @@ class ClangToolchain(toolchain.Toolchain):
localframeworks += list(variables['frameworks'])
if len(localframeworks) > 0:
localvariables += [('frameworks', self.make_frameworks(list(localframeworks)))]
libpaths = []
if 'libpaths' in variables:
libpaths = variables['libpaths']
@@ -511,40 +492,39 @@ class ClangToolchain(toolchain.Toolchain):
return localvariables
def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables, externalsources):
return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources))
def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables):
return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables))
def builder_cxx(self, writer, config, arch, targettype, infile, outfile, variables, externalsources):
return writer.build(outfile, 'cxx', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources))
def builder_cxx(self, writer, config, arch, targettype, infile, outfile, variables):
return writer.build(outfile, 'cxx', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables))
def builder_cm(self, writer, config, arch, targettype, infile, outfile, variables, externalsources):
return writer.build(outfile, 'cm', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources))
def builder_cm(self, writer, config, arch, targettype, infile, outfile, variables):
return writer.build(outfile, 'cm', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables))
def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables):
return writer.build(outfile, 'ar', infiles, implicit = self.implicit_deps(config, variables), variables = self.ar_variables(config, arch, targettype, variables))
def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables):
return writer.build(outfile, 'so', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables))
def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables):
return writer.build(outfile, 'link', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables))
#Apple universal targets
def builder_apple_multilib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_apple_multilib(self, writer, config, arch, targettype, infiles, outfile, variables):
localvariables = [('arflags', '-static -no_warning_for_no_symbols')]
if variables != None:
localvariables = variables + localvariables
return writer.build(os.path.join(outfile, self.buildtarget), 'ar', infiles, variables = localvariables);
def builder_apple_multisharedlib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
#return writer.build(outfile, 'so', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables))
return writer.build(os.path.join(outfile, self.buildtarget), 'lipo', infiles, variables = variables)
def builder_apple_multisharedlib(self, writer, config, arch, targettype, infiles, outfile, variables):
return writer.build(outfile, 'so', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables))
def builder_apple_multibin(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_apple_multibin(self, writer, config, arch, targettype, infiles, outfile, variables):
return writer.build(os.path.join(outfile, self.buildtarget), 'lipo', infiles, variables = variables)
#PNaCl finalizer
def builder_pnacl_multibin(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_pnacl_multibin(self, writer, config, arch, targettype, infiles, outfile, variables):
binfile = os.path.splitext(self.buildtarget)[0]
pexe = writer.build(os.path.join(outfile, binfile + '.pexe'), 'finalize', infiles)
nmf = writer.build(os.path.join(outfile, binfile + '.nmf'), 'nmf', pexe + infiles)

34
build/ninja/codesign.py
View File

@@ -14,7 +14,7 @@ parser.add_argument('file', type=str,
help = 'Bundle/package to sign')
parser.add_argument('--target', type=str,
help = 'Target',
choices = ['macosx', 'ios', 'android'],
choices = ['macos', 'ios', 'android'],
default = '')
parser.add_argument('--bundle', type=str,
help = 'Bundle identifier (OSX/iOS)',
@@ -65,13 +65,13 @@ options = parser.parse_args()
androidprefs = {}
iosprefs = {}
macosxprefs = {}
macosprefs = {}
def parse_prefs( prefsfile ):
global androidprefs
global iosprefs
global macosxprefs
global macosprefs
if not os.path.isfile( prefsfile ):
return
file = open( prefsfile, 'r' )
@@ -81,8 +81,8 @@ def parse_prefs( prefsfile ):
androidprefs = prefs['android']
if 'ios' in prefs:
iosprefs = prefs['ios']
if 'macosx' in prefs:
macosxprefs = prefs['macosx']
if 'macos' in prefs:
macosprefs = prefs['macos']
def codesign_ios():
@@ -127,7 +127,7 @@ def codesign_ios():
if os.path.isfile( os.path.join( options.file, '_CodeSignature', 'CodeResources' ) ):
os.remove( os.path.join( options.file, '_CodeSignature', 'CodeResources' ) )
os.system( '/usr/bin/codesign --force --sign ' + iosprefs['signature'] + ' --entitlements ' + plistpath + ' ' + options.file )
os.system( '/usr/bin/codesign --force --sign "' + iosprefs['signature'] + '" --entitlements ' + plistpath + ' ' + options.file )
if os.path.isfile( os.path.join( options.file, '_CodeSignature', 'CodeResources' ) ):
os.utime( os.path.join( options.file, '_CodeSignature', 'CodeResources' ), None )
@@ -135,13 +135,13 @@ def codesign_ios():
os.utime( options.file, None )
def codesign_macosx():
if not 'organisation' in macosxprefs:
macosxprefs['organisation'] = options.organisation
if not 'bundleidentifier' in macosxprefs:
macosxprefs['bundleidentifier'] = options.bundle
if not 'provisioning' in macosxprefs:
macosxprefs['provisioning'] = options.provisioning
def codesign_macos():
if not 'organisation' in macosprefs:
macosprefs['organisation'] = options.organisation
if not 'bundleidentifier' in macosprefs:
macosprefs['bundleidentifier'] = options.bundle
if not 'provisioning' in macosprefs:
macosprefs['provisioning'] = options.provisioning
codesign_allocate = subprocess.check_output( [ 'xcrun', '--sdk', 'macosx', '-f', 'codesign_allocate' ] ).strip()
sdkdir = subprocess.check_output( [ 'xcrun', '--sdk', 'macosx', '--show-sdk-path' ] ).strip()
@@ -150,8 +150,8 @@ def codesign_macosx():
if os.path.isfile( os.path.join( options.file, 'Contents', '_CodeSignature', 'CodeResources' ) ):
os.remove( os.path.join( options.file, 'Contents', '_CodeSignature', 'CodeResources' ) )
if 'signature' in macosxprefs:
os.system( 'export CODESIGN_ALLOCATE=' + codesign_allocate + '; /usr/bin/codesign --force --sign ' + macosxprefs['signature'] + ' ' + options.file )
if 'signature' in macosprefs:
os.system( 'export CODESIGN_ALLOCATE=' + codesign_allocate + '; /usr/bin/codesign --force --sign ' + macosprefs['signature'] + ' ' + options.file )
if os.path.isfile( os.path.join( options.file, 'Contents', '_CodeSignature', 'CodeResources' ) ):
os.utime( os.path.join( options.file, 'Contents', '_CodeSignature', 'CodeResources' ), None )
@@ -215,7 +215,7 @@ parse_prefs( options.prefs )
if options.target == 'ios':
codesign_ios()
elif options.target == 'macosx':
codesign_macosx()
elif options.target == 'macos':
codesign_macos()
elif options.target == 'android':
codesign_android()

115
build/ninja/gcc.py
View File

@@ -8,24 +8,24 @@ import toolchain
class GCCToolchain(toolchain.Toolchain):
def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables):
def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables, subninja):
#Local variable defaults
self.toolchain = ''
self.includepaths = includepaths
self.includepaths = []
self.libpaths = libpaths
self.ccompiler = 'gcc'
self.cxxompiler = 'g++'
self.cxxcompiler = 'g++'
self.archiver = 'ar'
self.linker = 'gcc'
self.cxxlinker = 'g++'
#Command definitions
self.cccmd = '$toolchain$cc -MMD -MT $out -MF $out.d -I. $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags -c $in -o $out'
self.cxxcmd = '$toolchain$cxx -MMD -MT $out -MF $out.d -I. $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags $cmoreflags -c $in -o $out'
self.cccmd = '$toolchain$cc -MMD -MT $out -MF $out.d $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags -c $in -o $out'
self.cxxcmd = '$toolchain$cxx -MMD -MT $out -MF $out.d $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags $cmoreflags -c $in -o $out'
self.ccdeps = 'gcc'
self.ccdepfile = '$out.d'
self.arcmd = self.rmcmd('$out') + ' && $toolchain$ar crsD $ararchflags $arflags $out $in'
self.linkcmd = '$toolchain$link $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags -o $out $in $libs $archlibs $oslibs'
self.linkcmd = '$toolchain$cc $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags -o $out $in $libs $archlibs $oslibs'
#Base flags
self.cflags = ['-D' + project.upper() + '_COMPILE=1',
@@ -33,17 +33,13 @@ class GCCToolchain(toolchain.Toolchain):
'-fno-math-errno','-ffinite-math-only', '-funsafe-math-optimizations',
'-fno-trapping-math', '-ffast-math']
self.cwarnflags = ['-Wextra', '-Wall', '-Werror']
self.cmoreflags = []
self.mflags = []
self.arflags = []
self.linkflags = []
self.oslibs = []
if self.target.is_linux() or self.target.is_bsd() or self.target.is_raspberrypi():
self.linkflags += ['-pthread']
self.oslibs += ['m']
if self.target.is_linux() or self.target.is_raspberrypi():
self.oslibs += ['dl']
self.initialize_subninja(subninja)
self.initialize_archs(archs)
self.initialize_configs(configs)
self.initialize_project(project)
@@ -53,8 +49,32 @@ class GCCToolchain(toolchain.Toolchain):
self.parse_default_variables(variables)
self.read_build_prefs()
if self.target.is_linux() or self.target.is_bsd() or self.target.is_raspberrypi():
self.cflags += ['-D_GNU_SOURCE=1']
self.linkflags += ['-pthread']
self.oslibs += ['m']
if self.target.is_linux() or self.target.is_raspberrypi():
self.oslibs += ['dl']
if self.target.is_bsd():
self.oslibs += ['execinfo']
self.includepaths = self.prefix_includepaths((includepaths or []) + ['.'])
if self.is_monolithic():
self.cflags += ['-DBUILD_MONOLITHIC=1']
if self.use_coverage():
self.cflags += ['--coverage']
self.linkflags += ['--coverage']
if not 'nowarning' in variables or not variables['nowarning']:
self.cflags += self.cwarnflags
self.cxxflags = list(self.cflags)
self.cflags += ['-std=c11']
if self.target.is_macos() or self.target.is_ios():
self.cxxflags += ['-std=c++14', '-stdlib=libc++']
else:
self.cxxflags += ['-std=gnu++14']
#Overrides
self.objext = '.o'
@@ -62,7 +82,6 @@ class GCCToolchain(toolchain.Toolchain):
#Builders
self.builders['c'] = self.builder_cc
self.builders['cc'] = self.builder_cxx
self.builders['cpp'] = self.builder_cxx
self.builders['lib'] = self.builder_lib
self.builders['multilib'] = self.builder_multicopy
self.builders['sharedlib'] = self.builder_sharedlib
@@ -73,15 +92,6 @@ class GCCToolchain(toolchain.Toolchain):
#Setup target platform
self.build_target_toolchain(self.target)
self.cmoreflags = []
self.cexternflags = []
self.cxxexternflags = []
self.cexternflags += self.cflags
self.cxxexternflags += self.cxxflags
self.cflags += self.cwarnflags
self.cxxflags += self.cwarnflags
def name(self):
return 'gcc'
@@ -98,7 +108,7 @@ class GCCToolchain(toolchain.Toolchain):
super(GCCToolchain, self).write_variables(writer)
writer.variable('toolchain', self.toolchain)
writer.variable('cc', self.ccompiler)
writer.variable('cxx', self.cxxompiler)
writer.variable('cxx', self.cxxcompiler)
writer.variable('ar', self.archiver)
writer.variable('link', self.linker)
writer.variable('includepaths', self.make_includepaths(self.includepaths))
@@ -133,31 +143,31 @@ class GCCToolchain(toolchain.Toolchain):
def build_target_toolchain(self, target):
if target.is_windows():
self.build_windows_toolchain()
else:
self.build_default_toolchain()
if self.toolchain != '' and not self.toolchain.endswith('/') and not self.toolchain.endswith('\\'):
self.toolchain += os.sep
def build_default_toolchain(self):
self.cxxflags = self.cflags + ['-std=c++11', '-D_GNU_SOURCE=1']
self.cflags += ['-std=c11', '-D_GNU_SOURCE=1']
if self.target.is_macosx() or self.target.is_ios():
self.cxxflags += ['-stdlib=libc++']
def build_windows_toolchain(self):
self.cxxflags = self.cflags
self.cxxflags += ['-U__STRICT_ANSI__', '-std=c++11']
self.cflags += ['-U__STRICT_ANSI__', '-std=c11']
self.cflags += ['-U__STRICT_ANSI__']
self.oslibs = ['kernel32', 'user32', 'shell32', 'advapi32']
def make_includepath(self, path):
if os.path.isabs(path) or self.subninja == '':
return self.path_escape(path)
if path == '.':
return self.path_escape(self.subninja)
return self.path_escape(os.path.join(self.subninja, path))
def make_includepaths(self, includepaths):
if not includepaths is None:
return ['-I' + self.path_escape(path) for path in list(includepaths)]
return ['-I' + self.make_includepath(path) for path in list(includepaths)]
return []
def make_libpath(self, path):
return self.path_escape(path)
def make_libpaths(self, libpaths):
if not libpaths is None:
return ['-L' + self.path_escape(path) for path in libpaths]
return ['-L' + self.make_libpath(path) for path in libpaths]
return []
def make_targetarchflags(self, arch, targettype):
@@ -172,8 +182,6 @@ class GCCToolchain(toolchain.Toolchain):
flags = []
if targettype == 'sharedlib':
flags += ['-DBUILD_DYNAMIC_LINK=1']
if self.target.is_linux():
flags += ['-fPIC']
flags += self.make_targetarchflags(arch, targettype)
return flags
@@ -204,17 +212,6 @@ class GCCToolchain(toolchain.Toolchain):
def make_linkconfigflags(self, config, targettype):
flags = []
if self.target.is_windows():
if targettype == 'sharedlib':
flags += ['-Xlinker', '/DLL']
elif targettype == 'bin':
flags += ['-Xlinker', '/SUBSYSTEM:CONSOLE']
else:
if targettype == 'sharedlib':
if self.target.is_macosx() or self.target.is_ios():
flags += ['-dynamiclib']
else:
flags += ['-shared']
return flags
def make_libs(self, libs):
@@ -236,7 +233,7 @@ class GCCToolchain(toolchain.Toolchain):
libpaths += [os.path.join(libpath, self.libpath, config, arch) for libpath in extralibpaths]
return self.make_libpaths(libpaths)
def cc_variables(self, config, arch, targettype, variables, externalsources):
def cc_variables(self, config, arch, targettype, variables):
localvariables = []
if 'includepaths' in variables:
moreincludepaths = self.make_includepaths(variables['includepaths'])
@@ -248,8 +245,6 @@ class GCCToolchain(toolchain.Toolchain):
cconfigflags = self.make_cconfigflags(config, targettype)
if cconfigflags != []:
localvariables += [('cconfigflags', cconfigflags)]
if externalsources:
localvariables += [('cflags', self.cexternflags), ('cxxflags', self.cxxexternflags)]
if 'defines' in variables:
localvariables += [('cmoreflags', ['-D' + define for define in variables['defines']])]
return localvariables
@@ -280,23 +275,25 @@ class GCCToolchain(toolchain.Toolchain):
if 'libpaths' in variables:
libpaths = variables['libpaths']
localvariables += [('configlibpaths', self.make_configlibpaths(config, arch, libpaths))]
if 'runtime' in variables and variables['runtime'] == 'c++':
localvariables += [('link', self.cxxlinker)]
return localvariables
def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables, externalsources):
return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources))
def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables):
return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables))
def builder_cxx(self, writer, config, arch, targettype, infile, outfile, variables, externalsources):
return writer.build(outfile, 'cxx', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources))
def builder_cxx(self, writer, config, arch, targettype, infile, outfile, variables):
return writer.build(outfile, 'cxx', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables))
def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables):
return writer.build(outfile, 'ar', infiles, implicit = self.implicit_deps(config, variables), variables = self.ar_variables(config, arch, targettype, variables))
def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables):
return writer.build(outfile, 'so', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables))
def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables):
return writer.build(outfile, 'link', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables))
def create(host, target, toolchain):

52
build/ninja/generator.py
View File

@@ -25,7 +25,7 @@ class Generator(object):
choices = toolchain.supported_toolchains())
parser.add_argument('-c', '--config', action = 'append',
help = 'Build configuration',
choices = ['debug', 'release'],
choices = ['debug', 'release', 'profile', 'deploy'],
default = [])
parser.add_argument('-a', '--arch', action = 'append',
help = 'Add architecture',
@@ -40,11 +40,15 @@ class Generator(object):
parser.add_argument('--coverage', action='store_true',
help = 'Build with code coverage',
default = False)
parser.add_argument('--subninja', action='store',
help = 'Build as subproject (exclude rules and pools) with the given subpath',
default = '')
options = parser.parse_args()
self.project = project
self.target = platform.Platform(options.target)
self.host = platform.Platform(options.host)
self.subninja = options.subninja
archs = options.arch
configs = options.config
if includepaths is None:
@@ -72,23 +76,20 @@ class Generator(object):
config_str = ' '.join([key + '=' + pipes.quote(configure_env[key]) for key in configure_env])
writer.variable('configure_env', config_str + '$ ')
if variables is None:
variables = {}
if not isinstance(variables, dict):
variables = dict(variables)
if options.monolithic:
if variables is None:
variables = {}
if isinstance(variables, dict):
variables['monolithic'] = True
else:
variables += [('monolithic', True)]
variables['monolithic'] = True
if options.coverage:
if variables is None:
variables = {}
if isinstance(variables, dict):
variables['coverage'] = True
else:
variables += [('coverage', True)]
variables['coverage'] = True
if self.subninja != '':
variables['internal_deps'] = True
self.toolchain = toolchain.make_toolchain(self.host, self.target, options.toolchain)
self.toolchain.initialize(project, archs, configs, includepaths, dependlibs, libpaths, variables)
self.toolchain.initialize(project, archs, configs, includepaths, dependlibs, libpaths, variables, self.subninja)
self.writer.variable('configure_toolchain', self.toolchain.name())
self.writer.variable('configure_archs', archs)
@@ -96,7 +97,8 @@ class Generator(object):
self.writer.newline()
self.toolchain.write_variables(self.writer)
self.toolchain.write_rules(self.writer)
if self.subninja == '':
self.toolchain.write_rules(self.writer)
def target(self):
return self.target
@@ -110,19 +112,23 @@ class Generator(object):
def writer(self):
return self.writer
def lib(self, module, sources, libname, basepath = None, configs = None, includepaths = None, variables = None, externalsources = False):
return self.toolchain.lib(self.writer, module, sources, libname, basepath, configs, includepaths, variables, None, externalsources)
def is_subninja(self):
return self.subninja != ''
def sharedlib(self, module, sources, libname, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, libs = None, frameworks = None, variables = None, externalsources = False):
return self.toolchain.sharedlib(self.writer, module, sources, libname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, None, externalsources)
def lib(self, module, sources, libname = None, basepath = None, configs = None, includepaths = None, variables = None):
return self.toolchain.lib(self.writer, module, sources, libname, basepath, configs, includepaths, variables)
def bin(self, module, sources, binname, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, libs = None, frameworks = None, variables = None, outpath = None, externalsources = False):
return self.toolchain.bin(self.writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, outpath, externalsources)
def sharedlib(self, module, sources, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, dependlibs = None, libs = None, frameworks = None, variables = None):
return self.toolchain.sharedlib(self.writer, module, sources, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables)
def app(self, module, sources, binname, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, libs = None, frameworks = None, variables = None, resources = None):
return self.toolchain.app(self.writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, resources)
def bin(self, module, sources, binname, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, dependlibs = None, libs = None, frameworks = None, variables = None):
return self.toolchain.bin(self.writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables)
def app(self, module, sources, binname, basepath = None, configs = None, includepaths = None, libpaths = None, implicit_deps = None, dependlibs = None, libs = None, frameworks = None, variables = None, resources = None):
return self.toolchain.app(self.writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables, resources)
def test_includepaths(self):
#TODO: This is ugly
if self.project == "foundation":
return ['test']
return ['test', os.path.join('..', 'foundation_lib', 'test')]

56
build/ninja/msvc.py
View File

@@ -9,11 +9,11 @@ import toolchain
class MSVCToolchain(toolchain.Toolchain):
def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables):
def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables, subninja):
#Local variable defaults
self.sdkpath = ''
self.toolchain = ''
self.includepaths = includepaths
self.includepaths = []
self.libpaths = libpaths
self.ccompiler = 'cl'
self.archiver = 'lib'
@@ -21,30 +21,20 @@ class MSVCToolchain(toolchain.Toolchain):
self.dller = 'dll'
#Command definitions
self.cccmd = '$toolchain$cc /showIncludes /I. $includepaths $moreincludepaths $cflags $carchflags $cconfigflags /c $in /Fo$out /Fd$pdbpath /FS /nologo'
self.cxxcmd = '$toolchain$cc /showIncludes /I. $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags /c $in /Fo$out /Fd$pdbpath /FS /nologo /EHsc'
self.cccmd = '$toolchain$cc /showIncludes /I. $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags /c $in /Fo$out /Fd$pdbpath /FS /nologo'
self.ccdepfile = None
self.ccdeps = 'msvc'
self.arcmd = '$toolchain$ar $arflags $ararchflags $arconfigflags /NOLOGO /OUT:$out $in'
self.linkcmd = '$toolchain$link $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags /DEBUG /NOLOGO /SUBSYSTEM:CONSOLE /DYNAMICBASE /NXCOMPAT /MANIFEST /MANIFESTUAC:\"level=\'asInvoker\' uiAccess=\'false\'\" /TLBID:1 /PDB:$pdbpath /OUT:$out $in $libs $archlibs $oslibs'
self.dllcmd = self.linkcmd + ' /DLL'
#Base flags
self.cflags = ['/D', '"' + project.upper() + '_COMPILE=1"', '/Zi', '/Oi', '/Oy-', '/GS-', '/Gy-', '/Qpar-', '/fp:fast', '/fp:except-', '/Zc:forScope', '/Zc:wchar_t', '/GR-', '/openmp-']
self.cxxflags = self.cflags
self.cwarnflags = ['/W3', '/WX']
self.cflags = ['/D', '"' + project.upper() + '_COMPILE=1"', '/D', '"_UNICODE"', '/D', '"UNICODE"', '/Zi', '/W3', '/WX', '/Oi', '/Oy-', '/GS-', '/Gy-', '/Qpar-', '/fp:fast', '/fp:except-', '/Zc:forScope', '/Zc:wchar_t', '/GR-', '/openmp-']
self.cmoreflags = []
self.arflags = ['/ignore:4221'] #Ignore empty object file warning]
self.linkflags = ['/DEBUG']
self.oslibs = ['kernel32', 'user32', 'shell32', 'advapi32']
self.cexternflags = ['/W0']
self.cxxexternflags = ['/W0', '/EHsc']
self.cexternflags += self.cflags
self.cxxexternflags += self.cxxflags
self.cflags += self.cwarnflags
self.cxxflags += self.cwarnflags
self.initialize_subninja(subninja)
self.initialize_archs(archs)
self.initialize_configs(configs)
self.initialize_project(project)
@@ -54,6 +44,8 @@ class MSVCToolchain(toolchain.Toolchain):
self.parse_default_variables(variables)
self.read_build_prefs()
self.includepaths = self.prefix_includepaths((includepaths or []) + ['.'])
if self.is_monolithic():
self.cflags += ['/D', '"BUILD_MONOLITHIC=1"']
@@ -62,8 +54,6 @@ class MSVCToolchain(toolchain.Toolchain):
#Builders
self.builders['c'] = self.builder_cc
self.builders['cc'] = self.builder_cxx
self.builders['cpp'] = self.builder_cxx
self.builders['lib'] = self.builder_lib
self.builders['multilib'] = self.builder_multicopy
self.builders['sharedlib'] = self.builder_sharedlib
@@ -99,7 +89,7 @@ class MSVCToolchain(toolchain.Toolchain):
writer.variable('cflags', self.cflags)
writer.variable('carchflags', '')
writer.variable('cconfigflags', '')
writer.variable('cxxflags', self.cxxflags)
writer.variable('cmoreflags', self.cmoreflags)
writer.variable('arflags', self.arflags)
writer.variable('ararchflags', '')
writer.variable('arconfigflags', '')
@@ -116,7 +106,6 @@ class MSVCToolchain(toolchain.Toolchain):
def write_rules(self, writer):
super(MSVCToolchain, self).write_rules(writer)
writer.rule('cc', command = self.cccmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CC $in')
writer.rule('cxx', command = self.cxxcmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CXX $in')
writer.rule('ar', command = self.arcmd, description = 'LIB $out')
writer.rule('link', command = self.linkcmd, description = 'LINK $out')
writer.rule('dll', command = self.dllcmd, description = 'DLL $out')
@@ -213,9 +202,12 @@ class MSVCToolchain(toolchain.Toolchain):
return ['/I' + self.path_escape(path) for path in list(includepaths)]
return []
def make_libpath(self, path):
return self.path_escape(path)
def make_libpaths(self, libpaths):
if not libpaths is None:
return ['/LIBPATH:' + self.path_escape(path) for path in list(libpaths)]
return ['/LIBPATH:' + self.make_libpath(path) for path in libpaths]
return []
def make_arch_toolchain_path(self, arch):
@@ -324,7 +316,7 @@ class MSVCToolchain(toolchain.Toolchain):
libpaths += [os.path.join( self.sdkpath, 'lib', self.sdkversionpath, 'ucrt', 'x64')]
return self.make_libpaths(libpaths)
def cc_variables(self, config, arch, targettype, variables, externalsources):
def cc_variables(self, config, arch, targettype, variables):
localvariables = [('toolchain', self.make_arch_toolchain_path(arch))]
if 'includepaths' in variables:
moreincludepaths = self.make_includepaths(variables['includepaths'])
@@ -338,8 +330,11 @@ class MSVCToolchain(toolchain.Toolchain):
cconfigflags = self.make_cconfigflags(config, targettype)
if cconfigflags != []:
localvariables += [('cconfigflags', cconfigflags)]
if externalsources:
localvariables += [('cflags', self.cexternflags), ('cxxflags', self.cxxexternflags)]
if 'defines' in variables:
definelist = []
for define in variables['defines']:
definelist += ['/D', '"' + define + '"']
localvariables += [('cmoreflags', definelist)]
return localvariables
def ar_variables(self, config, arch, targettype, variables):
@@ -372,19 +367,16 @@ class MSVCToolchain(toolchain.Toolchain):
localvariables += [('configlibpaths', self.make_configlibpaths(config, arch, libpaths))]
return localvariables
def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables, externalsources):
return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources))
def builder_cc(self, writer, config, arch, targettype, infile, outfile, variables):
return writer.build(outfile, 'cc', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables))
def builder_cxx(self, writer, config, arch, targettype, infile, outfile, variables, externalsources):
return writer.build(outfile, 'cxx', infile, implicit = self.implicit_deps(config, variables), variables = self.cc_variables(config, arch, targettype, variables, externalsources))
def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_lib(self, writer, config, arch, targettype, infiles, outfile, variables):
return writer.build(outfile, 'ar', infiles, implicit = self.implicit_deps(config, variables), variables = self.ar_variables(config, arch, targettype, variables))
def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_sharedlib(self, writer, config, arch, targettype, infiles, outfile, variables):
return writer.build(outfile, 'dll', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables))
def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables, externalsources):
def builder_bin(self, writer, config, arch, targettype, infiles, outfile, variables):
return writer.build(outfile, 'link', infiles, implicit = self.implicit_deps(config, variables), variables = self.link_variables(config, arch, targettype, variables))
def create(host, target, toolchain):

13
build/ninja/platform.py
View File

@@ -5,7 +5,7 @@
import sys
def supported_platforms():
return [ 'windows', 'linux', 'macosx', 'bsd', 'ios', 'android', 'raspberrypi', 'pnacl', 'tizen' ]
return [ 'windows', 'linux', 'macos', 'bsd', 'ios', 'android', 'raspberrypi', 'pnacl', 'tizen' ]
class Platform(object):
def __init__(self, platform):
@@ -15,9 +15,9 @@ class Platform(object):
if self.platform.startswith('linux'):
self.platform = 'linux'
elif self.platform.startswith('darwin'):
self.platform = 'macosx'
self.platform = 'macos'
elif self.platform.startswith('macos'):
self.platform = 'macosx'
self.platform = 'macos'
elif self.platform.startswith('win'):
self.platform = 'windows'
elif 'bsd' in self.platform:
@@ -42,8 +42,8 @@ class Platform(object):
def is_windows(self):
return self.platform == 'windows'
def is_macosx(self):
return self.platform == 'macosx'
def is_macos(self):
return self.platform == 'macos'
def is_bsd(self):
return self.platform == 'bsd'
@@ -62,6 +62,3 @@ class Platform(object):
def is_tizen(self):
return self.platform == 'tizen'
def get(self):
return self.platform

4
build/ninja/plist.py
View File

@@ -56,9 +56,9 @@ if not options.exename:
if not options.prodname:
options.prodname = 'unknown'
if not options.target:
options.target = 'macosx'
options.target = 'macos'
if not options.deploymenttarget:
if options.target == 'macosx':
if options.target == 'macos':
options.deploymenttarget = '10.7'
else:
options.deploymenttarget = '6.0'

136
build/ninja/toolchain.py
View File

@@ -10,6 +10,7 @@ import random
import string
import json
import zlib
import version
import android
import xcode
@@ -42,11 +43,13 @@ class Toolchain(object):
self.host = host
self.target = target
self.toolchain = toolchain
self.subninja = ''
#Set default values
self.build_monolithic = False
self.build_coverage = False
self.support_lua = False
self.internal_deps = False
self.python = 'python'
self.objext = '.o'
if target.is_windows():
@@ -70,10 +73,7 @@ class Toolchain(object):
else:
self.libprefix = 'lib'
self.staticlibext = '.a'
if target.is_macosx() or target.is_ios():
self.dynamiclibext = '.dylib'
else:
self.dynamiclibext = '.so'
self.dynamiclibext = '.so'
self.binprefix = ''
self.binext = ''
@@ -105,7 +105,7 @@ class Toolchain(object):
#Target functionality
if target.is_android():
self.android = android.make_target(self, host, target)
if target.is_macosx() or target.is_ios():
if target.is_macos() or target.is_ios():
self.xcode = xcode.make_target(self, host, target)
#Builders
@@ -114,8 +114,12 @@ class Toolchain(object):
#Paths created
self.paths_created = {}
def initialize_subninja(self, path):
self.subninja = path
def initialize_project(self, project):
self.project = project
version.generate_version(self.project, self.project)
def initialize_archs(self, archs):
self.archs = list(archs)
@@ -133,7 +137,7 @@ class Toolchain(object):
self.archs = ['x86']
else:
self.archs = [localarch]
elif self.target.is_macosx():
elif self.target.is_macos():
self.archs = ['x86-64']
elif self.target.is_ios():
self.archs = ['arm7', 'arm64']
@@ -152,7 +156,7 @@ class Toolchain(object):
self.initialize_default_configs()
def initialize_default_configs(self):
self.configs = ['debug', 'release']
self.configs = ['debug', 'release', 'profile', 'deploy']
def initialize_toolchain(self):
if self.android != None:
@@ -161,9 +165,29 @@ class Toolchain(object):
self.xcode.initialize_toolchain()
def initialize_depends(self, dependlibs):
#TODO: Improve localization of dependend libs
self.depend_includepaths = [os.path.join('..', lib + '_lib') for lib in dependlibs]
self.depend_libpaths = [os.path.join('..', lib + '_lib') for lib in dependlibs]
for lib in dependlibs:
includepath = ''
libpath = ''
testpaths = [
os.path.join('..', lib),
os.path.join('..', lib + '_lib')
]
for testpath in testpaths:
if os.path.isfile(os.path.join(testpath, lib, lib + '.h')):
if self.subninja != '':
basepath, _ = os.path.split(self.subninja)
_, libpath = os.path.split(testpath)
testpath = os.path.join(basepath, libpath)
includepath = testpath
libpath = testpath
break
if includepath == '':
print("Unable to locate dependent lib: " + lib)
sys.exit(-1)
else:
self.depend_includepaths += [includepath]
if self.subninja == '':
self.depend_libpaths += [libpath]
def build_toolchain(self):
if self.android != None:
@@ -185,6 +209,8 @@ class Toolchain(object):
self.build_coverage = get_boolean_flag(val)
elif key == 'support_lua':
self.support_lua = get_boolean_flag(val)
elif key == 'internal_deps':
self.internal_deps = get_boolean_flag(val)
if self.xcode != None:
self.xcode.parse_default_variables(variables)
@@ -256,6 +282,8 @@ class Toolchain(object):
def mkdir(self, writer, path, implicit = None, order_only = None):
if path in self.paths_created:
return self.paths_created[path]
if self.subninja != '':
return
cmd = writer.build(path, 'mkdir', None, implicit = implicit, order_only = order_only)
self.paths_created[path] = cmd
return cmd
@@ -263,7 +291,7 @@ class Toolchain(object):
def copy(self, writer, src, dst, implicit = None, order_only = None):
return writer.build(dst, 'copy', src, implicit = implicit, order_only = order_only)
def builder_multicopy(self, writer, config, archs, targettype, infiles, outpath, variables, externalsources):
def builder_multicopy(self, writer, config, archs, targettype, infiles, outpath, variables):
output = []
rootdir = self.mkdir(writer, outpath)
for file in infiles:
@@ -295,6 +323,16 @@ class Toolchain(object):
def paths_forward_slash(self, paths):
return [path.replace('\\', '/') for path in paths]
def prefix_includepath(self, path):
if os.path.isabs(path) or self.subninja == '':
return path
if path == '.':
return self.subninja
return os.path.join(self.subninja, path)
def prefix_includepaths(self, includepaths):
return [self.prefix_includepath(path) for path in includepaths]
def list_per_config(self, config_dicts, config):
if config_dicts is None:
return []
@@ -310,29 +348,45 @@ class Toolchain(object):
return self.list_per_config(variables['implicit_deps'], config)
return None
def compile_file(self, writer, config, arch, targettype, infile, outfile, variables, externalsources):
def make_implicit_deps(self, outpath, arch, config, dependlibs):
deps = {}
deps[config] = []
for lib in dependlibs:
if self.target.is_macos() or self.target.is_ios():
finalpath = os.path.join(self.libpath, config, self.libprefix + lib + self.staticlibext)
else:
finalpath = os.path.join(self.libpath, config, arch, self.libprefix + lib + self.staticlibext)
deps[config] += [finalpath]
return [deps]
def compile_file(self, writer, config, arch, targettype, infile, outfile, variables):
extension = os.path.splitext(infile)[1][1:]
if extension in self.builders:
return self.builders[extension](writer, config, arch, targettype, infile, outfile, variables, externalsources)
return self.builders[extension](writer, config, arch, targettype, infile, outfile, variables)
return []
def compile_node(self, writer, nodetype, config, arch, infiles, outfile, variables, externalsources):
def compile_node(self, writer, nodetype, config, arch, infiles, outfile, variables):
if nodetype in self.builders:
return self.builders[nodetype](writer, config, arch, nodetype, infiles, outfile, variables, externalsources)
return self.builders[nodetype](writer, config, arch, nodetype, infiles, outfile, variables)
return []
def build_sources(self, writer, nodetype, multitype, module, sources, binfile, basepath, outpath, configs, includepaths, libpaths, libs, implicit_deps, variables, frameworks, externalsources):
def build_sources(self, writer, nodetype, multitype, module, sources, binfile, basepath, outpath, configs, includepaths, libpaths, dependlibs, libs, implicit_deps, variables, frameworks):
if module != '':
decoratedmodule = module + make_pathhash(module, nodetype)
decoratedmodule = module + make_pathhash(self.subninja + module + binfile, nodetype)
else:
decoratedmodule = basepath + make_pathhash(basepath, nodetype)
decoratedmodule = basepath + make_pathhash(self.subninja + basepath + binfile, nodetype)
built = {}
if includepaths is None:
includepaths = []
if libpaths is None:
libpaths = []
sourcevariables = (variables or {}).copy()
sourcevariables.update({'includepaths': self.depend_includepaths + list(includepaths)})
sourcevariables.update({
'includepaths': self.depend_includepaths + self.prefix_includepaths(list(includepaths))})
if not libs and dependlibs != None:
libs = []
if dependlibs != None:
libs += (dependlibs or [])
nodevariables = (variables or {}).copy()
nodevariables.update({
'libs': libs,
@@ -350,47 +404,59 @@ class Toolchain(object):
modulepath = os.path.join(buildpath, basepath, decoratedmodule)
sourcevariables['modulepath'] = modulepath
nodevariables['modulepath'] = modulepath
#Make per-arch-and-config list of final implicit deps, including dependent libs
if self.internal_deps and dependlibs != None:
dep_implicit_deps = []
if implicit_deps:
dep_implicit_deps += implicit_deps
dep_implicit_deps += self.make_implicit_deps(outpath, arch, config, dependlibs)
nodevariables['implicit_deps'] = dep_implicit_deps
#Compile all sources
for name in sources:
if os.path.isabs(name):
infile = name
outfile = os.path.join(modulepath, os.path.splitext(os.path.basename(name))[0] + make_pathhash(binfile + infile, nodetype) + self.objext)
outfile = os.path.join(modulepath, os.path.splitext(os.path.basename(name))[0] + make_pathhash(infile, nodetype) + self.objext)
else:
infile = os.path.join(basepath, module, name)
outfile = os.path.join(modulepath, os.path.splitext(name)[0] + make_pathhash(binfile + infile, nodetype) + self.objext)
objs += self.compile_file(writer, config, arch, nodetype, infile, outfile, sourcevariables, externalsources)
outfile = os.path.join(modulepath, os.path.splitext(name)[0] + make_pathhash(infile, nodetype) + self.objext)
if self.subninja != '':
infile = os.path.join(self.subninja, infile)
objs += self.compile_file(writer, config, arch, nodetype, infile, outfile, sourcevariables)
#Build arch node (per-config-and-arch binary)
archoutpath = os.path.join(modulepath, binfile)
archnodes += self.compile_node(writer, nodetype, config, arch, objs, archoutpath, nodevariables, externalsources)
archnodes += self.compile_node(writer, nodetype, config, arch, objs, archoutpath, nodevariables)
#Build final config node (per-config binary)
finalpath = os.path.join(outpath, config)
built[config] += self.compile_node(writer, multitype, config, self.archs, archnodes, finalpath, None, externalsources)
built[config] += self.compile_node(writer, multitype, config, self.archs, archnodes, os.path.join(outpath, config), None)
writer.newline()
return built
def lib(self, writer, module, sources, libname, basepath, configs, includepaths, variables, outpath, externalsources):
def lib(self, writer, module, sources, libname, basepath, configs, includepaths, variables, outpath = None):
built = {}
if basepath == None:
basepath = ''
if configs is None:
configs = list(self.configs)
if libname is None:
libname = module
libfile = self.libprefix + libname + self.staticlibext
if outpath is None:
outpath = self.libpath
return self.build_sources(writer, 'lib', 'multilib', module, sources, libfile, basepath, outpath, configs, includepaths, None, None, None, variables, None, externalsources)
return self.build_sources(writer, 'lib', 'multilib', module, sources, libfile, basepath, outpath, configs, includepaths, None, None, None, None, variables, None)
def sharedlib(self, writer, module, sources, libname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, outpath, externalsources):
def sharedlib(self, writer, module, sources, libname, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables, outpath = None):
built = {}
if basepath == None:
basepath = ''
if configs is None:
configs = list(self.configs)
if libname is None:
libname = module
libfile = self.libprefix + libname + self.dynamiclibext
if outpath is None:
outpath = self.binpath
return self.build_sources(writer, 'sharedlib', 'multisharedlib', module, sources, libfile, basepath, outpath, configs, includepaths, libpaths, libs, implicit_deps, variables, frameworks, externalsources)
return self.build_sources(writer, 'sharedlib', 'multisharedlib', module, sources, libfile, basepath, outpath, configs, includepaths, libpaths, dependlibs, libs, implicit_deps, variables, frameworks)
def bin(self, writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, outpath, externalsources):
def bin(self, writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables, outpath = None):
built = {}
if basepath == None:
basepath = ''
@@ -399,12 +465,12 @@ class Toolchain(object):
binfile = self.binprefix + binname + self.binext
if outpath is None:
outpath = self.binpath
return self.build_sources(writer, 'bin', 'multibin', module, sources, binfile, basepath, outpath, configs, includepaths, libpaths, libs, implicit_deps, variables, frameworks, externalsources)
return self.build_sources(writer, 'bin', 'multibin', module, sources, binfile, basepath, outpath, configs, includepaths, libpaths, dependlibs, libs, implicit_deps, variables, frameworks)
def app(self, writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, libs, frameworks, variables, resources):
def app(self, writer, module, sources, binname, basepath, configs, includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables, resources):
builtbin = []
# Filter out platforms that do not have app concept
if not (self.target.is_macosx() or self.target.is_ios() or self.target.is_android() or self.target.is_tizen()):
if not (self.target.is_macos() or self.target.is_ios() or self.target.is_android() or self.target.is_tizen()):
return builtbin
if basepath is None:
basepath = ''
@@ -413,8 +479,8 @@ class Toolchain(object):
if configs is None:
configs = list(self.configs)
for config in configs:
archbins = self.bin(writer, module, sources, binname, basepath, [config], includepaths, libpaths, implicit_deps, libs, frameworks, variables, '$buildpath')
if self.target.is_macosx() or self.target.is_ios():
archbins = self.bin(writer, module, sources, binname, basepath, [config], includepaths, libpaths, implicit_deps, dependlibs, libs, frameworks, variables, '$buildpath')
if self.target.is_macos() or self.target.is_ios():
binpath = os.path.join(self.binpath, config, binname + '.app')
builtbin += self.xcode.app(self, writer, module, archbins, self.binpath, binname, basepath, config, None, resources, True)
if self.target.is_android():

2
build/ninja/version.py
View File

@@ -70,7 +70,7 @@ def write_version_string(output_path, str):
def generate_version(libname, output_path):
generated = generate_version_string(libname)
if generated == None:
return
return
previous = read_version_string(output_path)
if generated != previous:

38
build/ninja/xcode.py
View File

@@ -21,13 +21,13 @@ class XCode(object):
self.organisation = ''
self.bundleidentifier = ''
self.provisioning = ''
if self.target.is_macosx():
if self.target.is_macos():
self.deploymenttarget = '10.7'
elif self.target.is_ios():
self.deploymenttarget = '8.0'
def build_toolchain(self):
if self.target.is_macosx():
if self.target.is_macos():
sdk = 'macosx'
deploytarget = 'MACOSX_DEPLOYMENT_TARGET=' + self.deploymenttarget
elif self.target.is_ios():
@@ -43,7 +43,7 @@ class XCode(object):
self.dsymutil = "PATH=" + localpath + " " + subprocess.check_output(['xcrun', '--sdk', sdk, '-f', 'dsymutil']).strip()
self.plistcmd = 'build/ninja/plist.py --exename $exename --prodname $prodname --bundle $bundleidentifier --target $target --deploymenttarget $deploymenttarget --output $outpath $in'
if self.target.is_macosx():
if self.target.is_macos():
self.xcassetscmd = 'mkdir -p $outpath && $xcassets --output-format human-readable-text --output-partial-info-plist $outplist' \
' --app-icon AppIcon --launch-image LaunchImage --platform macosx --minimum-deployment-target ' + self.deploymenttarget + \
' --target-device mac --compress-pngs --compile $outpath $in >/dev/null'
@@ -88,16 +88,16 @@ class XCode(object):
self.bundleidentifier = iosprefs['bundleidentifier']
if 'provisioning' in iosprefs:
self.provisioning = iosprefs['provisioning']
elif self.target.is_macosx() and 'macosx' in prefs:
macosxprefs = prefs['macosx']
if 'deploymenttarget' in macosxprefs:
self.deploymenttarget = macosxprefs['deploymenttarget']
if 'organisation' in macosxprefs:
self.organisation = macosxprefs['organisation']
if 'bundleidentifier' in macosxprefs:
self.bundleidentifier = macosxprefs['bundleidentifier']
if 'provisioning' in macosxprefs:
self.provisioning = macosxprefs['provisioning']
elif self.target.is_macos() and 'macos' in prefs:
macosprefs = prefs['macos']
if 'deploymenttarget' in macosprefs:
self.deploymenttarget = macosprefs['deploymenttarget']
if 'organisation' in macosprefs:
self.organisation = macosprefs['organisation']
if 'bundleidentifier' in macosprefs:
self.bundleidentifier = macosprefs['bundleidentifier']
if 'provisioning' in macosprefs:
self.provisioning = macosprefs['provisioning']
def write_variables(self, writer):
writer.variable('plist', self.plist)
@@ -154,14 +154,14 @@ class XCode(object):
#First build everything except plist inputs
for resource in resources:
if resource.endswith('.xcassets'):
if self.target.is_macosx():
if self.target.is_macos():
assetsvars = [('outpath', os.path.join(os.getcwd(), apppath, 'Contents', 'Resources'))]
else:
assetsvars = [('outpath', apppath)]
outplist = os.path.join(os.getcwd(), builddir, os.path.splitext(os.path.basename(resource))[0] + '-xcassets.plist')
assetsvars += [('outplist', outplist)]
outfiles = [outplist]
if self.target.is_macosx():
if self.target.is_macos():
outfiles += [os.path.join(os.getcwd(), apppath, 'Contents', 'Resources', 'AppIcon.icns')]
elif self.target.is_ios():
pass #TODO: Need to list all icon and launch image files here
@@ -169,7 +169,7 @@ class XCode(object):
has_resources = True
elif resource.endswith('.xib'):
xibmodule = binname.replace('-', '_').replace('.', '_')
if self.target.is_macosx():
if self.target.is_macos():
nibpath = os.path.join(apppath, 'Contents', 'Resources', os.path.splitext(os.path.basename(resource))[0] + '.nib')
else:
nibpath = os.path.join(apppath, os.path.splitext(os.path.basename(resource))[0] + '.nib')
@@ -186,11 +186,11 @@ class XCode(object):
#Extra output files/directories
outfiles = []
if has_resources and self.target.is_macosx():
if has_resources and self.target.is_macos():
outfiles += [os.path.join(apppath, 'Contents', 'Resources')]
#Now build input plists appending partial plists created by previous resources
if self.target.is_macosx():
if self.target.is_macos():
plistpath = os.path.join(apppath, 'Contents', 'Info.plist')
pkginfopath = os.path.join(apppath, 'Contents', 'PkgInfo')
else:
@@ -210,7 +210,7 @@ class XCode(object):
if self.provisioning != '':
codesignvars += [('provisioning', self.provisioning)]
writer.build([os.path.join(apppath, '_CodeSignature', 'CodeResources'), os.path.join(apppath, '_CodeSignature'), apppath], 'codesign', builtbin, implicit = builtres + [os.path.join('build', 'ninja', 'codesign.py')], variables = codesignvars)
elif self.target.is_macosx():
elif self.target.is_macos():
if self.provisioning != '':
codesignvars += [('provisioning', self.provisioning)]
writer.build([os.path.join(apppath, 'Contents', '_CodeSignature', 'CodeResources'), os.path.join(apppath, 'Contents', '_CodeSignature'), os.path.join(apppath, 'Contents'), apppath], 'codesign', builtbin, implicit = builtres + [os.path.join('build', 'ninja', 'codesign.py')], variables = codesignvars)

6
configure.py vendored
View File

@@ -19,3 +19,9 @@ rpmallocwrap_lib = generator.lib(module = 'rpmalloc', libname = 'rpmallocwrap',
if not target.is_windows():
rpmalloc_so = generator.sharedlib(module = 'rpmalloc', libname = 'rpmalloc', sources = ['rpmalloc.c'])
rpmallocwrap_so = generator.sharedlib(module = 'rpmalloc', libname = 'rpmallocwrap', sources = ['rpmalloc.c', 'malloc.c', 'new.cc'], variables = {'runtime': 'c++', 'defines': ['ENABLE_PRELOAD=1']})
rpmallocguards_lib = generator.lib(module = 'rpmalloc', libname = 'rpmallocguards', sources = ['rpmalloc.c'], variables = {'defines': ['ENABLE_GUARDS=1']})
if not target.is_ios() and not target.is_android():
generator.bin(module = 'test', sources = ['thread.c', 'main.c'], binname = 'rpmalloc-test', implicit_deps = [rpmalloc_lib], libs = ['rpmalloc'], includepaths = ['rpmalloc', 'test'])
generator.bin(module = 'test', sources = ['thread.c', 'main.c'], binname = 'rpmalloc-test-guards', implicit_deps = [rpmallocguards_lib], libs = ['rpmallocguards'], includepaths = ['rpmalloc', 'test'], variables = {'defines': ['ENABLE_GUARDS=1']})

30
rpmalloc/rpmalloc.c
View File

@@ -118,7 +118,7 @@
# include <assert.h>
#else
# undef assert
# define assert(x)
# define assert(x) do {} while(0)
#endif
#if ENABLE_GUARDS
@@ -1423,6 +1423,7 @@ _memory_adjust_size_class(size_t iclass) {
//! Initialize the allocator and setup global data
int
rpmalloc_initialize(void) {
memset(&_memory_config, 0, sizeof(rpmalloc_config_t));
return rpmalloc_initialize_config(0);
}
@@ -1572,6 +1573,8 @@ rpmalloc_finalize(void) {
atomic_thread_fence_release();
set_thread_heap(0);
#if defined(__APPLE__) && ENABLE_PRELOAD
pthread_key_delete(_memory_thread_heap);
#endif
@@ -1791,12 +1794,24 @@ _memory_validate_integrity(void* p) {
uint32_t* deadzone = block_start;
//If these asserts fire, you have written to memory before the block start
for (int i = 0; i < 4; ++i) {
assert(deadzone[i] == MAGIC_GUARD);
if (deadzone[i] == MAGIC_GUARD)
continue;
if (_memory_config.memory_overwrite)
_memory_config.memory_overwrite(p);
else
assert(deadzone[i] == MAGIC_GUARD && "Memory overwrite before block start");
return;
}
deadzone = (uint32_t*)pointer_offset(block_start, block_size - 16);
//If these asserts fire, you have written to memory after the block end
for (int i = 0; i < 4; ++i) {
assert(deadzone[i] == MAGIC_GUARD);
if (deadzone[i] == MAGIC_GUARD)
continue;
if (_memory_config.memory_overwrite)
_memory_config.memory_overwrite(p);
else
assert(deadzone[i] == MAGIC_GUARD && "Memory overwrite after block end");
return;
}
}
#endif
@@ -1944,7 +1959,14 @@ rpposix_memalign(void **memptr, size_t alignment, size_t size) {
size_t
rpmalloc_usable_size(void* ptr) {
return ptr ? _memory_usable_size(ptr) : 0;
size_t size = 0;
if (ptr) {
size = _memory_usable_size(ptr);
#if ENABLE_GUARDS
size -= 32;
#endif
}
return size;
}
void

2
rpmalloc/rpmalloc.h
View File

@@ -79,6 +79,8 @@ typedef struct rpmalloc_config_t {
// size. If set to 0, rpmalloc will use system calls to determine the page size. The page
// size MUST be a power of two in [512,16384] range (2^9 to 2^14).
size_t page_size;
//! Debug callback if memory guards are enabled. Called if a memory overwrite is detected
void (*memory_overwrite)(void* address);
} rpmalloc_config_t;
extern int

573
test/main.c vendored Normal file
View File

@@ -0,0 +1,573 @@
#if defined(_WIN32) && !defined(_CRT_SECURE_NO_WARNINGS)
# define _CRT_SECURE_NO_WARNINGS
#endif
#include <rpmalloc.h>
#include <thread.h>
#include <test.h>
#include <stdint.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
#ifdef _MSC_VER
# define PRIsize "Iu"
#else
# define PRIsize "zu"
#endif
#define pointer_offset(ptr, ofs) (void*)((char*)(ptr) + (ptrdiff_t)(ofs))
#define pointer_diff(first, second) (ptrdiff_t)((const char*)(first) - (const char*)(second))
#ifndef ENABLE_GUARDS
# define ENABLE_GUARDS 0
#endif
static int
test_alloc(void) {
unsigned int iloop = 0;
unsigned int ipass = 0;
unsigned int icheck = 0;
unsigned int id = 0;
void* addr[8142];
char data[20000];
unsigned int datasize[7] = { 473, 39, 195, 24, 73, 376, 245 };
rpmalloc_initialize();
for (id = 0; id < 20000; ++id)
data[id] = (char)(id % 139 + id % 17);
for (iloop = 0; iloop < 64; ++iloop) {
for (ipass = 0; ipass < 8142; ++ipass) {
addr[ipass] = rpmalloc(500);
if (addr[ipass] == 0)
return -1;
memcpy(addr[ipass], data, 500);
for (icheck = 0; icheck < ipass; ++icheck) {
if (addr[icheck] == addr[ipass])
return -1;
if (addr[icheck] < addr[ipass]) {
if (pointer_offset(addr[icheck], 500) > addr[ipass])
return -1;
}
else if (addr[icheck] > addr[ipass]) {
if (pointer_offset(addr[ipass], 500) > addr[icheck])
return -1;
}
}
}
for (ipass = 0; ipass < 8142; ++ipass) {
if (memcmp(addr[ipass], data, 500))
return -1;
}
for (ipass = 0; ipass < 8142; ++ipass)
rpfree(addr[ipass]);
}
for (iloop = 0; iloop < 64; ++iloop) {
for (ipass = 0; ipass < 1024; ++ipass) {
unsigned int cursize = datasize[ipass%7] + ipass;
addr[ipass] = rpmalloc(cursize);
if (addr[ipass] == 0)
return -1;
memcpy(addr[ipass], data, cursize);
for (icheck = 0; icheck < ipass; ++icheck) {
if (addr[icheck] == addr[ipass])
return -1;
if (addr[icheck] < addr[ipass]) {
if (pointer_offset(addr[icheck], rpmalloc_usable_size(addr[icheck])) > addr[ipass])
return -1;
}
else if (addr[icheck] > addr[ipass]) {
if (pointer_offset(addr[ipass], rpmalloc_usable_size(addr[ipass])) > addr[icheck])
return -1;
}
}
}
for (ipass = 0; ipass < 1024; ++ipass) {
unsigned int cursize = datasize[ipass%7] + ipass;
if (memcmp(addr[ipass], data, cursize))
return -1;
}
for (ipass = 0; ipass < 1024; ++ipass)
rpfree(addr[ipass]);
}
for (iloop = 0; iloop < 128; ++iloop) {
for (ipass = 0; ipass < 1024; ++ipass) {
addr[ipass] = rpmalloc(500);
if (addr[ipass] == 0)
return -1;
memcpy(addr[ipass], data, 500);
for (icheck = 0; icheck < ipass; ++icheck) {
if (addr[icheck] == addr[ipass])
return -1;
if (addr[icheck] < addr[ipass]) {
if (pointer_offset(addr[icheck], 500) > addr[ipass])
return -1;
}
else if (addr[icheck] > addr[ipass]) {
if (pointer_offset(addr[ipass], 500) > addr[icheck])
return -1;
}
}
}
for (ipass = 0; ipass < 1024; ++ipass) {
if (memcmp(addr[ipass], data, 500))
return -1;
}
for (ipass = 0; ipass < 1024; ++ipass)
rpfree(addr[ipass]);
}
rpmalloc_finalize();
printf("Memory allocation tests passed\n");
return 0;
}
typedef struct _allocator_thread_arg {
unsigned int loops;
unsigned int passes; //max 4096
unsigned int datasize[32];
unsigned int num_datasize; //max 32
void** pointers;
} allocator_thread_arg_t;
static void
allocator_thread(void* argp) {
allocator_thread_arg_t arg = *(allocator_thread_arg_t*)argp;
unsigned int iloop = 0;
unsigned int ipass = 0;
unsigned int icheck = 0;
unsigned int id = 0;
void* addr[4096];
char data[8192];
unsigned int cursize;
unsigned int iwait = 0;
int ret = 0;
rpmalloc_thread_initialize();
for (id = 0; id < 8192; ++id)
data[id] = (char)id;
thread_sleep(1);
for (iloop = 0; iloop < arg.loops; ++iloop) {
for (ipass = 0; ipass < arg.passes; ++ipass) {
cursize = 4 + arg.datasize[(iloop + ipass + iwait) % arg.num_datasize] + (iloop % 1024);
addr[ipass] = rpmalloc(4 + cursize);
if (addr[ipass] == 0) {
ret = -1;
goto end;
}
*(uint32_t*)addr[ipass] = (uint32_t)cursize;
memcpy(pointer_offset(addr[ipass], 4), data, cursize);
for (icheck = 0; icheck < ipass; ++icheck) {
if (addr[icheck] == addr[ipass]) {
ret = -1;
goto end;
}
if (addr[icheck] < addr[ipass]) {
if (pointer_offset(addr[icheck], *(uint32_t*)addr[icheck]) > addr[ipass]) {
if (pointer_offset(addr[icheck], *(uint32_t*)addr[icheck]) > addr[ipass]) {
ret = -1;
goto end;
}
}
}
else if (addr[icheck] > addr[ipass]) {
if (pointer_offset(addr[ipass], *(uint32_t*)addr[ipass]) > addr[ipass]) {
if (pointer_offset(addr[ipass], *(uint32_t*)addr[ipass]) > addr[icheck]) {
ret = -1;
goto end;
}
}
}
}
}
for (ipass = 0; ipass < arg.passes; ++ipass) {
cursize = *(uint32_t*)addr[ipass];
if (memcmp(pointer_offset(addr[ipass], 4), data, cursize)) {
ret = -1;
goto end;
}
rpfree(addr[ipass]);
}
}
rpmalloc_thread_finalize();
end:
thread_exit((uintptr_t)ret);
}
static void
crossallocator_thread(void* argp) {
allocator_thread_arg_t arg = *(allocator_thread_arg_t*)argp;
unsigned int iloop = 0;
unsigned int ipass = 0;
unsigned int cursize;
unsigned int iwait = 0;
int ret = 0;
rpmalloc_thread_initialize();
thread_sleep(1);
for (iloop = 0; iloop < arg.loops; ++iloop) {
for (ipass = 0; ipass < arg.passes; ++ipass) {
cursize = arg.datasize[(iloop + ipass + iwait) % arg.num_datasize ] + (iloop % 1024);
void* addr = rpmalloc(cursize);
if (addr == 0) {
ret = -1;
goto end;
}
arg.pointers[iloop * arg.passes + ipass] = addr;
}
}
rpmalloc_thread_finalize();
end:
thread_exit((uintptr_t)ret);
}
static void
initfini_thread(void* argp) {
allocator_thread_arg_t arg = *(allocator_thread_arg_t*)argp;
unsigned int iloop = 0;
unsigned int ipass = 0;
unsigned int icheck = 0;
unsigned int id = 0;
void* addr[4096];
char data[8192];
unsigned int cursize;
unsigned int iwait = 0;
int ret = 0;
for (id = 0; id < 8192; ++id)
data[id] = (char)id;
thread_yield();
for (iloop = 0; iloop < arg.loops; ++iloop) {
rpmalloc_thread_initialize();
for (ipass = 0; ipass < arg.passes; ++ipass) {
cursize = 4 + arg.datasize[(iloop + ipass + iwait) % arg.num_datasize] + (iloop % 1024);
addr[ipass] = rpmalloc(4 + cursize);
if (addr[ipass] == 0) {
ret = -1;
goto end;
}
*(uint32_t*)addr[ipass] = (uint32_t)cursize;
memcpy(pointer_offset(addr[ipass], 4), data, cursize);
for (icheck = 0; icheck < ipass; ++icheck) {
if (addr[icheck] == addr[ipass]) {
ret = -1;
goto end;
}
if (addr[icheck] < addr[ipass]) {
if (pointer_offset(addr[icheck], *(uint32_t*)addr[icheck]) > addr[ipass]) {
if (pointer_offset(addr[icheck], *(uint32_t*)addr[icheck]) > addr[ipass]) {
ret = -1;
goto end;
}
}
}
else if (addr[icheck] > addr[ipass]) {
if (pointer_offset(addr[ipass], *(uint32_t*)addr[ipass]) > addr[ipass]) {
if (pointer_offset(addr[ipass], *(uint32_t*)addr[ipass]) > addr[icheck]) {
ret = -1;
goto end;
}
}
}
}
}
for (ipass = 0; ipass < arg.passes; ++ipass) {
cursize = *(uint32_t*)addr[ipass];
if (memcmp(pointer_offset(addr[ipass], 4), data, cursize)) {
ret = -1;
goto end;
}
rpfree(addr[ipass]);
}
rpmalloc_thread_finalize();
}
end:
rpmalloc_thread_finalize();
thread_exit((uintptr_t)ret);
}
static int
test_threaded(void) {
uintptr_t thread[32];
uintptr_t threadres[32];
unsigned int i;
size_t num_alloc_threads;
allocator_thread_arg_t arg;
rpmalloc_initialize();
num_alloc_threads = 3;
arg.datasize[0] = 19;
arg.datasize[1] = 249;
arg.datasize[2] = 797;
arg.datasize[3] = 3;
arg.datasize[4] = 79;
arg.datasize[5] = 34;
arg.datasize[6] = 389;
arg.num_datasize = 7;
arg.loops = 4096;
arg.passes = 1024;
thread_arg targ = { allocator_thread, &arg };
for (i = 0; i < num_alloc_threads; ++i)
thread[i] = thread_run(&targ);
thread_sleep(1000);
for (i = 0; i < num_alloc_threads; ++i)
threadres[i] = thread_join(thread[i]);
rpmalloc_finalize();
for (i = 0; i < num_alloc_threads; ++i) {
if (threadres[i])
return -1;
}
printf("Memory threaded tests passed\n");
return 0;
}
static int
test_crossthread(void) {
uintptr_t thread;
allocator_thread_arg_t arg;
rpmalloc_initialize();
arg.loops = 100;
arg.passes = 1024;
arg.pointers = rpmalloc(sizeof(void*) * arg.loops * arg.passes);
arg.datasize[0] = 19;
arg.datasize[1] = 249;
arg.datasize[2] = 797;
arg.datasize[3] = 3;
arg.datasize[4] = 79;
arg.datasize[5] = 34;
arg.datasize[6] = 389;
arg.num_datasize = 7;
thread_arg targ = { crossallocator_thread, &arg };
thread = thread_run(&targ);
thread_sleep(1000);
if (thread_join(thread) != 0)
return -1;
//Off-thread deallocation
for (size_t iptr = 0; iptr < arg.loops * arg.passes; ++iptr)
rpfree(arg.pointers[iptr]);
rpfree(arg.pointers);
//Simulate thread exit
rpmalloc_thread_finalize();
rpmalloc_finalize();
printf("Memory cross thread free tests passed\n");
return 0;
}
static int
test_threadspam(void) {
uintptr_t thread[64];
uintptr_t threadres[64];
unsigned int i, j;
size_t num_passes, num_alloc_threads;
allocator_thread_arg_t arg;
rpmalloc_initialize();
num_passes = 100;
num_alloc_threads = 5;
arg.loops = 500;
arg.passes = 10;
arg.datasize[0] = 19;
arg.datasize[1] = 249;
arg.datasize[2] = 797;
arg.datasize[3] = 3;
arg.datasize[4] = 79;
arg.datasize[5] = 34;
arg.datasize[6] = 389;
arg.num_datasize = 7;
thread_arg targ = { initfini_thread, &arg };
for (i = 0; i < num_alloc_threads; ++i)
thread[i] = thread_run(&targ);
for (j = 0; j < num_passes; ++j) {
thread_sleep(10);
thread_fence();
for (i = 0; i < num_alloc_threads; ++i) {
threadres[i] = thread_join(thread[i]);
if (threadres[i])
return -1;
thread[i] = thread_run(&targ);
}
}
thread_sleep(1000);
for (i = 0; i < num_alloc_threads; ++i)
threadres[i] = thread_join(thread[i]);
rpmalloc_finalize();
for (i = 0; i < num_alloc_threads; ++i) {
if (threadres[i])
return -1;
}
printf("Memory thread spam tests passed\n");
return 0;
}
#if ENABLE_GUARDS
static int test_overwrite_detected;
static void
test_overwrite_cb(void* addr) {
++test_overwrite_detected;
}
static int
test_overwrite(void) {
int ret = 0;
char* addr;
size_t istep, size;
rpmalloc_config_t config;
memset(&config, 0, sizeof(config));
config.memory_overwrite = test_overwrite_cb;
rpmalloc_initialize_config(&config);
for (istep = 0, size = 16; size < 16 * 1024 * 1024; size <<= 1, ++istep) {
test_overwrite_detected = 0;
addr = rpmalloc(size);
*(addr - 2) = 1;
rpfree(addr);
if (!test_overwrite_detected) {
printf("Failed to detect memory overwrite before start of block in step %" PRIsize " size %" PRIsize "\n", istep, size);
ret = -1;
goto cleanup;
}
test_overwrite_detected = 0;
addr = rpmalloc(size);
*(addr + rpmalloc_usable_size(addr) + 1) = 1;
rpfree(addr);
if (!test_overwrite_detected) {
printf("Failed to detect memory overwrite after end of block in step %" PRIsize " size %" PRIsize "\n", istep, size);
ret = -1;
goto cleanup;
}
}
printf("Memory overwrite tests passed\n");
cleanup:
rpmalloc_finalize();
return ret;
}
#else
static int
test_overwrite(void) {
return 0;
}
#endif
int
test_run(int argc, char** argv) {
if (test_alloc())
return -1;
if (test_threaded())
return -1;
if (test_crossthread())
return -1;
if (test_threadspam())
return -1;
if (test_overwrite())
return -1;
return 0;
}
#if ( defined( __APPLE__ ) && __APPLE__ )
# include <TargetConditionals.h>
# if defined( __IPHONE__ ) || ( defined( TARGET_OS_IPHONE ) && TARGET_OS_IPHONE ) || ( defined( TARGET_IPHONE_SIMULATOR ) && TARGET_IPHONE_SIMULATOR )
# define NO_MAIN 1
# endif
#endif
#if !defined(NO_MAIN)
int
main(int argc, char** argv) {
return test_run(argc, argv);
}
#endif

5
test/test.h vendored Normal file
View File

@@ -0,0 +1,5 @@
#pragma once
extern int
test_run(int argc, char** argv);

106
test/thread.c vendored Normal file
View File

@@ -0,0 +1,106 @@
#include <thread.h>
#ifdef _MSC_VER
# define ATTRIBUTE_NORETURN
#else
# define ATTRIBUTE_NORETURN __attribute__((noreturn))
#endif
#ifdef _WIN32
# include <windows.h>
# include <process.h>
static unsigned __stdcall
thread_entry(void* argptr) {
thread_arg* arg = argptr;
arg->fn(arg->arg);
return 0;
}
#else
# include <time.h>
# include <pthread.h>
# include <sched.h>
static void*
thread_entry(void* argptr) {
thread_arg* arg = argptr;
arg->fn(arg->arg);
return 0;
}
#endif
uintptr_t
thread_run(thread_arg* arg) {
#ifdef _WIN32
return _beginthreadex(0, 0, thread_entry, arg, 0, 0);
#else
pthread_t id = 0;
int err = pthread_create(&id, 0, thread_entry, arg);
if (err)
return 0;
return (uintptr_t)id;
#endif
}
void ATTRIBUTE_NORETURN
thread_exit(uintptr_t value) {
#ifdef _WIN32
_endthreadex((unsigned)value);
#else
pthread_exit((void*)value);
#endif
}
uintptr_t
thread_join(uintptr_t handle) {
if (!handle)
return (uintptr_t)-1;
uintptr_t ret;
#ifdef _WIN32
DWORD exit_code = 0;
WaitForSingleObject((HANDLE)handle, INFINITE);
GetExitCodeThread((HANDLE)handle, &exit_code);
CloseHandle((HANDLE)handle);
ret = exit_code;
#else
void* result = 0;
pthread_join((pthread_t)handle, &result);
ret = (uintptr_t)result;
#endif
return ret;
}
void
thread_sleep(int milliseconds) {
#ifdef _WIN32
SleepEx(milliseconds, 1);
#else
struct timespec ts;
ts.tv_sec = milliseconds / 1000;
ts.tv_nsec = (long)(milliseconds % 1000) * 1000000L;
nanosleep(&ts, 0);
#endif
}
void
thread_yield(void) {
#ifdef _WIN32
Sleep(0);
_ReadWriteBarrier();
#else
sched_yield();
__sync_synchronize();
#endif
}
void
thread_fence(void) {
#ifdef _WIN32
_ReadWriteBarrier();
#else
__sync_synchronize();
#endif
}

27
test/thread.h vendored Normal file
View File

@@ -0,0 +1,27 @@
#include <stdint.h>
struct thread_arg {
void (*fn)(void*);
void* arg;
};
typedef struct thread_arg thread_arg;
extern uintptr_t
thread_run(thread_arg* arg);
extern void
thread_exit(uintptr_t value);
extern uintptr_t
thread_join(uintptr_t handle);
extern void
thread_sleep(int milliseconds);
extern void
thread_yield(void);
extern void
thread_fence(void);