From 27b75103cad2f3067dc45e5835fce67de2cad163 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Mon, 8 Jun 2020 23:08:54 +0200 Subject: [PATCH] Refactor cffi build script to extract defines from header file This adds functionality to the cffi build script to also extract defines so that we can ask the compiler to figure out what the correct values are. To do this we need to be able to locate the header file used in the first place, for which we add a small utility in the header file itself guarded to only be compiled for this specific case. --- deltachat-ffi/deltachat.h | 7 + python/src/deltachat/_build.py | 229 +++++++++++++++++++++++++-------- 2 files changed, 181 insertions(+), 55 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index e2d363e6f..6cf0b895b 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -4434,6 +4434,13 @@ void dc_event_unref(dc_event_t* event); * @} */ +#ifdef PY_CFFI_INC +/* Helper utility to locate the header file when building python bindings. */ +char* _dc_header_file_location(void) { + return __FILE__; +} +#endif + #ifdef __cplusplus } diff --git a/python/src/deltachat/_build.py b/python/src/deltachat/_build.py index 25936755f..46176ab24 100644 --- a/python/src/deltachat/_build.py +++ b/python/src/deltachat/_build.py @@ -1,65 +1,64 @@ import distutils.ccompiler import distutils.log import distutils.sysconfig -import tempfile -import platform import os -import cffi +import platform +import re import shutil -from os.path import dirname as dn +import subprocess +import tempfile +import textwrap +import types from os.path import abspath +from os.path import dirname as dn + +import cffi -def ffibuilder(): - projdir = os.environ.get('DCC_RS_DEV') - if not projdir: - p = dn(dn(dn(dn(abspath(__file__))))) - projdir = os.environ["DCC_RS_DEV"] = p - target = os.environ.get('DCC_RS_TARGET', 'release') - if projdir: - if platform.system() == 'Darwin': - libs = ['resolv', 'dl'] - extra_link_args = [ - '-framework', 'CoreFoundation', - '-framework', 'CoreServices', - '-framework', 'Security', - ] - elif platform.system() == 'Linux': - libs = ['rt', 'dl', 'm'] - extra_link_args = [] - else: - raise NotImplementedError("Compilation not supported yet on Windows, can you help?") - target_dir = os.environ.get("CARGO_TARGET_DIR") - if target_dir is None: - target_dir = os.path.join(projdir, 'target') - objs = [os.path.join(target_dir, target, 'libdeltachat.a')] - assert os.path.exists(objs[0]), objs - incs = [os.path.join(projdir, 'deltachat-ffi')] +def local_build_flags(projdir, target): + """Construct build flags for building against a checkout. + + :param projdir: The root directory of the deltachat-core-rust project. + :param target: The rust build target, `debug` or `release`. + """ + flags = types.SimpleNamespace() + if platform.system() == 'Darwin': + flags.libs = ['resolv', 'dl'] + flags.extra_link_args = [ + '-framework', 'CoreFoundation', + '-framework', 'CoreServices', + '-framework', 'Security', + ] + elif platform.system() == 'Linux': + flags.libs = ['rt', 'dl', 'm'] + flags.extra_link_args = [] else: - libs = ['deltachat'] - objs = [] - incs = [] - extra_link_args = [] - builder = cffi.FFI() - builder.set_source( - 'deltachat.capi', - """ - #include - int dc_event_has_string_data(int e) - { - return DC_EVENT_DATA2_IS_STRING(e); - } - """, - include_dirs=incs, - libraries=libs, - extra_objects=objs, - extra_link_args=extra_link_args, - ) - builder.cdef(""" - typedef int... time_t; - void free(void *ptr); - extern int dc_event_has_string_data(int); - """) + raise NotImplementedError("Compilation not supported yet on Windows, can you help?") + target_dir = os.environ.get("CARGO_TARGET_DIR") + if target_dir is None: + target_dir = os.path.join(projdir, 'target') + flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')] + assert os.path.exists(flags.objs[0]), flags.objs + flags.incs = [os.path.join(projdir, 'deltachat-ffi')] + return flags + + +def system_build_flags(): + """Construct build flags for building against an installed libdeltachat.""" + flags = types.SimpleNamespace() + flags.libs = ['deltachat'] + flags.objs = [] + flags.incs = [] + flags.extra_link_args = [] + + +def extract_functions(flags): + """Extract the function definitions from deltachat.h. + + This creates a .h file with a single `#include ` line + in it. It then runs the C preprocessor to create an output file + which contains all function definitions found in `deltachat.h`. + """ distutils.log.set_verbosity(distutils.log.INFO) cc = distutils.ccompiler.new_compiler(force=True) distutils.sysconfig.customize_compiler(cc) @@ -71,13 +70,133 @@ def ffibuilder(): src_fp.write('#include ') cc.preprocess(source=src_name, output_file=dst_name, - include_dirs=incs, + include_dirs=flags.incs, macros=[('PY_CFFI', '1')]) with open(dst_name, "r") as dst_fp: - builder.cdef(dst_fp.read()) + return dst_fp.read() finally: shutil.rmtree(tmpdir) + +def find_header(flags): + """Use the compiler to find the deltachat.h header location. + + This uses a small utility in deltachat.h to find the location of + the header file location. + """ + distutils.log.set_verbosity(distutils.log.INFO) + cc = distutils.ccompiler.new_compiler(force=True) + distutils.sysconfig.customize_compiler(cc) + tmpdir = tempfile.mkdtemp() + try: + src_name = os.path.join(tmpdir, "where.c") + obj_name = os.path.join(tmpdir, "where.o") + dst_name = os.path.join(tmpdir, "where") + with open(src_name, "w") as src_fp: + src_fp.write(textwrap.dedent(""" + #include + #include + + int main(void) { + printf("%s", _dc_header_file_location()); + return 0; + } + """)) + cwd = os.getcwd() + try: + os.chdir(tmpdir) + cc.compile(sources=["where.c"], + include_dirs=flags.incs, + macros=[("PY_CFFI_INC", "1")]) + finally: + os.chdir(cwd) + cc.link_executable(objects=[obj_name], + output_progname="where", + output_dir=tmpdir) + return subprocess.check_output(dst_name) + finally: + shutil.rmtree(tmpdir) + + +def extract_defines(flags): + """Extract the required #DEFINEs from deltachat.h. + + Since #DEFINEs are interpreted by the C preprocessor we can not + use the compiler to extract these and need to parse the header + file ourselves. + + The defines are returned in a string that can be passed to CFFIs + cdef() method. + """ + header = find_header(flags) + defines_re = re.compile(r""" + \#define\s+ # The start of a define. + ( # Begin capturing group which captures the define name. + (?: # A nested group which is not captured, this allows us + # to build the list of prefixes to extract without + # creation another capture group. + DC_EVENT + | DC_QR + | DC_MSG + | DC_LP + | DC_EMPTY + | DC_CERTCK + | DC_STATE + | DC_STR + | DC_CONTACT_ID + | DC_GCL + | DC_CHAT + | DC_PROVIDER + | DC_KEY_GEN + ) # End of prefix matching + _[A-Z_]+ # Match the suffix, e.g. _TEXT in DC_MSG_TEXT + ) # Close the capturing group, this contains + # the entire name e.g. DC_MSG_TEXT. + \s+\S+ # Ensure there is whitespace followed by a value. + """, re.VERBOSE) + defines = [] + with open(header) as fp: + for line in fp: + match = defines_re.match(line) + if match: + defines.append(match.group(1)) + return '\n'.join('#define {} ...'.format(d) for d in defines) + + +def ffibuilder(): + projdir = os.environ.get('DCC_RS_DEV') + if not projdir: + p = dn(dn(dn(dn(abspath(__file__))))) + projdir = os.environ["DCC_RS_DEV"] = p + target = os.environ.get('DCC_RS_TARGET', 'release') + if projdir: + flags = local_build_flags(projdir, target) + else: + flags = system_build_flags() + builder = cffi.FFI() + builder.set_source( + 'deltachat.capi', + """ + #include + int dc_event_has_string_data(int e) + { + return DC_EVENT_DATA2_IS_STRING(e); + } + """, + include_dirs=flags.incs, + libraries=flags.libs, + extra_objects=flags.objs, + extra_link_args=flags.extra_link_args, + ) + builder.cdef(""" + typedef int... time_t; + void free(void *ptr); + extern int dc_event_has_string_data(int); + """) + function_defs = extract_functions(flags) + defines = extract_defines(flags) + builder.cdef(function_defs) + builder.cdef(defines) return builder