For development statically link dcc-rs

This links the python bindings statically to libdeltachat.a if the
DCC_RS_DEV environment variable is set to the project's root.  This is
a little simpler then requiring the manual CFLAGS and LD_LIBRARY_PATH
tweaking.

It also adds a script to easily invoke the integration tests locally
without forgetting steps.

.gitignore

@@ -11,3 +11,9 @@ include
 .dc-history.txt
 *.db
 *.db-blobs
+
+python/.eggs
+python/.tox
+*.egg-info
+__pycache__
+python/src/deltachat/capi*.so

ci_scripts/run_all.sh

@@ -1,13 +1,12 @@
 #!/bin/bash
 #
-# Build the Delta Chat C/Rust library
-# typically run in a docker container that contains all library deps
-# but should also work outside if you have the dependencies installed  
-# on your system. 
+# Build the Delta Chat C/Rust library typically run in a docker
+# container that contains all library deps but should also work
+# outside if you have the dependencies installed on your system.
 
 set -e -x
 
-# perform clean build of core and install 
+# Perform clean build of core and install.
 export TOXWORKDIR=.docker-tox
 
 # install core lib
@@ -16,12 +15,11 @@ export PATH=/root/.cargo/bin:$PATH
 cargo build --release -p deltachat_ffi
 # cargo test --all --all-features
 
-# make sure subsequent compiler invocations find header and libraries
-export CFLAGS=-I`pwd`/deltachat-ffi
-export LD_LIBRARY_PATH=`pwd`/target/release
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
 
-# configure access to a base python and 
-# to several python interpreters needed by tox below
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
 export PATH=$PATH:/opt/python/cp35-cp35m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
@@ -30,23 +28,23 @@ ln -s /opt/python/cp36-cp36m/bin/python3.6
 ln -s /opt/python/cp37-cp37m/bin/python3.7
 popd
 
-if [ -n "$TESTS" ]; then 
+if [ -n "$TESTS" ]; then
 
-    pushd python 
-    # prepare a clean tox run 
+    pushd python
+    # prepare a clean tox run
     rm -rf tests/__pycache__
     rm -rf src/deltachat/__pycache__
     export PYTHONDONTWRITEBYTECODE=1
 
-    # run tox 
+    # run tox
     tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37,auditwheels
     popd
 fi
 
 
-if [ -n "$DOCS" ]; then 
+if [ -n "$DOCS" ]; then
     echo -----------------------
     echo generating python docs
     echo -----------------------
-    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
+    (cd python && tox --workdir "$TOXWORKDIR" -e doc)
 fi

python/README.rst

@@ -50,18 +50,14 @@ to core deltachat library::
     cd deltachat-core-rust
     cargo build -p deltachat_ffi --release
 
-This will result in a ``libdeltachat.so`` and ``deltachat.h`` file
-in the ``deltachat_ffi`` directory. These files are needed for
+This will result in a ``libdeltachat.so`` and ``libdeltachat.a`` files
+in the ``target/release`` directory. These files are needed for
 creating the python bindings for deltachat::
 
     cd python
-    CFLAGS=I`pwd`/../deltachat-ffi pip install -e .
+    DCC_RS_DEV=`pwd`/.. pip install -e .
 
-You then need to set the load-dynamic-library path in your shell::
-
-    export LD_LIBRARY_PATH=`pwd`/../target/release
-
-so that importing the bindings finds the correct library::
+Now test if the bindings find the correct library::
 
     python -c 'import deltachat ; print(deltachat.__version__)'
 
@@ -73,6 +69,14 @@ This should print your deltachat bindings version.
     that'd be much appreciated! please then get
     `in contact with us <https://delta.chat/en/contribute>`_.
 
+Using a system-installed deltachat-core-rust
+--------------------------------------------
+
+When calling ``pip`` without specifying the ``DCC_RS_DEV`` environment
+variable cffi will try to use a ``deltachat.h`` from a system location
+like ``/usr/local/include`` and will try to dynamically link against a
+``libdeltachat.so`` in a similar location (e.g. ``/usr/local/lib``).
+
 
 Code examples
 =============
@@ -83,13 +87,10 @@ You may look at `examples <https://py.delta.chat/examples.html>`_.
 Running tests
 =============
 
-Get a checkout of the `deltachat-core github repository`_ and type::
+Get a checkout of the `deltachat-core-rust github repository`_ and type::
 
-    cd python
-    export CFLAGS=I`pwd`/../deltachat-ffi
-    export LD_LIBRARY_PATH=`pwd`/../target/release
     pip install tox
-    tox
+    ./run-integration-tests.sh
 
 If you want to run functional tests with real
 e-mail test accounts, generate a "liveconfig" file where each
@@ -103,12 +104,14 @@ The "keyword=value" style allows to specify any
 `deltachat account config setting <https://c.delta.chat/classdc__context__t.html#aff3b894f6cfca46cab5248fdffdf083d>`_ so you can also specify smtp or imap servers, ports, ssl modes etc.
 Typically DC's automatic configuration allows to not specify these settings.
 
-You can now run tests with this ``liveconfig`` file::
+The ``run-integration-tests.sh`` script will automatically use
+``python/liveconfig`` if it exists, to manually run tests with this
+``liveconfig`` file use::
 
     tox -- --liveconfig liveconfig
 
 
-.. _`deltachat-core github repository`: https://github.com/deltachat/deltachat-core-rust
+.. _`deltachat-core-rust github repository`: https://github.com/deltachat/deltachat-core-rust
 .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
 
 

python/src/deltachat/__init__.py

@@ -2,7 +2,7 @@ from deltachat import capi, const
 from deltachat.capi import ffi
 from deltachat.account import Account  # noqa
 
-__version__ = "0.10.0dev1"
+__version__ = "0.10.0.dev2"
 
 
 _DC_CALLBACK_MAP = {}

python/src/deltachat/_build.py

@@ -1,17 +1,21 @@
 import distutils.ccompiler
+import distutils.log
 import distutils.sysconfig
 import tempfile
 import os
 import cffi
 
-# XXX hack out the header and library dirs
-# relying on CFLAGS and LD_LIBRARY_PATH being set properly is not good
-# (but we also don't want to rely on global installs of headers and libs)
-HEADERDIR = os.environ["CFLAGS"].split("-I", 1)[1]
-LIBDIR = os.environ["LD_LIBRARY_PATH"]
-
 
 def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if projdir:
+        libs = []
+        objs = [os.path.join(projdir, 'target', 'release', 'libdeltachat.a')]
+        incs = [os.path.join(projdir, 'deltachat-ffi')]
+    else:
+        libs = ['deltachat']
+        objs = []
+        incs = []
     builder = cffi.FFI()
     builder.set_source(
         'deltachat.capi',
@@ -35,9 +39,9 @@ def ffibuilder():
                 return result;
             }
         """,
-        libraries=['deltachat'],
-        include_dirs=[HEADERDIR],
-        library_dirs=[LIBDIR],
+        include_dirs=incs,
+        libraries=libs,
+        extra_objects=objs,
     )
     builder.cdef("""
         typedef int... time_t;
@@ -45,6 +49,7 @@ def ffibuilder():
         extern const char * dupstring_helper(const char* string);
         extern int dc_get_event_signature_types(int);
     """)
+    distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
     with tempfile.NamedTemporaryFile(mode='w', suffix='.h') as src_fp:
@@ -53,7 +58,7 @@ def ffibuilder():
         with tempfile.NamedTemporaryFile(mode='r') as dst_fp:
             cc.preprocess(source=src_fp.name,
                           output_file=dst_fp.name,
-                          include_dirs=[HEADERDIR],
+                          include_dirs=incs,
                           macros=[('PY_CFFI', '1')])
             builder.cdef(dst_fp.read())
     builder.cdef("""

python/tox.ini

@@ -10,15 +10,17 @@ envlist =
 commands = 
     pytest -rsXx {posargs:tests}
     pip wheel . -w {toxworkdir}/wheelhouse
- 
 passenv = 
     TRAVIS 
-    LD_LIBRARY_PATH 
-    CFLAGS
+    DCC_RS_DEV
 deps = 
     pytest
     pytest-faulthandler
     pdbpp
+    cffi
+    requests
+    attrs
+    six
 
 [testenv:auditwheels]
 skipsdist = True

run-integration-tests.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Small helper to easily run integration tests locally for development
+# purposes.  Any arguments are passed straight to tox.  E.g. to run
+# only one environment run with:
+#
+#   ./run-integration-tests.sh -e py35
+#
+# To also run with `pytest -x` use:
+#
+#   ./run-integration-tests.sh -e py35 -- -x
+
+cargo build -p deltachat_ffi --release
+
+# CFLAGS=-I`pwd`/deltachat-ffi
+# LD_LIBRARY_PATH=`pwd`/target/release
+# export CFLAGS
+# export LD_LIBRARY_PATH
+export DCC_RS_DEV=$(pwd)
+
+pushd python
+toxargs="$@"
+if [ -e liveconfig ]; then
+    toxargs="--liveconfig liveconfig $@"
+fi
+tox $toxargs
+ret=$?
+popd
+exit $ret