diff --git a/ci_scripts/ci_run.sh b/ci_scripts/ci_run.sh index 3291bd10f..a8c38e5f6 100755 --- a/ci_scripts/ci_run.sh +++ b/ci_scripts/ci_run.sh @@ -16,7 +16,7 @@ export BRANCH=${CIRCLE_BRANCH:-test7} #fi # run everything else inside docker (TESTS, DOCS, WHEELS) -docker run -e BRANCH -e TESTS -e DOCS \ +docker run -e DCC_PY_LIVECONFIG -e BRANCH -e TESTS -e DOCS \ --rm -it -v $(pwd):/mnt -w /mnt \ deltachat/coredeps ci_scripts/run_all.sh diff --git a/ci_scripts/docker-coredeps/deps/build_rust.sh b/ci_scripts/docker-coredeps/deps/build_rust.sh index 800ea81f2..d1d678348 100755 --- a/ci_scripts/docker-coredeps/deps/build_rust.sh +++ b/ci_scripts/docker-coredeps/deps/build_rust.sh @@ -3,9 +3,9 @@ set -e -x # Install Rust -curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-04-19 -y +curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-07-10 -y export PATH=/root/.cargo/bin:$PATH rustc --version # remove some 300-400 MB that we don't need for automated builds -rm -rf /root/.rustup/toolchains/nightly-2019-04-19-x86_64-unknown-linux-gnu/share/ +rm -rf /root/.rustup/toolchains/nightly-2019-07-10-x86_64-unknown-linux-gnu/share/ diff --git a/ci_scripts/run_all.sh b/ci_scripts/run_all.sh index da8cf9c4c..542d37f36 100755 --- a/ci_scripts/run_all.sh +++ b/ci_scripts/run_all.sh @@ -37,6 +37,10 @@ if [ -n "$TESTS" ]; then export PYTHONDONTWRITEBYTECODE=1 # run tox + # XXX we don't run liveconfig tests because they hang sometimes + # see https://github.com/deltachat/deltachat-core-rust/issues/331 + unset DCC_PY_LIVECONFIG + tox --workdir "$TOXWORKDIR" -e lint,py27,py35,py36,py37,auditwheels popd fi diff --git a/python/README.rst b/python/README.rst index 4fec44e5a..5b208cda4 100644 --- a/python/README.rst +++ b/python/README.rst @@ -39,6 +39,12 @@ and push them to a python package index. To install the latest github ``master`` pip install -i https://m.devpi.net/dc/master deltachat +.. note:: + + If you can help to automate the building of wheels for Mac or Windows, + that'd be much appreciated! please then get + `in contact with us `_. + Installing bindings from source =============================== @@ -48,34 +54,56 @@ to core deltachat library:: git clone https://github.com/deltachat/deltachat-core-rust cd deltachat-core-rust - cargo build -p deltachat_ffi --release - -This will result in a ``libdeltachat.so`` and ``libdeltachat.a`` files -in the ``target/release`` directory. These files are needed for -creating the python bindings for deltachat:: - cd python - DCC_RS_DEV=`pwd`/.. pip install -e . -Now test if the bindings find the correct library:: +It is always a good idea to create a python "virtualenv". +Install "virtualenv" on your system and run:: - python -c 'import deltachat ; print(deltachat.__version__)' + virtualenv venv + source venv/bin/activate -This should print your deltachat bindings version. +Afterwards ``which python`` tells you that it comes out of the "venv" +directory that contains all python install artifacts. Let's first +install test tools:: + + pip install pytest pytest-timeout pytest-faulthandler requests + +then cargo-build and install the deltachat bindings:: + + python install_python_bindings.py + +The bindings will be installed in release mode but with debug symbols. +The release mode is neccessary because some tests generate RSA keys +which is prohibitively slow in debug mode. + +After succcessul binding installation you can finally run the tests:: + + pytest -v tests .. note:: - If you can help to automate the building of wheels for Mac or Windows, - that'd be much appreciated! please then get - `in contact with us `_. + Some tests are sometimes failing/hanging because of + https://github.com/deltachat/deltachat-core-rust/issues/331 + and + https://github.com/deltachat/deltachat-core-rust/issues/326 -Using a system-installed deltachat-core-rust --------------------------------------------- -When calling ``pip`` without specifying the ``DCC_RS_DEV`` environment -variable cffi will try to use a ``deltachat.h`` from a system location -like ``/usr/local/include`` and will try to dynamically link against a -``libdeltachat.so`` in a similar location (e.g. ``/usr/local/lib``). +running "live" tests (experimental) +----------------------------------- + +If you want to run "liveconfig" functional tests you can set +``DCC_PY_LIVECONFIG`` to: + +- a particular https-url that you can ask for from the delta + chat devs. + +- or the path of a file that contains two lines, each describing + via "addr=... mail_pwd=..." a test account login that will + be used for the live tests. + +With ``DCC_PY_LIVECONFIG`` set pytest invocations will use real +e-mail accounts and run through all functional "liveconfig" tests. + Code examples @@ -84,68 +112,34 @@ Code examples You may look at `examples `_. -Running tests -============= - -Get a checkout of the `deltachat-core-rust github repository`_ and type:: - - pip install tox - ./run-integration-tests.sh - -If you want to run functional tests with real -e-mail test accounts, generate a "liveconfig" file where each -lines contains test account settings, for example:: - - # 'liveconfig' file specifying imap/smtp accounts - addr=some-email@example.org mail_pw=password - addr=other-email@example.org mail_pw=otherpassword - -The "keyword=value" style allows to specify any -`deltachat account config setting `_ so you can also specify smtp or imap servers, ports, ssl modes etc. -Typically DC's automatic configuration allows to not specify these settings. - -The ``run-integration-tests.sh`` script will automatically use -``python/liveconfig`` if it exists, to manually run tests with this -``liveconfig`` file use:: - - tox -- --liveconfig liveconfig - - .. _`deltachat-core-rust github repository`: https://github.com/deltachat/deltachat-core-rust .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust -Running test using a debug build --------------------------------- - -If you need to examine e.g. a coredump you may want to run the tests -using a debug build:: - - DCC_RS_TARGET=debug ./run-integration-tests.sh -e py37 -- -x -v -k failing_test - Building manylinux1 wheels ========================== +.. note:: + + This section may not fully work. + Building portable manylinux1 wheels which come with libdeltachat.so and all it's dependencies is easy using the provided docker tooling. using docker pull / premade images ------------------------------------ -We publish a build environment under the ``deltachat/wheel`` tag so +We publish a build environment under the ``deltachat/coredeps`` tag so that you can pull it from the ``hub.docker.com`` site's "deltachat" organization:: - $ docker pull deltachat/wheel + $ docker pull deltachat/coredeps -The ``deltachat/wheel`` image can be used to build both libdeltachat.so -and the Python wheels:: +This docker image can be used to run tests and build Python wheels for all interpreters:: - $ docker run --rm -it -v $(pwd):/io/ deltachat/wheel /io/python/wheelbuilder/build-wheels.sh + $ bash ci_scripts/ci_run.sh -This command runs a script within the image, after mounting ``$(pwd)`` as ``/io`` within -the docker image. The script is specified as a path within the docker image's filesystem. -The resulting wheel files will be in ``python/wheelhouse``. +This command runs tests and build-wheel scripts in a docker container. Optionally build your own docker image @@ -154,10 +148,10 @@ Optionally build your own docker image If you want to build your own custom docker image you can do this:: $ cd deltachat-core # cd to deltachat-core checkout directory - $ docker build -t deltachat/wheel python/wheelbuilder/ + $ docker build -t deltachat/coredeps ci_scripts/docker_coredeps -This will use the ``python/wheelbuilder/Dockerfile`` to build -up docker image called ``deltachat/wheel``. You can afterwards +This will use the ``ci_scripts/docker_coredeps/Dockerfile`` to build +up docker image called ``deltachat/coredeps``. You can afterwards find it with:: $ docker images diff --git a/python/install_python_bindings.py b/python/install_python_bindings.py index 8effe5c4b..30e766d59 100755 --- a/python/install_python_bindings.py +++ b/python/install_python_bindings.py @@ -6,29 +6,18 @@ import os import subprocess +import os if __name__ == "__main__": os.environ["DCC_RS_TARGET"] = target = "release" + if "DCC_RS_DEV" not in os.environ: + dn = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + os.environ["DCC_RS_DEV"] = dn - toml = os.path.join(os.getcwd(), "..", "Cargo.toml") - assert os.path.exists(toml) - with open(toml) as f: - s = orig = f.read() - s += "\n" - s += "[profile.release]\n" - s += "debug = true\n" - with open(toml, "w") as f: - f.write(s) - print("temporarily modifying Cargo.toml to provide release build with debug symbols ") - try: - subprocess.check_call([ - "cargo", "build", "-p", "deltachat_ffi", "--" + target - ]) - finally: - with open(toml, "w") as f: - f.write(orig) - print("\nreseted Cargo.toml to previous original state") - + os.environ["RUSTFLAGS"] = "-g" + subprocess.check_call([ + "cargo", "build", "-p", "deltachat_ffi", "--" + target + ]) subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True) subprocess.check_call([ diff --git a/python/src/deltachat/_build.py b/python/src/deltachat/_build.py index 2d2c0b0f7..69820b1db 100644 --- a/python/src/deltachat/_build.py +++ b/python/src/deltachat/_build.py @@ -6,10 +6,15 @@ import platform import os import cffi import shutil +from os.path import dirname as dn +from os.path import abspath def ffibuilder(): projdir = os.environ.get('DCC_RS_DEV') + if not projdir: + p = dn(dn(dn(dn(abspath(__file__))))) + projdir = os.environ["DCC_RS_DEV"] = p target = os.environ.get('DCC_RS_TARGET', 'release') if projdir: if platform.system() == 'Darwin': diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 777ce0eed..342fe8293 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -50,8 +50,9 @@ class Account(object): self._configkeys = self.get_config("sys.config_keys").split() self._imex_completed = threading.Event() - def __del__(self): - self.shutdown() + # XXX this can cause "illegal instructions" at test ends so we omit it for now + # def __del__(self): + # self.shutdown() def _check_config_key(self, name): if name not in self._configkeys: diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 2ee4ad7fd..b97370093 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -1,9 +1,9 @@ from __future__ import print_function import os import pytest +import requests import time from deltachat import Account -from deltachat import props from deltachat.capi import lib import tempfile @@ -36,6 +36,8 @@ def pytest_runtest_call(item): def pytest_report_header(config, startdir): + summary = [] + t = tempfile.mktemp() try: ac = Account(t, eventlogging=False) @@ -43,13 +45,18 @@ def pytest_report_header(config, startdir): ac.shutdown() finally: os.remove(t) - summary = ['Deltachat core={} sqlite={}'.format( + summary.extend(['Deltachat core={} sqlite={}'.format( info['deltachat_core_version'], info['sqlite_version'], - )] - cfg = config.getoption('--liveconfig') + )]) + + cfg = config.option.liveconfig if cfg: - summary.append('Liveconfig: {}'.format(os.path.abspath(cfg))) + if "#" in cfg: + url, token = cfg.split("#", 1) + summary.append('Liveconfig provider: {}#'.format(url)) + else: + summary.append('Liveconfig file: {}'.format(cfg)) return summary @@ -66,9 +73,56 @@ def data(): return Data() +class SessionLiveConfigFromFile: + def __init__(self, fn): + self.fn = fn + self.configlist = [] + for line in open(fn): + if line.strip() and not line.strip().startswith('#'): + d = {} + for part in line.split(): + name, value = part.split("=") + d[name] = value + self.configlist.append(d) + + def get(self, index): + return self.configlist[index] + + def exists(self): + return bool(self.configlist) + + +class SessionLiveConfigFromURL: + def __init__(self, url, create_token): + self.configlist = [] + for i in range(2): + res = requests.post(url, json={"token_create_user": int(create_token)}) + if res.status_code != 200: + pytest.skip("creating newtmpuser failed {!r}".format(res)) + d = res.json() + config = dict(addr=d["email"], mail_pw=d["password"]) + self.configlist.append(config) + + def get(self, index): + return self.configlist[index] + + def exists(self): + return bool(self.configlist) + + +@pytest.fixture(scope="session") +def session_liveconfig(request): + liveconfig_opt = request.config.option.liveconfig + if liveconfig_opt: + if liveconfig_opt.startswith("http"): + url, create_token = liveconfig_opt.split("#", 1) + return SessionLiveConfigFromURL(url, create_token) + else: + return SessionLiveConfigFromFile(liveconfig_opt) + + @pytest.fixture -def acfactory(pytestconfig, tmpdir, request): - fn = pytestconfig.getoption("--liveconfig") +def acfactory(pytestconfig, tmpdir, request, session_liveconfig): class AccountMaker: def __init__(self): @@ -82,18 +136,6 @@ def acfactory(pytestconfig, tmpdir, request): fin = self._finalizers.pop() fin() - @props.cached - def configlist(self): - configlist = [] - for line in open(fn): - if line.strip() and not line.strip().startswith('#'): - d = {} - for part in line.split(): - name, value = part.split("=") - d[name] = value - configlist.append(d) - return configlist - def get_unconfigured_account(self): self.offline_count += 1 tmpdb = tmpdir.join("offlinedb%d" % self.offline_count) @@ -116,10 +158,10 @@ def acfactory(pytestconfig, tmpdir, request): return ac def get_online_configuring_account(self): - if not fn: - pytest.skip("specify a --liveconfig file to run tests with real accounts") + if not session_liveconfig: + pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig") + configdict = session_liveconfig.get(self.live_count) self.live_count += 1 - configdict = self.configlist.pop(0) if "e2ee_enabled" not in configdict: configdict["e2ee_enabled"] = "1" tmpdb = tmpdir.join("livedb%d" % self.live_count) diff --git a/python/tox.ini b/python/tox.ini index bd350c5d3..55f2d3712 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -8,7 +8,7 @@ envlist = [testenv] commands = - pytest -v -rsXx {posargs:tests} + pytest -s -v -rsXx {posargs:tests} python tests/package_wheels.py {toxworkdir}/wheelhouse passenv = TRAVIS @@ -19,6 +19,7 @@ deps = pytest pytest-faulthandler pdbpp + requests [testenv:auditwheels] skipsdist = True @@ -51,6 +52,7 @@ commands = [pytest] +addopts = -v -rs python_files = tests/test_*.py norecursedirs = .tox xfail_strict=true diff --git a/run-integration-tests.sh b/run-integration-tests.sh index 8633bdd77..d92f516f7 100755 --- a/run-integration-tests.sh +++ b/run-integration-tests.sh @@ -23,7 +23,7 @@ if [ $? != 0 ]; then fi pushd python -if [ -e "./liveconfig" ]; then +if [ -e "./liveconfig" && -z "$DCC_PY_LIVECONFIG" ]; then export DCC_PY_LIVECONFIG=liveconfig fi tox "$@"