diff --git a/.travis.sh b/.travis.sh
index dd794e8cd..b6bc015e1 100755
--- a/.travis.sh
+++ b/.travis.sh
@@ -20,12 +20,12 @@ if [ "$1" == "tests" ]; then
 	cd src
 	python manage.py check
 	make all compress
-	coverage run -m py.test tests && coverage report
+	coverage run -m py.test --rerun 5 tests && coverage report
 fi
 if [ "$1" == "tests-cov" ]; then
 	pip3 install -r src/requirements.txt -Ur src/requirements/dev.txt -r src/requirements/py34.txt
 	cd src
 	python manage.py check
 	make all compress
-	coverage run -m py.test tests && coveralls
+	coverage run -m py.test --rerun 5 tests && coveralls
 fi
diff --git a/doc/development/setup.rst b/doc/development/setup.rst
index 0d842ddcb..83aeb7098 100644
--- a/doc/development/setup.rst
+++ b/doc/development/setup.rst
@@ -92,11 +92,14 @@ Before you check in your code into git, always run the static checkers and unit
     python manage.py check
     py.test
 
-If you have multiple CPU cores and want to speed up the test suite, you can install the python
-package ``pytest-xdist`` using ``pip install pytest-xdist`` and then run ``py.test -n NUM`` with
-``NUM`` being the number of threads you want to use.
+.. note:: If you have multiple CPU cores and want to speed up the test suite, you can install the python
+          package ``pytest-xdist`` using ``pip install pytest-xdist`` and then run ``py.test -n NUM`` with
+          ``NUM`` being the number of threads you want to use.
 
-It is therefore a good idea to put this command into your git hook ``.git/hooks/pre-commit``,
+.. note:: We currently have some flaky tests that we haven't yet been able to debug. If you just run
+          ``py.test --rerun 5`` they'll be rerun and only reported as failures if they fail repeatedly.
+
+It is a good idea to put this command into your git hook ``.git/hooks/pre-commit``,
 for example::
 
     #!/bin/sh
diff --git a/src/requirements/dev.txt b/src/requirements/dev.txt
index dcdc1095e..bc512b756 100644
--- a/src/requirements/dev.txt
+++ b/src/requirements/dev.txt
@@ -11,3 +11,4 @@ pytest==2.9.*
 pytest-django
 isort
 pytest-mock
+pytest-rerunfailures