Compare commits

...

20 Commits

Author SHA1 Message Date
Mira Weller
b23fdf2000 rename 2024-10-25 18:01:11 +02:00
Mira
80d1dd16f4 Merge branch 'master' into hacky-debug-output 2024-10-25 17:57:06 +02:00
dependabot[bot]
82ed8abecc Update fakeredis requirement from ==2.24.* to ==2.26.* (#4551)
Updates the requirements on [fakeredis](https://github.com/cunla/fakeredis-py) to permit the latest version.
- [Release notes](https://github.com/cunla/fakeredis-py/releases)
- [Commits](https://github.com/cunla/fakeredis-py/compare/v2.24.0...v2.26.0)

---
updated-dependencies:
- dependency-name: fakeredis
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-10-25 17:55:13 +02:00
Raphael Michel
54f077665c CI: Remove mentions of travis CI, we have migrated away years ago (#4562) 2024-10-25 17:47:04 +02:00
Mira Weller
c0849c9d9c disable reruns 2024-10-25 17:33:36 +02:00
Raphael Michel
e3f6b3f70a pytest: Fix failure with fakeserver and xdist 2024-10-25 17:30:47 +02:00
Raphael Michel
482a66c546 CI: Run PostgreSQL as a native service (#4560)
* GH Actions: Fix PostgreSQL issues

* Map PostgreSQL port
2024-10-25 17:18:42 +02:00
Mira
67934e51fc Merge branch 'master' into hacky-debug-output 2024-10-25 17:08:02 +02:00
Raphael Michel
e4cef6e46b CI: Make sure apt is non-interactive (to not break act) (#4559) 2024-10-25 17:06:11 +02:00
Raphael Michel
cbee1b71fe CI: Install dependencies with uv for speedup (#4558) 2024-10-25 17:05:58 +02:00
Raphael Michel
0cd1290624 Tests: Fix two flaky tests (#4557)
* Test: Fix test that relies on unstable database result order

* Tests: Fix test that fails between 0am and 1am
2024-10-25 17:04:16 +02:00
Mira Weller
d4ea0990b3 readd matrix 2024-10-25 17:03:55 +02:00
Raphael Michel
9b4506d5e5 Speed up process with uv 2024-10-25 17:02:22 +02:00
Mira Weller
f559cd2618 flush, always print 2024-10-25 17:01:33 +02:00
Raphael Michel
565f5e2ea7 Tests: Run fakeredis on different virtual ports per pytest worker (#4555) 2024-10-25 16:44:22 +02:00
Mira Weller
33991cfb77 add some hacky debug output 2024-10-25 16:43:21 +02:00
Mira Weller
fe12eea591 Revert "Update fakeredis requirement from ==2.24.* to ==2.26.* (#4551)"
This reverts commit cf3087453c.
2024-10-25 16:06:21 +02:00
Mira Weller
7000d4897d Revert "Fix fakeredis usage"
This reverts commit 39c3aef7bc.
2024-10-25 16:06:20 +02:00
Mira Weller
e6337e07e1 Revert "pytest: Fix failure with fakeserver and xdist"
This reverts commit b46c0eba0c.
2024-10-25 16:06:19 +02:00
Raphael Michel
b46c0eba0c pytest: Fix failure with fakeserver and xdist 2024-10-25 14:32:28 +02:00
15 changed files with 90 additions and 33 deletions

View File

@@ -38,7 +38,7 @@ jobs:
restore-keys: |
${{ runner.os }}-pip-
- name: Install system dependencies
run: sudo apt update && sudo apt install gettext unzip
run: sudo apt update && sudo apt install -y gettext unzip
- name: Install Python dependencies
run: pip3 install -U setuptools build pip check-manifest
- name: Run check-manifest

View File

@@ -37,7 +37,7 @@ jobs:
restore-keys: |
${{ runner.os }}-pip-
- name: Install system packages
run: sudo apt update && sudo apt install enchant-2 hunspell aspell-en
run: sudo apt update && sudo apt install -y enchant-2 hunspell aspell-en
- name: Install Dependencies
run: pip3 install -Ur requirements.txt
working-directory: ./doc

View File

@@ -35,9 +35,9 @@ jobs:
restore-keys: |
${{ runner.os }}-pip-
- name: Install system packages
run: sudo apt update && sudo apt install gettext
run: sudo apt update && sudo apt -y install gettext
- name: Install Dependencies
run: pip3 install -e ".[dev]"
run: pip3 install uv && uv pip install --system -e ".[dev]"
- name: Compile messages
run: python manage.py compilemessages
working-directory: ./src
@@ -62,7 +62,7 @@ jobs:
- name: Install system packages
run: sudo apt update && sudo apt install enchant-2 hunspell hunspell-de-de aspell-en aspell-de
- name: Install Dependencies
run: pip3 install -e ".[dev]"
run: pip3 install uv && uv pip install --system -e ".[dev]"
- name: Spellcheck translations
run: potypo
working-directory: ./src

View File

@@ -35,7 +35,7 @@ jobs:
restore-keys: |
${{ runner.os }}-pip-
- name: Install Dependencies
run: pip3 install -e ".[dev]" psycopg2-binary
run: pip3 install uv && uv pip install --system -e ".[dev]" psycopg2-binary
- name: Run isort
run: isort -c .
working-directory: ./src
@@ -55,7 +55,7 @@ jobs:
restore-keys: |
${{ runner.os }}-pip-
- name: Install Dependencies
run: pip3 install -e ".[dev]" psycopg2-binary
run: pip3 install uv && uv pip install --system -e ".[dev]" psycopg2-binary
- name: Run flake8
run: flake8 .
working-directory: ./src

View File

@@ -22,23 +22,30 @@ jobs:
runs-on: ubuntu-22.04
name: Tests
strategy:
fail-fast: false
matrix:
python-version: ["3.9", "3.10", "3.11"]
database: [sqlite, postgres]
database: [ postgres]
exclude:
- database: sqlite
python-version: "3.9"
- database: sqlite
python-version: "3.10"
services:
postgres:
image: postgres:15
env:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: pretix
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- uses: actions/checkout@v4
- uses: harmon758/postgresql-action@v1
with:
postgresql version: '15'
postgresql db: 'pretix'
postgresql user: 'postgres'
postgresql password: 'postgres'
if: matrix.database == 'postgres'
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
@@ -50,9 +57,9 @@ jobs:
restore-keys: |
${{ runner.os }}-pip-
- name: Install system dependencies
run: sudo apt update && sudo apt install gettext
run: sudo apt update && sudo apt install -y gettext
- name: Install Python dependencies
run: pip3 install --ignore-requires-python -e ".[dev]" psycopg2-binary # We ignore that flake8 needs newer python as we don't run flake8 during tests
run: pip3 install uv && uv pip install --system -e ".[dev]" psycopg2-binary
- name: Run checks
run: python manage.py check
working-directory: ./src
@@ -64,10 +71,16 @@ jobs:
run: make all compress
- name: Run tests
working-directory: ./src
run: PRETIX_CONFIG_FILE=tests/travis_${{ matrix.database }}.cfg py.test -n 3 -p no:sugar --cov=./ --cov-report=xml --reruns 3 tests --maxfail=100
run: PRETIX_CONFIG_FILE=tests/ci_${{ matrix.database }}.cfg py.test -n 3 -p no:sugar --cov=./ --cov-report=xml --reruns 0 tests --maxfail=100 || true
# XXXXXXXXXXXXXX for test only
- name: print debug output
working-directory: ./src
run: cat /tmp/test.txt
- name: Run concurrency tests
working-directory: ./src
run: PRETIX_CONFIG_FILE=tests/travis_${{ matrix.database }}.cfg py.test tests/concurrency_tests/ --reruns 0 --reuse-db
run: PRETIX_CONFIG_FILE=tests/ci_${{ matrix.database }}.cfg py.test tests/concurrency_tests/ --reruns 0 --reuse-db
if: matrix.database == 'postgres'
- name: Upload coverage
uses: codecov/codecov-action@v1

View File

@@ -10,7 +10,7 @@ tests:
- cd src
- python manage.py check
- make all compress
- PRETIX_CONFIG_FILE=tests/travis_sqlite.cfg py.test --reruns 3 -n 3 tests --maxfail=100
- PRETIX_CONFIG_FILE=tests/ci_sqlite.cfg py.test --reruns 3 -n 3 tests --maxfail=100
except:
- pypi
pypi:

View File

@@ -156,7 +156,7 @@ def event_list(request):
max_fromto=Greatest(Max('subevents__date_to'), Max('subevents__date_from'))
).annotate(
order_from=Coalesce('min_from', 'date_from'),
).order_by('-order_from')
).order_by('-order_from', 'slug')
total = qs.count()
pagesize = 20
@@ -318,7 +318,7 @@ def nav_context_list(request):
max_fromto=Greatest(Max('subevents__date_to'), Max('subevents__date_from'))
).annotate(
order_from=Coalesce('min_from', 'date_from'),
).order_by('-order_from')
).order_by('-order_from', 'slug')
if request.user.has_active_staff_session(request.session.session_key):
qs_orga = Organizer.objects.all()

View File

@@ -210,7 +210,7 @@ class EventListMixin:
)
).annotate(
order_to=Coalesce('max_fromto', 'max_to', 'max_from', 'date_to', 'date_from'),
).order_by('-order_to')
).order_by('-order_to', 'name', 'slug')
else:
date_q = Q(date_to__gte=now()) | (Q(date_to__isnull=True) & Q(date_from__gte=now()))
qs = qs.filter(
@@ -219,7 +219,7 @@ class EventListMixin:
)
).annotate(
order_from=Coalesce('min_from', 'date_from'),
).order_by('order_from')
).order_by('order_from', 'name', 'slug')
qs = Event.annotated(filter_qs_by_attr(
qs, self.request, match_subevents_with_conditions=Q(active=True) & Q(is_public=True) & date_q
), self.request.sales_channel)

View File

@@ -19,6 +19,7 @@
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <https://www.gnu.org/licenses/>.
#
import os
from contextlib import contextmanager
import fakeredis
@@ -38,4 +39,9 @@ def mocker_context():
def get_redis_connection(alias="default", write=True):
return fakeredis.FakeStrictRedis(server=fakeredis.FakeServer.get_server("127.0.0.1:None:v(7, 0)", (7, 0), server_type="redis"))
worker_id = os.environ.get("PYTEST_XDIST_WORKER")
if worker_id.startswith("gw"):
redis_port = 1000 + int(worker_id.replace("gw", ""))
else:
redis_port = 1000
return fakeredis.FakeStrictRedis(server=fakeredis.FakeServer.get_server(f"127.0.0.1:{redis_port}:redis:v(7, 0)", (7, 0), server_type="redis"))

View File

@@ -95,5 +95,5 @@ class DisableMigrations(object):
return None
if not os.environ.get("TRAVIS", "") and not os.environ.get("GITHUB_WORKFLOW", ""):
if not os.environ.get("GITHUB_WORKFLOW", ""):
MIGRATION_MODULES = DisableMigrations()

View File

@@ -451,7 +451,7 @@ def test_order_create_invoice(token_client, organizer, event, order):
"invoice_to_vat_id": "DE123",
"invoice_to_beneficiary": "",
"custom_field": None,
'date': now().date().isoformat(),
'date': now().astimezone(event.timezone).date().isoformat(),
'refers': None,
'locale': 'en',
'introductory_text': '',

View File

@@ -20,8 +20,11 @@
# <https://www.gnu.org/licenses/>.
#
import inspect
import os
import time
import pytest
from django.db import connection
from django.test import override_settings
from django.utils import translation
from django_scopes import scopes_disabled
@@ -36,8 +39,8 @@ CRASHED_ITEMS = set()
@pytest.hookimpl(trylast=True)
def pytest_configure(config):
"""
Somehow, somewhere, our test suite causes a segfault in SQLite, but only when run
on Travis CI in full. Therefore, we monkeypatch pytest-xdist to retry segfaulted
Somehow, somewhere, our test suite causes a segfault in SQLite in the past, but only when run
on CI in full. Therefore, we monkeypatch pytest-xdist to retry segfaulted
tests and keep fingers crossed that this doesn't turn into an infinite loop.
"""
@@ -82,27 +85,32 @@ def reset_locale():
@pytest.fixture
def fakeredis_client(monkeypatch):
worker_id = os.environ.get("PYTEST_XDIST_WORKER")
if worker_id.startswith("gw"):
redis_port = 1000 + int(worker_id.replace("gw", ""))
else:
redis_port = 1000
with override_settings(
HAS_REDIS=True,
REAL_CACHE_USED=True,
CACHES={
'redis': {
'BACKEND': 'django.core.cache.backends.redis.RedisCache',
'LOCATION': 'redis://127.0.0.1',
'LOCATION': f'redis://127.0.0.1:{redis_port}',
'OPTIONS': {
'connection_class': FakeConnection
}
},
'redis_session': {
'BACKEND': 'django.core.cache.backends.redis.RedisCache',
'LOCATION': 'redis://127.0.0.1',
'LOCATION': f'redis://127.0.0.1:{redis_port}',
'OPTIONS': {
'connection_class': FakeConnection
}
},
'default': {
'BACKEND': 'django.core.cache.backends.redis.RedisCache',
'LOCATION': 'redis://127.0.0.1',
'LOCATION': f'redis://127.0.0.1:{redis_port}',
'OPTIONS': {
'connection_class': FakeConnection
}
@@ -113,3 +121,33 @@ def fakeredis_client(monkeypatch):
redis.flushall()
monkeypatch.setattr('django_redis.get_redis_connection', get_redis_connection, raising=False)
yield redis
# XXXXXXXXXXXXXXXXXXX for test only
f = open("/tmp/test.txt","w")
if os.environ.get("GITHUB_WORKFLOW", ""):
@pytest.fixture(autouse=True)
def ensure_healthy_connection(request, worker_id):
# We have no idea why this is neccessary. It shouldn't be, and it costs some performance.
# However, in ~August 2024 our tests became really flake on GitHub Actions (failing more than 80% of the time)
# for no apparent reason with some error messages related to PostgreSQL connection issues. This appears to
# work around it...
# Check if the test even has DB access
marker = request.node.get_closest_marker("django_db")
f.write(str(time.time())+"\t"+ str(worker_id)+"\t"+str(request.path)+"\t"+ str(request.module.__name__)+"\t"+ str(request.function.__name__)+"\tstart\n")
f.flush()
# Run actual test
yield
f.write(str(time.time())+"\t"+ str(worker_id)+"\t"+str(request.path)+"\t"+ str(request.module.__name__)+"\t"+ str(request.function.__name__)+"\tend\n")
f.flush()
# If yes, do a dummy query at the end of the test
#if marker:
# with connection.cursor() as cursor:
# cursor.execute("SELECT 1")

View File

@@ -29,7 +29,7 @@ def test_crash():
"""
This is a test that crashes with SIGKILL every (n+1)-th time it runs (n = 0, 1, 2, …).
This is useful for debugging our pytest-xdist monkeypatch that we apply in conftest.py
to deal with random test crashes on Travis CI using SQLite. Usually, this test is
to deal with random test crashes on CI using SQLite. Usually, this test is
skipped to avoid causing additional crashes in real runs.
"""
if os.path.exists('crashed.tmp'):