From 2ec0d2bea4df4a6baaaf8fa41b2cbf361b75d03a Mon Sep 17 00:00:00 2001 From: Andreas Axelsson Date: Mon, 19 Aug 2024 15:26:58 +0800 Subject: [PATCH 1/3] [matches] Add index to start_date and use JSONB --- ...8757c_add_indices_for_fast_match_lookup.py | 30 +++++++++++++++++++ driftbase/models/db.py | 15 +++++----- scripts/local-alembic.sh | 2 +- scripts/local-provision.sh | 2 +- 4 files changed, 40 insertions(+), 9 deletions(-) create mode 100644 alembic/versions/a4b515b8757c_add_indices_for_fast_match_lookup.py diff --git a/alembic/versions/a4b515b8757c_add_indices_for_fast_match_lookup.py b/alembic/versions/a4b515b8757c_add_indices_for_fast_match_lookup.py new file mode 100644 index 00000000..a81c1097 --- /dev/null +++ b/alembic/versions/a4b515b8757c_add_indices_for_fast_match_lookup.py @@ -0,0 +1,30 @@ +"""Add indices for fast match lookup + +Revision ID: a4b515b8757c +Revises: 58c1b2a9640f +Create Date: 2024-08-19 14:10:04.813456 + +""" + +# revision identifiers, used by Alembic. +revision = 'a4b515b8757c' +down_revision = '58c1b2a9640f' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(engine_name): + print("Upgrading {}".format(engine_name)) + op.create_index('ix_gs_matches_start_date', 'gs_matches', ['start_date']) + op.alter_column('gs_matches', 'details', type_=postgresql.JSONB) + op.alter_column('gs_matches', 'match_statistics', type_=postgresql.JSONB) + + +def downgrade(engine_name): + print("Downgrading {}".format(engine_name)) + op.alter_column('gs_matches', 'match_statistics', type_=postgresql.JSON) + op.alter_column('gs_matches', 'details', type_=postgresql.JSON) + op.drop_index('ix_gs_matches_start_date') diff --git a/driftbase/models/db.py b/driftbase/models/db.py index bc4fb5b0..effc5cab 100644 --- a/driftbase/models/db.py +++ b/driftbase/models/db.py @@ -15,7 +15,7 @@ Boolean, ) from sqlalchemy import DDL, event -from sqlalchemy.dialects.postgresql import ENUM, INET, JSON, UUID +from sqlalchemy.dialects.postgresql import ENUM, INET, JSON, UUID, JSONB from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship, backref from sqlalchemy.schema import Sequence, Index @@ -178,9 +178,9 @@ class Client(ModelBase): def is_online(self): _, heartbeat_timeout = get_client_heartbeat_config() if ( - self.status == "active" - and self.heartbeat + datetime.timedelta(seconds=heartbeat_timeout) - >= utcnow() + self.status == "active" + and self.heartbeat + datetime.timedelta(seconds=heartbeat_timeout) + >= utcnow() ): return True return False @@ -344,15 +344,15 @@ class Match(ModelBase): match_id = Column(Integer, primary_key=True) server_id = Column(Integer, nullable=False) - start_date = Column(DateTime, nullable=True, server_default=utc_now) + start_date = Column(DateTime, nullable=True, server_default=utc_now, index=True) end_date = Column(DateTime, nullable=True) status = Column(String(50), nullable=True) num_players = Column(Integer, nullable=True) max_players = Column(Integer, nullable=True) game_mode = Column(String(50), nullable=True) map_name = Column(String(50), nullable=True) - match_statistics = Column(JSON, nullable=True) - details = Column(JSON, nullable=True) + match_statistics = Column(JSONB, nullable=True) + details = Column(JSONB, nullable=True) status_date = Column(DateTime, nullable=True) unique_key = Column(String(50), nullable=True) @@ -572,6 +572,7 @@ class FriendInvite(ModelBase): UniqueConstraint(token, name="uq_ck_friend_invites_token") + event.listen( CorePlayer.__table__, "after_create", diff --git a/scripts/local-alembic.sh b/scripts/local-alembic.sh index cbbc2231..aecc9457 100755 --- a/scripts/local-alembic.sh +++ b/scripts/local-alembic.sh @@ -9,5 +9,5 @@ set +a docker run --rm -ti --network backend --env-file $DIR/local.env \ --entrypoint /bin/bash \ - app_drift-base:latest \ + app-drift-base:latest \ -c "alembic $(printf "${1+ %q}" "$@")" diff --git a/scripts/local-provision.sh b/scripts/local-provision.sh index 49d746ac..1a90581c 100755 --- a/scripts/local-provision.sh +++ b/scripts/local-provision.sh @@ -16,7 +16,7 @@ docker run --rm -ti --network backend --env-file $DIR/local.env \ --mount "type=bind,source=$CONFIG_STORE,target=$CONFIG_STORE" \ --mount "type=bind,source=$CONFIG_ORIGIN,target=$CONFIG_ORIGIN" \ --entrypoint /bin/bash \ - app_drift-base:latest \ + app-drift-base:latest \ -c "driftconfig provision-tenant $TENANT $DEPLOYABLE" driftconfig cache $CONFIG From 78a2a86b40daa2446256958f2923a82ad4044723 Mon Sep 17 00:00:00 2001 From: "Einar Th. Einarsson" Date: Tue, 3 Sep 2024 10:34:41 +0000 Subject: [PATCH 2/3] Add index to 'status' as we usually include that as a filter, and try out a BRIN for 'start_date' --- .../a4b515b8757c_add_indices_for_fast_match_lookup.py | 4 +++- driftbase/models/db.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/alembic/versions/a4b515b8757c_add_indices_for_fast_match_lookup.py b/alembic/versions/a4b515b8757c_add_indices_for_fast_match_lookup.py index a81c1097..1b680db6 100644 --- a/alembic/versions/a4b515b8757c_add_indices_for_fast_match_lookup.py +++ b/alembic/versions/a4b515b8757c_add_indices_for_fast_match_lookup.py @@ -18,7 +18,8 @@ def upgrade(engine_name): print("Upgrading {}".format(engine_name)) - op.create_index('ix_gs_matches_start_date', 'gs_matches', ['start_date']) + op.create_index('ix_gs_matches_start_date', 'gs_matches', ['start_date'], postgresql_using='brin') + op.create_index('ix_gs_matches_status', 'gs_matches', ['status']) op.alter_column('gs_matches', 'details', type_=postgresql.JSONB) op.alter_column('gs_matches', 'match_statistics', type_=postgresql.JSONB) @@ -27,4 +28,5 @@ def downgrade(engine_name): print("Downgrading {}".format(engine_name)) op.alter_column('gs_matches', 'match_statistics', type_=postgresql.JSON) op.alter_column('gs_matches', 'details', type_=postgresql.JSON) + op.drop_index('ix_gs_matches_status') op.drop_index('ix_gs_matches_start_date') diff --git a/driftbase/models/db.py b/driftbase/models/db.py index effc5cab..d5d78560 100644 --- a/driftbase/models/db.py +++ b/driftbase/models/db.py @@ -344,9 +344,9 @@ class Match(ModelBase): match_id = Column(Integer, primary_key=True) server_id = Column(Integer, nullable=False) - start_date = Column(DateTime, nullable=True, server_default=utc_now, index=True) + start_date = Column(DateTime, nullable=True, server_default=utc_now) end_date = Column(DateTime, nullable=True) - status = Column(String(50), nullable=True) + status = Column(String(50), nullable=True, index=True) num_players = Column(Integer, nullable=True) max_players = Column(Integer, nullable=True) game_mode = Column(String(50), nullable=True) @@ -356,6 +356,10 @@ class Match(ModelBase): status_date = Column(DateTime, nullable=True) unique_key = Column(String(50), nullable=True) + __table_args__ = ( + Index('ix_brin_gs_matches_start_date', 'start_date', postgresql_using='brin'), + ) + class MatchPlayer(ModelBase): __tablename__ = "gs_matchplayers" From 7e01eddf4af6328092c3dd92f6c3b27abdfeab35 Mon Sep 17 00:00:00 2001 From: "Einar Th. Einarsson" Date: Tue, 3 Sep 2024 10:36:12 +0000 Subject: [PATCH 3/3] use same index name in model and in alembic --- driftbase/models/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driftbase/models/db.py b/driftbase/models/db.py index d5d78560..a3d14ea7 100644 --- a/driftbase/models/db.py +++ b/driftbase/models/db.py @@ -357,7 +357,7 @@ class Match(ModelBase): unique_key = Column(String(50), nullable=True) __table_args__ = ( - Index('ix_brin_gs_matches_start_date', 'start_date', postgresql_using='brin'), + Index('ix_gs_matches_start_date', 'start_date', postgresql_using='brin'), )