From 632c83eeae071f06c07971c5af162578b6004fbf Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Mar 2026 10:57:06 -0400 Subject: [PATCH 001/106] Add experimental feature flag `msc4429_enabled` --- synapse/config/experimental.py | 3 +++ synapse/rest/client/versions.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index a8c9305704e..416f6b240e4 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -563,6 +563,9 @@ def read_config( # MSC4133: Custom profile fields self.msc4133_enabled: bool = experimental.get("msc4133_enabled", False) + # MSC4429: Profile updates for legacy /sync + self.msc4429_enabled: bool = experimental.get("msc4429_enabled", False) + # MSC4143: Matrix RTC Transport using Livekit Backend self.msc4143_enabled: bool = experimental.get("msc4143_enabled", False) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index f8d7a1a4d9d..3aae506eb45 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -198,6 +198,8 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # Arbitrary key-value profile fields. "uk.tcpip.msc4133": self.config.experimental.msc4133_enabled, "uk.tcpip.msc4133.stable": True, + # MSC4429: Profile updates for legacy /sync. + "org.matrix.msc4429": self.config.experimental.msc4429_enabled, # MSC4155: Invite filtering "org.matrix.msc4155": self.config.experimental.msc4155_enabled, # MSC4306: Support for thread subscriptions From d0b616ddceb0639c06d7e0f50d77645b7dedee69 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Mar 2026 16:40:02 -0400 Subject: [PATCH 002/106] Schema for new profile updates stream table We create a stream table dedicated to tracking profile updates. The sync machinary can this stream to understand whether a profile update needs to be included in a client's sync response. --- synapse/storage/schema/__init__.py | 5 +++- .../main/delta/94/01_profile_updates.sql | 28 +++++++++++++++++++ .../94/01_profile_updates_seq.sql.postgres | 18 ++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/main/delta/94/01_profile_updates.sql create mode 100644 synapse/storage/schema/main/delta/94/01_profile_updates_seq.sql.postgres diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index c4c4d7bcc4a..8990bb0c177 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -19,7 +19,7 @@ # # -SCHEMA_VERSION = 93 # remember to update the list below when updating +SCHEMA_VERSION = 94 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -171,6 +171,9 @@ Changes in SCHEMA_VERSION = 93 - MSC4140: Set delayed events to be uniquely identifiable by their delay ID. + +Changes in SCHEMA_VERSION = 94 + - MSC4429: Track updates to user profile fields via a new stream. """ diff --git a/synapse/storage/schema/main/delta/94/01_profile_updates.sql b/synapse/storage/schema/main/delta/94/01_profile_updates.sql new file mode 100644 index 00000000000..18a3deef6d5 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/01_profile_updates.sql @@ -0,0 +1,28 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd. +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Track updates to profile fields for MSC4429 legacy /sync. +CREATE TABLE profile_updates ( + stream_id BIGINT NOT NULL PRIMARY KEY, + instance_name TEXT NOT NULL, + + user_id TEXT NOT NULL, + field_name TEXT NOT NULL, + + CONSTRAINT profile_updates_fk_users + FOREIGN KEY (user_id) + REFERENCES users(name) ON DELETE CASCADE +); + +CREATE INDEX profile_updates_by_user ON profile_updates (user_id, stream_id); +CREATE INDEX profile_updates_by_field ON profile_updates (field_name, stream_id); diff --git a/synapse/storage/schema/main/delta/94/01_profile_updates_seq.sql.postgres b/synapse/storage/schema/main/delta/94/01_profile_updates_seq.sql.postgres new file mode 100644 index 00000000000..9abf79b68de --- /dev/null +++ b/synapse/storage/schema/main/delta/94/01_profile_updates_seq.sql.postgres @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd. +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE SEQUENCE profile_updates_sequence; +-- Synapse streams start at 2, because the default position is 1 +-- so any item inserted at position 1 is ignored. +-- We have to use nextval not START WITH 2, see https://github.com/element-hq/synapse/issues/18712 +SELECT nextval('profile_updates_sequence'); From 63cfbe7e2dd6f3aa4bb9031a8eee1f394db71d63 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Mar 2026 16:51:21 -0400 Subject: [PATCH 003/106] Add profile updates stream Add a new stream type for profile updates. This allows sync processes to determine which profile updates a given user has or hasn't seen yet. --- synapse/config/workers.py | 25 +++- synapse/handlers/profile.py | 45 ++++++ synapse/notifier.py | 1 + synapse/storage/databases/main/profile.py | 174 +++++++++++++++++++++- synapse/streams/events.py | 3 + synapse/types/__init__.py | 13 +- 6 files changed, 254 insertions(+), 7 deletions(-) diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 996be88cb26..f4d5d687a0c 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -142,6 +142,9 @@ class WriterLocations: push_rules: The instances that write to the push stream. Currently can only be a single instance. device_lists: The instances that write to the device list stream. + thread_subscriptions: The instances that write to the thread subscriptions + stream. + profile_updates: The instances that write to the profile updates stream. """ events: list[str] = attr.ib( @@ -177,7 +180,11 @@ class WriterLocations: converter=_instance_to_list_converter, ) thread_subscriptions: list[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], + converter=_instance_to_list_converter, + ) + profile_updates: list[str] = attr.ib( + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) @@ -355,8 +362,7 @@ def read_config( writers = config.get("stream_writers") or {} self.writers = WriterLocations(**writers) - # Check that the configured writers for events and typing also appears in - # `instance_map`. + # Check that the configured writers also appear in `instance_map`. for stream in ( "events", "typing", @@ -365,6 +371,9 @@ def read_config( "receipts", "presence", "push_rules", + "device_lists", + "thread_subscriptions", + "profile_updates", ): instances = _instance_to_list_converter(getattr(self.writers, stream)) for instance in instances: @@ -415,6 +424,16 @@ def read_config( "Must specify at least one instance to handle `device_lists` messages." ) + if len(self.writers.thread_subscriptions) == 0: + raise ConfigError( + "Must specify at least one instance to handle `thread_subscriptions` messages." + ) + + if len(self.writers.profile_updates) == 0: + raise ConfigError( + "Must specify at least one instance to handle `profile_updates` messages." + ) + self.events_shard_config = RoutableShardedWorkerHandlingConfig( self.writers.events ) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index d123bcdd367..b3bffb0cc2e 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -42,6 +42,7 @@ JsonValue, Requester, ScheduledTask, + StreamKeyType, TaskStatus, UserID, create_requester, @@ -75,6 +76,8 @@ def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() # nb must be called this for @cached self.store = hs.get_datastores().main self.hs = hs + self._notifier = hs.get_notifier() + self._msc4429_enabled = hs.config.experimental.msc4429_enabled self.federation = hs.get_federation_client() hs.get_federation_registry().register_query_handler( @@ -99,6 +102,24 @@ def __init__(self, hs: "HomeServer"): ) self._worker_locks = hs.get_worker_locks_handler() + async def _notify_profile_update(self, user_id: UserID, stream_id: int) -> None: + room_ids = await self.store.get_rooms_for_user(user_id.to_string()) + if not room_ids: + return + + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, stream_id, rooms=room_ids + ) + + async def _record_profile_updates( + self, user_id: UserID, updates: list[tuple[str, JsonValue | None]] + ) -> None: + if not self._msc4429_enabled or not updates: + return + + stream_id = await self.store.add_profile_updates(user_id, updates) + await self._notify_profile_update(user_id, stream_id) + async def get_profile(self, user_id: str, ignore_backoff: bool = True) -> JsonDict: """ Get a user's profile as a JSON dictionary. @@ -253,6 +274,9 @@ async def set_displayname( ) await self.store.set_profile_displayname(target_user, displayname_to_set) + await self._record_profile_updates( + target_user, [(ProfileFields.DISPLAYNAME, displayname_to_set)] + ) profile = await self.store.get_profileinfo(target_user) @@ -362,6 +386,9 @@ async def set_avatar_url( ) await self.store.set_profile_avatar_url(target_user, avatar_url_to_set) + await self._record_profile_updates( + target_user, [(ProfileFields.AVATAR_URL, avatar_url_to_set)] + ) profile = await self.store.get_profileinfo(target_user) @@ -406,6 +433,8 @@ async def delete_profile_upon_deactivation( # have it. raise AuthError(400, "Cannot remove another user's profile") + profile_updates: list[tuple[str, JsonValue | None]] = [] + current_profile: ProfileInfo | None = None if not by_admin: current_profile = await self.store.get_profileinfo(target_user) if not self.hs.config.registration.enable_set_displayname: @@ -428,7 +457,21 @@ async def delete_profile_upon_deactivation( Codes.FORBIDDEN, ) + if self._msc4429_enabled: + if current_profile is None: + current_profile = await self.store.get_profileinfo(target_user) + + if current_profile.display_name is not None: + profile_updates.append((ProfileFields.DISPLAYNAME, None)) + if current_profile.avatar_url is not None: + profile_updates.append((ProfileFields.AVATAR_URL, None)) + + custom_fields = await self.store.get_profile_fields(target_user) + for field_name in custom_fields.keys(): + profile_updates.append((field_name, None)) + await self.store.delete_profile(target_user) + await self._record_profile_updates(target_user, profile_updates) await self._third_party_rules.on_profile_update( target_user.to_string(), @@ -582,6 +625,7 @@ async def set_profile_field( raise AuthError(403, "Cannot set another user's profile") await self.store.set_profile_field(target_user, field_name, new_value) + await self._record_profile_updates(target_user, [(field_name, new_value)]) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) @@ -617,6 +661,7 @@ async def delete_profile_field( raise AuthError(400, "Cannot set another user's profile") await self.store.delete_profile_field(target_user, field_name) + await self._record_profile_updates(target_user, [(field_name, None)]) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) diff --git a/synapse/notifier.py b/synapse/notifier.py index 93d438def71..a1fa432dfbe 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -527,6 +527,7 @@ def on_new_event( StreamKeyType.UN_PARTIAL_STATED_ROOMS, StreamKeyType.THREAD_SUBSCRIPTIONS, StreamKeyType.STICKY_EVENTS, + StreamKeyType.PROFILE_UPDATES, ], new_token: int, users: Collection[str | UserID] | None = None, diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 9b787e19a3d..a79d6eab635 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -19,13 +19,15 @@ # # import json -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Collection, Iterable, Sequence, cast +import attr from canonicaljson import encode_canonical_json from synapse.api.constants import ProfileFields from synapse.api.errors import Codes, StoreError -from synapse.storage._base import SQLBaseStore +from synapse.replication.tcp.streams._base import ProfileUpdatesStream +from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, @@ -33,6 +35,7 @@ ) from synapse.storage.databases.main.roommember import ProfileInfo from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import JsonDict, JsonValue, UserID if TYPE_CHECKING: @@ -43,6 +46,15 @@ MAX_PROFILE_SIZE = 65536 +@attr.s(slots=True, frozen=True, auto_attribs=True) +class ProfileUpdate: + """An update to a user's profile.""" + + stream_id: int + user_id: str + field_name: str + + class ProfileWorkerStore(SQLBaseStore): def __init__( self, @@ -52,6 +64,7 @@ def __init__( ): super().__init__(database, db_conn, hs) self.server_name: str = hs.hostname + self._instance_name: str = hs.get_instance_name() self.database_engine = database.engine self.db_pool.updates.register_background_index_update( "profiles_full_user_id_key_idx", @@ -65,6 +78,23 @@ def __init__( "populate_full_user_id_profiles", self.populate_full_user_id_profiles ) + self._can_write_to_profile_updates = ( + self._instance_name in hs.config.worker.writers.profile_updates + ) + self._profile_updates_id_gen: MultiWriterIdGenerator = MultiWriterIdGenerator( + db_conn=db_conn, + db=database, + notifier=hs.get_replication_notifier(), + stream_name="profile_updates", + server_name=self.server_name, + instance_name=self._instance_name, + tables=[ + ("profile_updates", "instance_name", "stream_id"), + ], + sequence_name="profile_updates_sequence", + writers=hs.config.worker.writers.profile_updates, + ) + async def populate_full_user_id_profiles( self, progress: JsonDict, batch_size: int ) -> int: @@ -291,6 +321,146 @@ async def get_profile_fields(self, user_id: UserID) -> dict[str, str]: result = json.loads(result) return result or {} + def get_max_profile_updates_stream_id(self) -> int: + """Get the current maximum stream_id for profile updates.""" + return self._profile_updates_id_gen.get_current_token() + + def get_profile_updates_stream_id_generator(self) -> MultiWriterIdGenerator: + return self._profile_updates_id_gen + + async def get_updated_profile_updates( + self, from_id: int, to_id: int, limit: int + ) -> list[tuple[int, str, str]]: + """Get profile updates that have changed, for the profile_updates stream.""" + if from_id == to_id: + return [] + + def _get_updated_profile_updates_txn( + txn: LoggingTransaction, + ) -> list[tuple[int, str, str]]: + sql = ( + "SELECT stream_id, user_id, field_name" + " FROM profile_updates" + " WHERE ? < stream_id AND stream_id <= ?" + " ORDER BY stream_id ASC LIMIT ?" + ) + txn.execute(sql, (from_id, to_id, limit)) + return cast(list[tuple[int, str, str]], txn.fetchall()) + + return await self.db_pool.runInteraction( + "get_updated_profile_updates", _get_updated_profile_updates_txn + ) + + async def get_profile_updates_for_fields( + self, + *, + from_id: int, + to_id: int, + field_names: Iterable[str], + ) -> list[ProfileUpdate]: + """Get profile update markers for the given fields in a stream range.""" + if from_id == to_id: + return [] + + field_names = list(field_names) + if not field_names: + return [] + + def _get_profile_updates_for_fields_txn( + txn: LoggingTransaction, + ) -> list[ProfileUpdate]: + clause, args = make_in_list_sql_clause( + txn.database_engine, "field_name", field_names + ) + sql = ( + "SELECT stream_id, user_id, field_name" + " FROM profile_updates" + f" WHERE ? < stream_id AND stream_id <= ? AND {clause}" + " ORDER BY stream_id ASC" + ) + txn.execute(sql, (from_id, to_id, *args)) + rows = cast(list[tuple[int, str, str]], txn.fetchall()) + + updates: list[ProfileUpdate] = [] + for stream_id, user_id, field_name in rows: + updates.append( + ProfileUpdate( + stream_id=stream_id, + user_id=user_id, + field_name=field_name, + ) + ) + + return updates + + return await self.db_pool.runInteraction( + "get_profile_updates_for_fields", _get_profile_updates_for_fields_txn + ) + + async def get_profile_data_for_users( + self, user_ids: Collection[str] + ) -> dict[str, tuple[str | None, str | None, JsonDict]]: + """Fetch displayname/avatar_url/custom fields for a list of users.""" + if not user_ids: + return {} + + rows = cast( + list[tuple[str, str | None, str | None, object | None]], + await self.db_pool.simple_select_many_batch( + table="profiles", + column="full_user_id", + iterable=user_ids, + retcols=("full_user_id", "displayname", "avatar_url", "fields"), + desc="get_profile_data_for_users", + ), + ) + + results: dict[str, tuple[str | None, str | None, JsonDict]] = {} + for full_user_id, displayname, avatar_url, fields in rows: + if fields is None: + fields_dict: JsonDict = {} + elif isinstance(fields, (str, bytes, bytearray, memoryview)): + fields_dict = cast(JsonDict, db_to_json(fields)) + else: + fields_dict = cast(JsonDict, fields) + + results[full_user_id] = (displayname, avatar_url, fields_dict) + + return results + + async def add_profile_updates( + self, user_id: UserID, updates: Sequence[tuple[str, JsonValue | None]] + ) -> int: + """Persist profile update markers and return the last stream ID.""" + assert self._can_write_to_profile_updates + + if not updates: + return self._profile_updates_id_gen.get_current_token() + + user_id_str = user_id.to_string() + + def _add_profile_updates_txn(txn: LoggingTransaction) -> int: + stream_ids = self._profile_updates_id_gen.get_next_mult_txn( + txn, len(updates) + ) + for stream_id, (field_name, _value) in zip(stream_ids, updates): + self.db_pool.simple_insert_txn( + txn, + table="profile_updates", + values={ + "stream_id": stream_id, + "instance_name": self._instance_name, + "user_id": user_id_str, + "field_name": field_name, + }, + ) + + return stream_ids[-1] + + return await self.db_pool.runInteraction( + "add_profile_updates", _add_profile_updates_txn + ) + async def create_profile(self, user_id: UserID) -> None: """ Create a blank profile for a user. diff --git a/synapse/streams/events.py b/synapse/streams/events.py index d2720fb9592..90b74c75c7b 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -85,6 +85,7 @@ def get_current_token(self) -> StreamToken: ) thread_subscriptions_key = self.store.get_max_thread_subscriptions_stream_id() sticky_events_key = self.store.get_max_sticky_events_stream_id() + profile_updates_key = self.store.get_max_profile_updates_stream_id() token = StreamToken( room_key=self.sources.room.get_current_key(), @@ -100,6 +101,7 @@ def get_current_token(self) -> StreamToken: un_partial_stated_rooms_key=un_partial_stated_rooms_key, thread_subscriptions_key=thread_subscriptions_key, sticky_events_key=sticky_events_key, + profile_updates_key=profile_updates_key, ) return token @@ -128,6 +130,7 @@ async def bound_future_token(self, token: StreamToken) -> StreamToken: StreamKeyType.UN_PARTIAL_STATED_ROOMS: self.store.get_un_partial_stated_rooms_id_generator(), StreamKeyType.THREAD_SUBSCRIPTIONS: self.store.get_thread_subscriptions_stream_id_generator(), StreamKeyType.STICKY_EVENTS: self.store.get_sticky_events_stream_id_generator(), + StreamKeyType.PROFILE_UPDATES: self.store.get_profile_updates_stream_id_generator(), } for _, key in StreamKeyType.__members__.items(): diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index fb1f1192b70..915188fe1ef 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -1007,6 +1007,7 @@ class StreamKeyType(Enum): UN_PARTIAL_STATED_ROOMS = "un_partial_stated_rooms_key" THREAD_SUBSCRIPTIONS = "thread_subscriptions_key" STICKY_EVENTS = "sticky_events_key" + PROFILE_UPDATES = "profile_updates_key" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -1014,7 +1015,7 @@ class StreamToken: """A collection of keys joined together by underscores in the following order and which represent the position in their respective streams. - ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242` + ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242_4141_101` 1. `room_key`: `s2633508` which is a `RoomStreamToken` - `RoomStreamToken`'s can also look like `t426-2633508` or `m56~2.58~3.59` - See the docstring for `RoomStreamToken` for more details. @@ -1029,6 +1030,7 @@ class StreamToken: 10. `un_partial_stated_rooms_key`: `379` 11. `thread_subscriptions_key`: 4242 12. `sticky_events_key`: 4141 + 13. `profile_updates_key`: 101 You can see how many of these keys correspond to the various fields in a "/sync" response: @@ -1089,6 +1091,7 @@ class StreamToken: un_partial_stated_rooms_key: int thread_subscriptions_key: int sticky_events_key: int + profile_updates_key: int _SEPARATOR = "_" START: ClassVar["StreamToken"] @@ -1118,6 +1121,7 @@ async def from_string(cls, store: "DataStore", string: str) -> "StreamToken": un_partial_stated_rooms_key, thread_subscriptions_key, sticky_events_key, + profile_updates_key, ) = keys return cls( @@ -1135,6 +1139,7 @@ async def from_string(cls, store: "DataStore", string: str) -> "StreamToken": un_partial_stated_rooms_key=int(un_partial_stated_rooms_key), thread_subscriptions_key=int(thread_subscriptions_key), sticky_events_key=int(sticky_events_key), + profile_updates_key=int(profile_updates_key), ) except CancelledError: raise @@ -1159,6 +1164,7 @@ async def to_string(self, store: "DataStore") -> str: str(self.un_partial_stated_rooms_key), str(self.thread_subscriptions_key), str(self.sticky_events_key), + str(self.profile_updates_key), ] ) @@ -1225,6 +1231,7 @@ def get_field( StreamKeyType.UN_PARTIAL_STATED_ROOMS, StreamKeyType.THREAD_SUBSCRIPTIONS, StreamKeyType.STICKY_EVENTS, + StreamKeyType.PROFILE_UPDATES, ], ) -> int: ... @@ -1281,7 +1288,8 @@ def __str__(self) -> str: f"account_data: {self.account_data_key}, push_rules: {self.push_rules_key}, " f"to_device: {self.to_device_key}, device_list: {self.device_list_key}, " f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key}," - f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key})" + f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key}, " + f"profile_updates: {self.profile_updates_key})" ) @@ -1298,6 +1306,7 @@ def __str__(self) -> str: un_partial_stated_rooms_key=0, thread_subscriptions_key=0, sticky_events_key=0, + profile_updates_key=0, ) From e45045d2a599186964d5969e6a087502427b2c50 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Mar 2026 16:51:48 -0400 Subject: [PATCH 004/106] Implement replication Replicate changes to the profile updates stream so that sync workers know there's a new profile update, and wake up sync waiters accordingly. Updates are keyed by `room_id` - sync waiters should only wake up if a user they share a room with updates their profile. --- synapse/replication/tcp/client.py | 19 +++++++++ synapse/replication/tcp/handler.py | 7 ++++ synapse/replication/tcp/streams/__init__.py | 3 ++ synapse/replication/tcp/streams/_base.py | 43 +++++++++++++++++++++ synapse/storage/databases/main/profile.py | 7 ++++ 5 files changed, 79 insertions(+) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index bc7e46d4c92..c2155f05174 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -44,6 +44,7 @@ UnPartialStatedRoomStream, ) from synapse.replication.tcp.streams._base import ( + ProfileUpdatesStream, StickyEventsStream, ThreadSubscriptionsStream, ) @@ -265,6 +266,24 @@ async def on_rdata( token, users=[row.user_id for row in rows], ) + elif stream_name == ProfileUpdatesStream.NAME: + updated_user_ids = {row.user_id for row in rows} + if updated_user_ids: + room_ids: set[str] = set() + user_ids_to_room_ids = await self.store.get_rooms_for_users( + updated_user_ids + ) + # Typing: `user_ids_to_room_ids.values` is a frozenset, but one + # can still iterate over it, hence the ignore. + for batched_room_ids in user_ids_to_room_ids.values(): # type: ignore[assignment] + room_ids.update(batched_room_ids) + + if room_ids: + self.notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + token, + rooms=room_ids, + ) elif stream_name == StickyEventsStream.NAME: self.notifier.on_new_event( StreamKeyType.STICKY_EVENTS, diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index ad9fed72dd8..65befb77064 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -67,6 +67,7 @@ ) from synapse.replication.tcp.streams._base import ( DeviceListsStream, + ProfileUpdatesStream, StickyEventsStream, ThreadSubscriptionsStream, ) @@ -218,6 +219,12 @@ def __init__(self, hs: "HomeServer"): continue + if isinstance(stream, ProfileUpdatesStream): + if hs.get_instance_name() in hs.config.worker.writers.profile_updates: + self._streams_to_replicate.append(stream) + + continue + if isinstance(stream, StickyEventsStream): if hs.get_instance_name() in hs.config.worker.writers.events: self._streams_to_replicate.append(stream) diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py index 067847617fa..98bfa04d4af 100644 --- a/synapse/replication/tcp/streams/__init__.py +++ b/synapse/replication/tcp/streams/__init__.py @@ -37,6 +37,7 @@ DeviceListsStream, PresenceFederationStream, PresenceStream, + ProfileUpdatesStream, PushersStream, PushRulesStream, ReceiptsStream, @@ -69,6 +70,7 @@ ToDeviceStream, FederationStream, AccountDataStream, + ProfileUpdatesStream, StickyEventsStream, ThreadSubscriptionsStream, UnPartialStatedRoomStream, @@ -92,6 +94,7 @@ "ToDeviceStream", "FederationStream", "AccountDataStream", + "ProfileUpdatesStream", "StickyEventsStream", "ThreadSubscriptionsStream", "UnPartialStatedRoomStream", diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 1ea6b4fa857..aa5bd6b22f9 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -765,6 +765,49 @@ async def _update_function( return rows, rows[-1][0], len(updates) == limit +@attr.s(slots=True, auto_attribs=True) +class ProfileUpdatesStreamRow: + """Stream to inform workers about profile updates.""" + + user_id: str + field_name: str + + +class ProfileUpdatesStream(_StreamFromIdGen): + """A user profile field was changed.""" + + NAME = "profile_updates" + ROW_TYPE = ProfileUpdatesStreamRow + + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + super().__init__( + hs.get_instance_name(), + self._update_function, + self.store._profile_updates_id_gen, + ) + + async def _update_function( + self, instance_name: str, from_token: int, to_token: int, limit: int + ) -> StreamUpdateResult: + updates = await self.store.get_updated_profile_updates( + from_id=from_token, to_id=to_token, limit=limit + ) + rows = [ + ( + stream_id, + # These are the args to `ProfileUpdatesStreamRow` + (user_id, field_name), + ) + for stream_id, user_id, field_name in updates + ] + + if not rows: + return [], to_token, False + + return rows, rows[-1][0], len(updates) == limit + + @attr.s(slots=True, auto_attribs=True) class StickyEventsStreamRow: """Stream to inform workers about changes to sticky events.""" diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index a79d6eab635..a328d0478bd 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -182,6 +182,13 @@ def _final_batch(txn: LoggingTransaction, lower_bound_id: str) -> None: return 50 + def process_replication_position( + self, stream_name: str, instance_name: str, token: int + ) -> None: + if stream_name == ProfileUpdatesStream.NAME: + self._profile_updates_id_gen.advance(instance_name, token) + super().process_replication_position(stream_name, instance_name, token) + async def get_profileinfo(self, user_id: UserID) -> ProfileInfo: """ Fetch the display name and avatar URL of a user. From d9adfd85d340b11e90b600bd3fb51f56447c0463 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Mar 2026 16:52:46 -0400 Subject: [PATCH 005/106] Allow fetching latest changes via /sync and filtering Based on the provided since token and filter. --- synapse/api/filtering.py | 25 +++++++ synapse/handlers/sync.py | 128 ++++++++++++++++++++++++++++++++++++ synapse/rest/client/sync.py | 7 ++ 3 files changed, 160 insertions(+) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 9b47c20437b..bb46ead53c9 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -123,6 +123,13 @@ "filter": FILTER_SCHEMA, "room_filter": ROOM_FILTER_SCHEMA, "room_event_filter": ROOM_EVENT_FILTER_SCHEMA, + "profile_fields_filter": { + "type": "object", + "properties": { + "ids": {"type": "array", "items": {"type": "string"}}, + }, + "additionalProperties": True, + }, }, "properties": { "presence": {"$ref": "#/definitions/filter"}, @@ -130,6 +137,10 @@ "room": {"$ref": "#/definitions/room_filter"}, "event_format": {"type": "string", "enum": ["client", "federation"]}, "event_fields": {"type": "array", "items": {"type": "string"}}, + "profile_fields": {"$ref": "#/definitions/profile_fields_filter"}, + "org.matrix.msc4429.profile_fields": { + "$ref": "#/definitions/profile_fields_filter" + }, }, "additionalProperties": True, # Allow new fields for forward compatibility } @@ -217,6 +228,20 @@ def __init__(self, hs: "HomeServer", filter_json: JsonMapping): self.event_fields = filter_json.get("event_fields", []) self.event_format = filter_json.get("event_format", "client") + self.profile_fields: list[str] = [] + if hs.config.experimental.msc4429_enabled: + profile_fields_filter = filter_json.get("profile_fields") + if profile_fields_filter is None: + profile_fields_filter = filter_json.get( + "org.matrix.msc4429.profile_fields" + ) + + if isinstance(profile_fields_filter, Mapping): + ids = profile_fields_filter.get("ids", []) + if ids is None: + ids = [] + self.profile_fields = list(ids) + def __repr__(self) -> str: return "" % (json.dumps(self._filter_json),) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index c8ef5e2aa6c..b2c24e26ca1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -37,6 +37,7 @@ EventContentFields, EventTypes, Membership, + ProfileFields, StickyEvent, ) from synapse.api.filtering import FilterCollection @@ -63,6 +64,7 @@ DeviceListUpdates, JsonDict, JsonMapping, + JsonValue, MultiWriterStreamToken, MutableStateMap, Requester, @@ -223,6 +225,7 @@ class SyncResult: next_batch: Token for the next sync presence: List of presence events for the user. account_data: List of account_data events for the user. + profile_updates: Map of user_id to profile field updates for that user. joined: JoinedSyncResult for each joined room. invited: InvitedSyncResult for each invited room. knocked: KnockedSyncResult for each knocked on room. @@ -238,6 +241,7 @@ class SyncResult: next_batch: StreamToken presence: list[UserPresenceState] account_data: list[JsonDict] + profile_updates: dict[str, dict[str, JsonValue | None]] joined: list[JoinedSyncResult] invited: list[InvitedSyncResult] knocked: list[KnockedSyncResult] @@ -259,6 +263,7 @@ def __bool__(self) -> bool: or self.knocked or self.archived or self.account_data + or self.profile_updates or self.to_device or self.device_lists ) @@ -274,6 +279,7 @@ def empty( next_batch=next_batch, presence=[], account_data=[], + profile_updates={}, joined=[], invited=[], knocked=[], @@ -290,6 +296,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.hs_config = hs.config self.store = hs.get_datastores().main + self._is_mine_id = hs.is_mine_id self.notifier = hs.get_notifier() self.presence_handler = hs.get_presence_handler() self._relations_handler = hs.get_relations_handler() @@ -1733,6 +1740,9 @@ async def generate_sync_result( if not sync_config.filter_collection.blocks_all_global_account_data(): await self._generate_sync_entry_for_account_data(sync_result_builder) + if self.hs_config.experimental.msc4429_enabled: + await self._generate_sync_entry_for_profile_updates(sync_result_builder) + # Presence data is included if the server has it enabled and not filtered out. include_presence_data = bool( self.hs_config.server.presence_enabled @@ -1832,6 +1842,7 @@ async def generate_sync_result( return SyncResult( presence=sync_result_builder.presence, account_data=sync_result_builder.account_data, + profile_updates=sync_result_builder.profile_updates, joined=sync_result_builder.joined, invited=sync_result_builder.invited, knocked=sync_result_builder.knocked, @@ -2096,6 +2107,121 @@ async def _generate_sync_entry_for_account_data( sync_result_builder.account_data = account_data_for_user + async def _generate_initial_sync_entry_for_profile_updates( + self, + user_id: str, + sync_result_builder: "SyncResultBuilder", + profile_fields: list[str], + ) -> None: + user_ids = await self.store.get_users_who_share_room_with_user(user_id) + user_ids = {u for u in user_ids if self._is_mine_id(u)} + if not user_ids: + return + + profile_data_by_user = await self.store.get_profile_data_for_users(user_ids) + + all_updates: dict[str, dict[str, JsonValue | None]] = {} + for other_user_id in user_ids: + displayname = None + avatar_url = None + custom_fields: JsonDict = {} + + profile_data = profile_data_by_user.get(other_user_id) + if profile_data is not None: + displayname, avatar_url, custom_fields = profile_data + + per_user_updates: dict[str, JsonValue | None] = {} + for field_name in profile_fields: + if field_name == ProfileFields.DISPLAYNAME: + per_user_updates[field_name] = displayname + elif field_name == ProfileFields.AVATAR_URL: + per_user_updates[field_name] = avatar_url + else: + per_user_updates[field_name] = custom_fields.get(field_name) + + all_updates[other_user_id] = per_user_updates + + sync_result_builder.profile_updates = all_updates + return + + async def _generate_sync_entry_for_profile_updates( + self, sync_result_builder: "SyncResultBuilder" + ) -> None: + """Generates the profile update portion of the sync response.""" + sync_config = sync_result_builder.sync_config + profile_fields = sync_config.filter_collection.profile_fields + if not profile_fields: + return + + user_id = sync_config.user.to_string() + since_token = sync_result_builder.since_token + now_token = sync_result_builder.now_token + + if since_token is None: + # TODO: Refactor this into a separate function. + await self._generate_initial_sync_entry_for_profile_updates( + user_id, sync_result_builder, profile_fields + ) + return + + if since_token.profile_updates_key == now_token.profile_updates_key: + return + + updates = await self.store.get_profile_updates_for_fields( + from_id=since_token.profile_updates_key, + to_id=now_token.profile_updates_key, + field_names=profile_fields, + ) + if not updates: + return + + updated_user_ids = {update.user_id for update in updates} + if not updated_user_ids: + return + + shared_user_ids = await self.store.do_users_share_a_room( + user_id, updated_user_ids + ) + shared_user_ids.add(user_id) + + user_fields: dict[str, set[str]] = {} + for update in updates: + if update.user_id not in shared_user_ids: + continue + user_fields.setdefault(update.user_id, set()).add(update.field_name) + + if not user_fields: + return + + profile_data_by_user = await self.store.get_profile_data_for_users( + user_fields.keys() + ) + + profile_updates: dict[str, dict[str, JsonValue | None]] = {} + for other_user_id, fields in user_fields.items(): + displayname = None + avatar_url = None + custom_fields: JsonDict = {} + + profile_data = profile_data_by_user.get(other_user_id) + if profile_data is not None: + displayname, avatar_url, custom_fields = profile_data + + per_user_updates: dict[str, JsonValue | None] = {} + for field_name in fields: + if field_name == ProfileFields.DISPLAYNAME: + per_user_updates[field_name] = displayname + elif field_name == ProfileFields.AVATAR_URL: + per_user_updates[field_name] = avatar_url + else: + per_user_updates[field_name] = custom_fields.get(field_name) + + if per_user_updates: + profile_updates[other_user_id] = per_user_updates + + if profile_updates: + sync_result_builder.profile_updates = profile_updates + async def _generate_sync_entry_for_presence( self, sync_result_builder: "SyncResultBuilder", @@ -3108,6 +3234,7 @@ class SyncResultBuilder: # The following mirror the fields in a sync response presence account_data + profile_updates joined invited knocked @@ -3126,6 +3253,7 @@ class SyncResultBuilder: presence: list[UserPresenceState] = attr.Factory(list) account_data: list[JsonDict] = attr.Factory(list) + profile_updates: dict[str, dict[str, JsonValue | None]] = attr.Factory(dict) joined: list[JoinedSyncResult] = attr.Factory(list) invited: list[InvitedSyncResult] = attr.Factory(list) knocked: list[KnockedSyncResult] = attr.Factory(list) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 710d097eab0..c696cbb1437 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -124,6 +124,7 @@ def __init__(self, hs: "HomeServer"): self._event_serializer = hs.get_event_client_serializer() self._msc2654_enabled = hs.config.experimental.msc2654_enabled self._msc3773_enabled = hs.config.experimental.msc3773_enabled + self._msc4429_enabled = hs.config.experimental.msc4429_enabled self._json_filter_cache: LruCache[str, bool] = LruCache( max_size=1000, @@ -352,6 +353,12 @@ async def encode_response( if sync_result.to_device: response["to_device"] = {"events": sync_result.to_device} + if self._msc4429_enabled and sync_result.profile_updates: + response["org.matrix.msc4429.users"] = { + user_id: {"profile_updates": updates} + for user_id, updates in sync_result.profile_updates.items() + } + if sync_result.device_lists.changed: response["device_lists"]["changed"] = list(sync_result.device_lists.changed) if sync_result.device_lists.left: From 30e8737c96e8f94c8e63e80ba583ef2d028553e7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Mar 2026 16:54:12 -0400 Subject: [PATCH 006/106] Add `msc4429_enabled` to Complement tests --- .../conf/workers-shared-extra.yaml.j2 | 2 + synapse/handlers/sync.py | 38 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 9fd7fc954a3..c649bb4daa7 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -143,6 +143,8 @@ experimental_features: msc4354_enabled: true # `/sync` `state_after` msc4222_enabled: true + # Profile updates down legacy /sync + msc4429_enabled: true server_notices: system_mxid_localpart: _server diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b2c24e26ca1..f4e6211f1bb 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2113,6 +2113,20 @@ async def _generate_initial_sync_entry_for_profile_updates( sync_result_builder: "SyncResultBuilder", profile_fields: list[str], ) -> None: + """ + Build an initial sync entry for profile updates and attach it to the + given `sync_result_builder`. + + Note: Only the profile information for local users is returned. This is + to prevent fetching *too* many profiles in one request. Clients should + ideally instead first fetch profiles on-demand, then track updates for + all users via incremental `/sync`. + + Args: + user_id: The Matrix ID of the user to generate the sync entry for. + sync_result_builder: + profile_fields: The list of field IDs to filter for. + """ user_ids = await self.store.get_users_who_share_room_with_user(user_id) user_ids = {u for u in user_ids if self._is_mine_id(u)} if not user_ids: @@ -2147,7 +2161,13 @@ async def _generate_initial_sync_entry_for_profile_updates( async def _generate_sync_entry_for_profile_updates( self, sync_result_builder: "SyncResultBuilder" ) -> None: - """Generates the profile update portion of the sync response.""" + """ + Build a sync entry for profile updates and attach it to the given + `sync_result_builder`. + + Args: + sync_result_builder: + """ sync_config = sync_result_builder.sync_config profile_fields = sync_config.filter_collection.profile_fields if not profile_fields: @@ -2158,7 +2178,6 @@ async def _generate_sync_entry_for_profile_updates( now_token = sync_result_builder.now_token if since_token is None: - # TODO: Refactor this into a separate function. await self._generate_initial_sync_entry_for_profile_updates( user_id, sync_result_builder, profile_fields ) @@ -2176,9 +2195,6 @@ async def _generate_sync_entry_for_profile_updates( return updated_user_ids = {update.user_id for update in updates} - if not updated_user_ids: - return - shared_user_ids = await self.store.do_users_share_a_room( user_id, updated_user_ids ) @@ -2190,13 +2206,19 @@ async def _generate_sync_entry_for_profile_updates( continue user_fields.setdefault(update.user_id, set()).add(update.field_name) - if not user_fields: - return - + # Note: there's a small race condition here where a profile update may + # occur between fetching `now_token` above and reaching this step. In + # that case, the profile information will be newer than `now_token`. + # This is fine, as users will generally always want the latest profile + # information. However, it does mean that on the next sync, the same + # profile update will come down a second time. + # + # Hopefully clients can just filter these out. profile_data_by_user = await self.store.get_profile_data_for_users( user_fields.keys() ) + # Serialise the profile updates into the sync response format. profile_updates: dict[str, dict[str, JsonValue | None]] = {} for other_user_id, fields in user_fields.items(): displayname = None From cfb703418f2aa43a483a2b2a2a0f8551eed34559 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Mar 2026 16:57:42 -0400 Subject: [PATCH 007/106] newsfile --- changelog.d/19556.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/19556.feature diff --git a/changelog.d/19556.feature b/changelog.d/19556.feature new file mode 100644 index 00000000000..28ad5b06573 --- /dev/null +++ b/changelog.d/19556.feature @@ -0,0 +1 @@ +Implement experimental support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). \ No newline at end of file From 5b4d5054884c600bb8aac8cf6951d8e39e1a0115 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 17 Mar 2026 11:54:34 +0000 Subject: [PATCH 008/106] Update unit tests with new token format --- tests/rest/admin/test_room.py | 4 ++-- tests/rest/client/test_rooms.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index b32665eb73b..2130674cee2 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2545,7 +2545,7 @@ def test_timestamp_to_event(self) -> None: def test_topo_token_is_accepted(self) -> None: """Test Topo Token is accepted.""" - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), @@ -2559,7 +2559,7 @@ def test_topo_token_is_accepted(self) -> None: def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: """Test that stream token is accepted for forward pagination.""" - token = "s0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index f85c9939ce4..828514e85d1 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -2245,7 +2245,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.room_id = self.helper.create_room_as(self.user_id) def test_topo_token_is_accepted(self) -> None: - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) @@ -2256,7 +2256,7 @@ def test_topo_token_is_accepted(self) -> None: self.assertTrue("end" in channel.json_body) def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: - token = "s0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) From 86572c63ed57679d49c2fcb4bb2238f3724645cd Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 17 Mar 2026 16:19:49 +0000 Subject: [PATCH 009/106] Constrain profile endpoint handling to `profile_updates` writer workers And update relevant documentation/integration test config. --- .../complement/conf/start_for_complement.sh | 3 +- docker/configure_workers_and_start.py | 10 +++++++ docs/upgrade.md | 28 +++++++++++++++++++ .../configuration/config_documentation.md | 2 ++ schema/synapse-config.schema.yaml | 3 ++ synapse/rest/client/profile.py | 17 ++++++++--- 6 files changed, 58 insertions(+), 5 deletions(-) diff --git a/docker/complement/conf/start_for_complement.sh b/docker/complement/conf/start_for_complement.sh index da1b26a2836..20c57d5c78f 100755 --- a/docker/complement/conf/start_for_complement.sh +++ b/docker/complement/conf/start_for_complement.sh @@ -60,12 +60,13 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then federation_inbound, \ federation_reader, \ federation_sender, \ + profile_updates, \ synchrotron, \ client_reader, \ appservice, \ pusher, \ device_lists:2, \ - stream_writers=account_data+presence+receipts+to_device+typing" + stream_writers=account_data+presence+profile_updates+receipts+to_device+typing" fi log "Workers requested: $SYNAPSE_WORKER_TYPES" diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index ed15e8efef5..1db2a8a779e 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -308,6 +308,15 @@ "shared_extra_conf": {}, "worker_extra_conf": "", }, + "profile_updates": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/(unstable/uk.tcpip.msc4133|api/v1|r0|v3|unstable)/profile/.*/" + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, "device_lists": { "app": "synapse.app.generic_worker", "listener_resources": ["client", "replication"], @@ -517,6 +526,7 @@ def add_worker_roles_to_shared_config( "typing", "push_rules", "thread_subscriptions", + "profile_updates", } # Worker-type specific sharding config. Now a single worker can fulfill multiple diff --git a/docs/upgrade.md b/docs/upgrade.md index 777e57c4926..3c2fb7cb67e 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,34 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.151.0 + +## Profile Updates Stream Writer Workers + +This version of Synapse adds a new `profile_updates` writer stream. The +following endpoints may now only be handled by either the main process, or a +worker that is designed a "profile_updates" writer. If you are already routing +the following endpoints to a worker: + +``` +/_matrix/client/(api/v1|r0|v3)/profile//(?) +/_matrix/client/unstable/uk.tcpip.msc4133/profile//(?) +``` + +those worker(s) need to be marked as a stream writer for the `profile_updates` +stream in the shared config, using the +[`stream_writers`](https://element-hq.github.io/synapse/v1.151/usage/configuration/config_documentation.html#stream_writers) +config option: + +```yaml +stream_writers: + profile_updates: worker1 +``` + +as well as included in the +[`instance_map`](https://element-hq.github.io/synapse/v1.151/usage/configuration/config_documentation.html#instance_map) +config option. + # Upgrading to v1.150.0 ## Removal of the `systemd` pip extra diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 1335def5857..f80c007493b 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4475,6 +4475,8 @@ This setting has the following sub-options: * `device_lists` (string): Name of a worker assigned to the `device_lists` stream. +* `profile_updates` (string): Name of a worker assigned to the `profile_updates` stream. + Example configuration: ```yaml stream_writers: diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 3a61a4c6fc9..eea102d73e0 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -5522,6 +5522,9 @@ properties: device_lists: type: string description: Name of a worker assigned to the `device_lists` stream. + profile_updates: + type: string + description: Name of a worker assigned to the `profile_updates` stream. default: {} examples: - events: worker1 diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index c2ec5b36114..d3655440e98 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -284,8 +284,17 @@ class UnstableProfileFieldRestServlet(ProfileFieldRestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - # The specific field endpoint *must* appear before the generic profile endpoint. - ProfileFieldRestServlet(hs).register(http_server) + # Updating user profiles requires the ability to write to the + # `profile_updates` stream. + if hs.get_instance_name() in hs.config.worker.writers.profile_updates: + # The specific field endpoint *must* appear before the generic profile + # endpoint (below). + + # TODO: Is it possible to still allow any generic_worker to handle the + # `GET` endpoint? + ProfileFieldRestServlet(hs).register(http_server) + + if hs.config.experimental.msc4133_enabled: + UnstableProfileFieldRestServlet(hs).register(http_server) + ProfileRestServlet(hs).register(http_server) - if hs.config.experimental.msc4133_enabled: - UnstableProfileFieldRestServlet(hs).register(http_server) From 322127102c5d3e089fc7447dba90acb2bc3678ce Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 17 Mar 2026 18:07:37 +0000 Subject: [PATCH 010/106] Prevent 405 Method Not Allowed Prevent the `ProfileRestServlet` from handling requests intended for the `ProfileFieldRestServlet`. If a requester tried to call PUT `.../profile/@user:domain/(key_id?)` a worker that did not mount `ProfileFieldRestServlet`, they would then receive a `405 Method Not Allowed` instead of the expected `404 Not Found`. --- synapse/rest/client/profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index d3655440e98..7b8bb151f96 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -58,7 +58,7 @@ def _read_propagate(hs: "HomeServer", request: SynapseRequest) -> bool: class ProfileRestServlet(RestServlet): - PATTERNS = client_patterns("/profile/(?P[^/]*)", v1=True) + PATTERNS = client_patterns("/profile/(?P[^/]*)$", v1=True) CATEGORY = "Event sending requests" def __init__(self, hs: "HomeServer"): From f27ca70b9c0a17f53c54503d53d4a88a30e9d4a8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 17 Mar 2026 18:13:10 +0000 Subject: [PATCH 011/106] Correct the Complement worker endpoint configuration --- docker/configure_workers_and_start.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 1db2a8a779e..ee42eec1cb4 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -269,7 +269,10 @@ "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$", "^/_matrix/client/(api/v1|r0|v3|unstable)/join/", "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/", - "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/", + # The [^/] differentiates this endpoint from + # `ProfileRestFieldsServlet`, which we want to instead go to the + # `profile_updates` worker below. + "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/[^/]+", ], "shared_extra_conf": {}, "worker_extra_conf": "", @@ -312,7 +315,7 @@ "app": "synapse.app.generic_worker", "listener_resources": ["client", "replication"], "endpoint_patterns": [ - "^/_matrix/client/(unstable/uk.tcpip.msc4133|api/v1|r0|v3|unstable)/profile/.*/" + "^/_matrix/client/(unstable/uk.tcpip.msc4133|api/v1|r0|v3|unstable)/profile/.+/" ], "shared_extra_conf": {}, "worker_extra_conf": "", From f95d28ac55e1819dc1b1f274f11a434ba502a03b Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Fri, 1 May 2026 16:25:32 +0100 Subject: [PATCH 012/106] Bump schema for profile updates --- .../main/delta/95/01_profile_updates.sql | 28 +++++++++++++++++++ .../95/01_profile_updates_seq.sql.postgres | 18 ++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 synapse/storage/schema/main/delta/95/01_profile_updates.sql create mode 100644 synapse/storage/schema/main/delta/95/01_profile_updates_seq.sql.postgres diff --git a/synapse/storage/schema/main/delta/95/01_profile_updates.sql b/synapse/storage/schema/main/delta/95/01_profile_updates.sql new file mode 100644 index 00000000000..18a3deef6d5 --- /dev/null +++ b/synapse/storage/schema/main/delta/95/01_profile_updates.sql @@ -0,0 +1,28 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd. +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Track updates to profile fields for MSC4429 legacy /sync. +CREATE TABLE profile_updates ( + stream_id BIGINT NOT NULL PRIMARY KEY, + instance_name TEXT NOT NULL, + + user_id TEXT NOT NULL, + field_name TEXT NOT NULL, + + CONSTRAINT profile_updates_fk_users + FOREIGN KEY (user_id) + REFERENCES users(name) ON DELETE CASCADE +); + +CREATE INDEX profile_updates_by_user ON profile_updates (user_id, stream_id); +CREATE INDEX profile_updates_by_field ON profile_updates (field_name, stream_id); diff --git a/synapse/storage/schema/main/delta/95/01_profile_updates_seq.sql.postgres b/synapse/storage/schema/main/delta/95/01_profile_updates_seq.sql.postgres new file mode 100644 index 00000000000..9abf79b68de --- /dev/null +++ b/synapse/storage/schema/main/delta/95/01_profile_updates_seq.sql.postgres @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd. +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE SEQUENCE profile_updates_sequence; +-- Synapse streams start at 2, because the default position is 1 +-- so any item inserted at position 1 is ignored. +-- We have to use nextval not START WITH 2, see https://github.com/element-hq/synapse/issues/18712 +SELECT nextval('profile_updates_sequence'); From 343d4dffe0ba156f16dc88c1eeb1011e126d86f2 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 27 May 2026 15:39:58 +0300 Subject: [PATCH 013/106] Drop `profile_updates_fk_users` foreign key The profile updates table should also allow supporting federated user profile updates. --- synapse/storage/schema/main/delta/95/01_profile_updates.sql | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/synapse/storage/schema/main/delta/95/01_profile_updates.sql b/synapse/storage/schema/main/delta/95/01_profile_updates.sql index 18a3deef6d5..8f988cecdf8 100644 --- a/synapse/storage/schema/main/delta/95/01_profile_updates.sql +++ b/synapse/storage/schema/main/delta/95/01_profile_updates.sql @@ -17,11 +17,7 @@ CREATE TABLE profile_updates ( instance_name TEXT NOT NULL, user_id TEXT NOT NULL, - field_name TEXT NOT NULL, - - CONSTRAINT profile_updates_fk_users - FOREIGN KEY (user_id) - REFERENCES users(name) ON DELETE CASCADE + field_name TEXT NOT NULL ); CREATE INDEX profile_updates_by_user ON profile_updates (user_id, stream_id); From 386958cbda235b19f780682abdf0b84093d023fc Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 27 May 2026 15:57:48 +0300 Subject: [PATCH 014/106] Add a timestamp column to `profile_updates` Also add some comments to some fields. --- synapse/storage/databases/main/profile.py | 1 + .../storage/schema/main/delta/95/01_profile_updates.sql | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index a328d0478bd..ecd9c53c934 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -459,6 +459,7 @@ def _add_profile_updates_txn(txn: LoggingTransaction) -> int: "instance_name": self._instance_name, "user_id": user_id_str, "field_name": field_name, + "inserted_ts": self.clock.time_msec(), }, ) diff --git a/synapse/storage/schema/main/delta/95/01_profile_updates.sql b/synapse/storage/schema/main/delta/95/01_profile_updates.sql index 8f988cecdf8..557d01f9ba3 100644 --- a/synapse/storage/schema/main/delta/95/01_profile_updates.sql +++ b/synapse/storage/schema/main/delta/95/01_profile_updates.sql @@ -16,8 +16,14 @@ CREATE TABLE profile_updates ( stream_id BIGINT NOT NULL PRIMARY KEY, instance_name TEXT NOT NULL, + -- The full user ID user_id TEXT NOT NULL, - field_name TEXT NOT NULL + -- Profile field name that has been updated, + -- see https://spec.matrix.org/unstable/client-server-api/#profiles + field_name TEXT NOT NULL, + + -- Unix timestamp for debugging purposes + inserted_ts BIGINT NOT NULL ); CREATE INDEX profile_updates_by_user ON profile_updates (user_id, stream_id); From c02e6a6181a3304a496822495776e255dcdf4ef1 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 27 May 2026 16:14:47 +0300 Subject: [PATCH 015/106] Only pass field names to `add_profile_updates` --- synapse/handlers/profile.py | 20 ++++++++++++-------- synapse/storage/databases/main/profile.py | 12 +++++++----- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index b3bffb0cc2e..5a052baec20 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -112,12 +112,12 @@ async def _notify_profile_update(self, user_id: UserID, stream_id: int) -> None: ) async def _record_profile_updates( - self, user_id: UserID, updates: list[tuple[str, JsonValue | None]] + self, user_id: UserID, updated_fields: list[str] ) -> None: - if not self._msc4429_enabled or not updates: + if not self._msc4429_enabled or not updated_fields: return - stream_id = await self.store.add_profile_updates(user_id, updates) + stream_id = await self.store.add_profile_updates(user_id, updated_fields) await self._notify_profile_update(user_id, stream_id) async def get_profile(self, user_id: str, ignore_backoff: bool = True) -> JsonDict: @@ -275,7 +275,8 @@ async def set_displayname( await self.store.set_profile_displayname(target_user, displayname_to_set) await self._record_profile_updates( - target_user, [(ProfileFields.DISPLAYNAME, displayname_to_set)] + target_user, + [ProfileFields.DISPLAYNAME], ) profile = await self.store.get_profileinfo(target_user) @@ -387,7 +388,8 @@ async def set_avatar_url( await self.store.set_profile_avatar_url(target_user, avatar_url_to_set) await self._record_profile_updates( - target_user, [(ProfileFields.AVATAR_URL, avatar_url_to_set)] + target_user, + [ProfileFields.AVATAR_URL], ) profile = await self.store.get_profileinfo(target_user) @@ -471,7 +473,9 @@ async def delete_profile_upon_deactivation( profile_updates.append((field_name, None)) await self.store.delete_profile(target_user) - await self._record_profile_updates(target_user, profile_updates) + await self._record_profile_updates( + target_user, [field_name for field_name, _value in profile_updates] + ) await self._third_party_rules.on_profile_update( target_user.to_string(), @@ -625,7 +629,7 @@ async def set_profile_field( raise AuthError(403, "Cannot set another user's profile") await self.store.set_profile_field(target_user, field_name, new_value) - await self._record_profile_updates(target_user, [(field_name, new_value)]) + await self._record_profile_updates(target_user, [field_name]) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) @@ -661,7 +665,7 @@ async def delete_profile_field( raise AuthError(400, "Cannot set another user's profile") await self.store.delete_profile_field(target_user, field_name) - await self._record_profile_updates(target_user, [(field_name, None)]) + await self._record_profile_updates(target_user, [field_name]) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index ecd9c53c934..e6fcfc711df 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -19,7 +19,7 @@ # # import json -from typing import TYPE_CHECKING, Collection, Iterable, Sequence, cast +from typing import TYPE_CHECKING, Collection, Iterable, cast import attr from canonicaljson import encode_canonical_json @@ -436,21 +436,23 @@ async def get_profile_data_for_users( return results async def add_profile_updates( - self, user_id: UserID, updates: Sequence[tuple[str, JsonValue | None]] + self, + user_id: UserID, + updated_fields: list[str], ) -> int: """Persist profile update markers and return the last stream ID.""" assert self._can_write_to_profile_updates - if not updates: + if not updated_fields: return self._profile_updates_id_gen.get_current_token() user_id_str = user_id.to_string() def _add_profile_updates_txn(txn: LoggingTransaction) -> int: stream_ids = self._profile_updates_id_gen.get_next_mult_txn( - txn, len(updates) + txn, len(updated_fields) ) - for stream_id, (field_name, _value) in zip(stream_ids, updates): + for stream_id, field_name in zip(stream_ids, updated_fields): self.db_pool.simple_insert_txn( txn, table="profile_updates", From 4e820ca96915c5c00576ccf73aad5d3e150c3d79 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 27 May 2026 16:18:41 +0300 Subject: [PATCH 016/106] Multiline string for `_get_updated_profile_updates_txn` --- synapse/storage/databases/main/profile.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index e6fcfc711df..ae1279fbf30 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -345,12 +345,14 @@ async def get_updated_profile_updates( def _get_updated_profile_updates_txn( txn: LoggingTransaction, ) -> list[tuple[int, str, str]]: - sql = ( - "SELECT stream_id, user_id, field_name" - " FROM profile_updates" - " WHERE ? < stream_id AND stream_id <= ?" - " ORDER BY stream_id ASC LIMIT ?" - ) + sql = """ + SELECT + stream_id, user_id, field_name + FROM profile_updates + WHERE + ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC LIMIT ? + """ txn.execute(sql, (from_id, to_id, limit)) return cast(list[tuple[int, str, str]], txn.fetchall()) From 7e964a9206dc8329ea90bf0571a8cd216ec7457e Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 27 May 2026 16:21:48 +0300 Subject: [PATCH 017/106] `user_id: UserID` in `ProfileUpdatesStreamRow` Also add docstrings for the fields. --- synapse/replication/tcp/streams/_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 0836e32b373..fab534a211d 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -33,6 +33,7 @@ from synapse.api.constants import AccountDataTypes from synapse.replication.http.streams import ReplicationGetStreamUpdates +from synapse.types import UserID if TYPE_CHECKING: from synapse.server import HomeServer @@ -769,8 +770,10 @@ async def _update_function( class ProfileUpdatesStreamRow: """Stream to inform workers about profile updates.""" - user_id: str + user_id: UserID + """The full user ID with the profile update.""" field_name: str + """The profile field that was updated, see https://spec.matrix.org/unstable/client-server-api/#profiles """ class ProfileUpdatesStream(_StreamFromIdGen): From fcbceee2fc8548751cf1c44999a5365849eaa17c Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 27 May 2026 16:25:51 +0300 Subject: [PATCH 018/106] Don't return anything for users without profiles in `_generate_initial_sync_entry_for_profile_updates` --- synapse/handlers/sync.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 32d8b569def..35dac31cce0 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2161,13 +2161,12 @@ async def _generate_initial_sync_entry_for_profile_updates( all_updates: dict[str, dict[str, JsonValue | None]] = {} for other_user_id in user_ids: - displayname = None - avatar_url = None - custom_fields: JsonDict = {} - profile_data = profile_data_by_user.get(other_user_id) - if profile_data is not None: - displayname, avatar_url, custom_fields = profile_data + if profile_data is None: + # Don't generate anything for users with no profile data + continue + + displayname, avatar_url, custom_fields = profile_data per_user_updates: dict[str, JsonValue | None] = {} for field_name in profile_fields: From 2fa20e0fee548a38cbf5ead123e358fb77dc4912 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 27 May 2026 16:36:47 +0300 Subject: [PATCH 019/106] Only support unstable prefix for `profile_fields` in API filters --- synapse/api/filtering.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index bb46ead53c9..62f55416b5c 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -137,7 +137,6 @@ "room": {"$ref": "#/definitions/room_filter"}, "event_format": {"type": "string", "enum": ["client", "federation"]}, "event_fields": {"type": "array", "items": {"type": "string"}}, - "profile_fields": {"$ref": "#/definitions/profile_fields_filter"}, "org.matrix.msc4429.profile_fields": { "$ref": "#/definitions/profile_fields_filter" }, @@ -230,11 +229,7 @@ def __init__(self, hs: "HomeServer", filter_json: JsonMapping): self.profile_fields: list[str] = [] if hs.config.experimental.msc4429_enabled: - profile_fields_filter = filter_json.get("profile_fields") - if profile_fields_filter is None: - profile_fields_filter = filter_json.get( - "org.matrix.msc4429.profile_fields" - ) + profile_fields_filter = filter_json.get("org.matrix.msc4429.profile_fields") if isinstance(profile_fields_filter, Mapping): ids = profile_fields_filter.get("ids", []) From 6ec8ee2a5e6da184b88eb482b6f109322e8f4bc0 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 28 May 2026 12:52:25 +0300 Subject: [PATCH 020/106] Mount ProfileFieldRestServlet on all Synapse instances But only allow PUT/DELETE from the `profile_updates` worker (which defaults to being `main`). --- synapse/rest/client/profile.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 7b8bb151f96..3ecba9659f7 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -109,6 +109,9 @@ def __init__(self, hs: "HomeServer"): self.hs = hs self.profile_handler = hs.get_profile_handler() self.auth = hs.get_auth() + self._is_profile_worker = ( + hs.get_instance_name() in hs.config.worker.writers.profile_updates + ) if hs.config.experimental.msc4133_enabled: self.PATTERNS.append( re.compile( @@ -157,6 +160,13 @@ async def on_GET( async def on_PUT( self, request: SynapseRequest, user_id: str, field_name: str ) -> tuple[int, JsonDict]: + if not self._is_profile_worker: + raise SynapseError( + HTTPStatus.METHOD_NOT_ALLOWED, + "Can only handle PUT /profile on instances configured to handle the profile_updates stream writer", + Codes.UNRECOGNIZED, + ) + if not UserID.is_valid(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM @@ -222,6 +232,12 @@ async def on_PUT( async def on_DELETE( self, request: SynapseRequest, user_id: str, field_name: str ) -> tuple[int, JsonDict]: + if not self._is_profile_worker: + raise SynapseError( + HTTPStatus.METHOD_NOT_ALLOWED, + "Can only handle DELETE /profile on instances configured to handle the profile_updates stream writer", + Codes.UNRECOGNIZED, + ) if not UserID.is_valid(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM @@ -284,17 +300,9 @@ class UnstableProfileFieldRestServlet(ProfileFieldRestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - # Updating user profiles requires the ability to write to the - # `profile_updates` stream. - if hs.get_instance_name() in hs.config.worker.writers.profile_updates: - # The specific field endpoint *must* appear before the generic profile - # endpoint (below). + ProfileFieldRestServlet(hs).register(http_server) - # TODO: Is it possible to still allow any generic_worker to handle the - # `GET` endpoint? - ProfileFieldRestServlet(hs).register(http_server) - - if hs.config.experimental.msc4133_enabled: - UnstableProfileFieldRestServlet(hs).register(http_server) + if hs.config.experimental.msc4133_enabled: + UnstableProfileFieldRestServlet(hs).register(http_server) ProfileRestServlet(hs).register(http_server) From df1b58733241780c0ef3c5aee114f2332fe785fc Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 28 May 2026 22:10:06 +0300 Subject: [PATCH 021/106] Add `ReplicationProfileSetFieldValue` endpoint If we get PUT for profile fields on an instance that isn't the profile updates stream writer, we'll route the request to be finished on the correct stream writer via http replication. --- docs/workers.md | 9 +++ synapse/handlers/profile.py | 36 ++++++++++ synapse/replication/http/__init__.py | 2 + synapse/replication/http/profile.py | 101 +++++++++++++++++++++++++++ synapse/rest/client/profile.py | 41 ++++++----- 5 files changed, 173 insertions(+), 16 deletions(-) create mode 100644 synapse/replication/http/profile.py diff --git a/docs/workers.md b/docs/workers.md index 8d3aad19c66..e37592c9a6c 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -584,6 +584,15 @@ configured as stream writer for the `quarantined_media_changes` stream: ^/_synapse/admin/v1/quarantine_media/.*$ +#### The `profile_updates` stream + +The `profile_updates` stream supports multiple writers. The following endpoints +can be handled by any worker, but PUT and DELETE should be routed directly to one of the +workers configured as stream writer for the `profile_updates` stream: + + ^/_matrix/client/(api/v1|r0|v3|unstable)/profile/.*/[^/]+$ + + #### Restrict outbound federation traffic to a specific set of workers The diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 5a052baec20..ce044aa4804 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -599,6 +599,42 @@ async def get_profile_field( return result.get(field_name) + async def set_field( + self, + *, + target_user: UserID, + requester: Requester, + field_name: str, + new_value: str, + by_admin: bool = False, + propagate: bool = False, + ) -> None: + """Wrapper function for setting any profile field for a user.""" + if field_name == ProfileFields.DISPLAYNAME: + await self.set_displayname( + target_user=target_user, + requester=requester, + new_displayname=new_value, + by_admin=by_admin, + propagate=propagate, + ) + elif field_name == ProfileFields.AVATAR_URL: + await self.set_avatar_url( + target_user=target_user, + requester=requester, + new_avatar_url=new_value, + by_admin=by_admin, + propagate=propagate, + ) + else: + await self.set_profile_field( + target_user=target_user, + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=by_admin, + ) + async def set_profile_field( self, target_user: UserID, diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index 68cc6ce1fc6..d934ef80678 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -30,6 +30,7 @@ login, membership, presence, + profile, push, register, send_events, @@ -59,6 +60,7 @@ def register_servlets(self, hs: "HomeServer") -> None: push.register_servlets(hs, self) state.register_servlets(hs, self) devices.register_servlets(hs, self) + profile.register_servlets(hs, self) # The following can't currently be instantiated on workers. if hs.config.worker.worker_app is None: diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py new file mode 100644 index 00000000000..ccefc4ed4a0 --- /dev/null +++ b/synapse/replication/http/profile.py @@ -0,0 +1,101 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +import logging +from typing import TYPE_CHECKING + +from twisted.web.server import Request + +from synapse.http.server import HttpServer +from synapse.replication.http._base import ReplicationEndpoint +from synapse.types import JsonDict, UserID, create_requester + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class ReplicationProfileSetFieldValue(ReplicationEndpoint): + """Set profile field for a user. + + The POST looks like: + + POST /_synapse/replication/profile_set_field_value/ + + { + "requester_id": "@user:domain.tld", + "field_name": "displayname", + "new_value": "User Display Name", + "by_admin": False, + "propagate": False, + } + + 200 OK + + {} + """ + + NAME = "profile_set_field_value" + PATH_ARGS = ("user_id",) + METHOD = "POST" + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self._profile_handler = hs.get_profile_handler() + + @staticmethod + async def _serialize_payload( # type: ignore[override] + requester_id: str, + field_name: str, + new_value: str | None, + by_admin: bool = False, + propagate: bool = False, + authenticated_entity: str | None = None, + ) -> JsonDict: + return { + "requester_id": requester_id, + "field_name": field_name, + "new_value": new_value, + "by_admin": by_admin, + "propagate": propagate, + "authenticated_entity": authenticated_entity, + } + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_id: str + ) -> tuple[int, JsonDict]: + # Create a requester object with potentially an authenticated_entity, + # ie an admin who has done the request on behalf of the user. + requester = create_requester( + user_id=user_id, + authenticated_entity=content["authenticated_entity"] if content["by_admin"] else None, + ) + await self._profile_handler.set_field( + target_user=UserID.from_string(user_id), + requester=requester, + field_name=content["field_name"], + new_value=content["new_value"], + by_admin=content["by_admin"], + propagate=content["propagate"], + ) + + return (200, {}) + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + ReplicationProfileSetFieldValue(hs).register(http_server) diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 3ecba9659f7..e7b2e4bcda5 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -35,6 +35,7 @@ parse_json_object_from_request, ) from synapse.http.site import SynapseRequest +from synapse.replication.http.profile import ReplicationProfileSetFieldValue from synapse.rest.client._base import client_patterns from synapse.types import JsonDict, JsonValue, UserID from synapse.util.stringutils import is_namedspaced_grammar @@ -160,13 +161,6 @@ async def on_GET( async def on_PUT( self, request: SynapseRequest, user_id: str, field_name: str ) -> tuple[int, JsonDict]: - if not self._is_profile_worker: - raise SynapseError( - HTTPStatus.METHOD_NOT_ALLOWED, - "Can only handle PUT /profile on instances configured to handle the profile_updates stream writer", - Codes.UNRECOGNIZED, - ) - if not UserID.is_valid(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM @@ -214,17 +208,32 @@ async def on_PUT( Codes.USER_ACCOUNT_SUSPENDED, ) - if field_name == ProfileFields.DISPLAYNAME: - await self.profile_handler.set_displayname( - user, requester, new_value, by_admin=is_admin, propagate=propagate - ) - elif field_name == ProfileFields.AVATAR_URL: - await self.profile_handler.set_avatar_url( - user, requester, new_value, by_admin=is_admin, propagate=propagate + if self._is_profile_worker: + await self.profile_handler.set_field( + target_user=user, + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=is_admin, + propagate=propagate, ) else: - await self.profile_handler.set_profile_field( - user, requester, field_name, new_value, by_admin=is_admin + # Offload to the right worker via http replication + set_profile_data_client = ReplicationProfileSetFieldValue.make_client( + self.hs + ) + profile_updates_writer_instance = ( + self.hs.config.worker.writers.profile_updates[0] + ) + await set_profile_data_client( + instance_name=profile_updates_writer_instance, + user_id=user.to_string(), + requester_id=requester.user.to_string(), + field_name=field_name, + new_value=new_value, + by_admin=is_admin, + propagate=propagate, + authenticated_entity=requester.authenticated_entity, ) return 200, {} From e4c35cef26c7a4dbc5b3ed372bba55538248805b Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 28 May 2026 22:13:54 +0300 Subject: [PATCH 022/106] Add missing value to docstring --- synapse/replication/http/profile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py index ccefc4ed4a0..93b7be2d074 100644 --- a/synapse/replication/http/profile.py +++ b/synapse/replication/http/profile.py @@ -41,6 +41,7 @@ class ReplicationProfileSetFieldValue(ReplicationEndpoint): "new_value": "User Display Name", "by_admin": False, "propagate": False, + "authenticated_entity": "@admin:domain.tld", } 200 OK From c30e0d9103b3228a160fa8f444f2a5a0692edd5e Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 28 May 2026 22:26:58 +0300 Subject: [PATCH 023/106] Also replicate DELETE profile fields to the right stream worker --- synapse/handlers/profile.py | 24 ++++++++++++------ synapse/replication/http/profile.py | 6 +++-- synapse/rest/client/profile.py | 39 ++++++++++++++++++----------- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index ce044aa4804..5c4a54b9208 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -627,13 +627,23 @@ async def set_field( propagate=propagate, ) else: - await self.set_profile_field( - target_user=target_user, - requester=requester, - field_name=field_name, - new_value=new_value, - by_admin=by_admin, - ) + # For custom fields, we need to call a separate delete method + # for empty strings. + if new_value == "": + await self.delete_profile_field( + target_user=target_user, + requester=requester, + field_name=field_name, + by_admin=by_admin, + ) + else: + await self.set_profile_field( + target_user=target_user, + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=by_admin, + ) async def set_profile_field( self, diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py index 93b7be2d074..c5e5448f8ef 100644 --- a/synapse/replication/http/profile.py +++ b/synapse/replication/http/profile.py @@ -63,7 +63,7 @@ def __init__(self, hs: "HomeServer"): async def _serialize_payload( # type: ignore[override] requester_id: str, field_name: str, - new_value: str | None, + new_value: str, by_admin: bool = False, propagate: bool = False, authenticated_entity: str | None = None, @@ -84,7 +84,9 @@ async def _handle_request( # type: ignore[override] # ie an admin who has done the request on behalf of the user. requester = create_requester( user_id=user_id, - authenticated_entity=content["authenticated_entity"] if content["by_admin"] else None, + authenticated_entity=content["authenticated_entity"] + if content["by_admin"] + else None, ) await self._profile_handler.set_field( target_user=UserID.from_string(user_id), diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index e7b2e4bcda5..7cd4678bc06 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -241,12 +241,6 @@ async def on_PUT( async def on_DELETE( self, request: SynapseRequest, user_id: str, field_name: str ) -> tuple[int, JsonDict]: - if not self._is_profile_worker: - raise SynapseError( - HTTPStatus.METHOD_NOT_ALLOWED, - "Can only handle DELETE /profile on instances configured to handle the profile_updates stream writer", - Codes.UNRECOGNIZED, - ) if not UserID.is_valid(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM @@ -286,17 +280,32 @@ async def on_DELETE( Codes.USER_ACCOUNT_SUSPENDED, ) - if field_name == ProfileFields.DISPLAYNAME: - await self.profile_handler.set_displayname( - user, requester, "", by_admin=is_admin, propagate=propagate - ) - elif field_name == ProfileFields.AVATAR_URL: - await self.profile_handler.set_avatar_url( - user, requester, "", by_admin=is_admin, propagate=propagate + if self._is_profile_worker: + await self.profile_handler.set_field( + target_user=user, + requester=requester, + field_name=field_name, + new_value="", + by_admin=is_admin, + propagate=propagate, ) else: - await self.profile_handler.delete_profile_field( - user, requester, field_name, by_admin=is_admin + # Offload to the right worker via http replication + set_profile_data_client = ReplicationProfileSetFieldValue.make_client( + self.hs + ) + profile_updates_writer_instance = ( + self.hs.config.worker.writers.profile_updates[0] + ) + await set_profile_data_client( + instance_name=profile_updates_writer_instance, + user_id=user.to_string(), + requester_id=requester.user.to_string(), + field_name=field_name, + new_value="", + by_admin=is_admin, + propagate=propagate, + authenticated_entity=requester.authenticated_entity, ) return 200, {} From 352efe18c9fff09e3ea8e35f7aa110851ae3f1d3 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 29 May 2026 11:55:03 +0300 Subject: [PATCH 024/106] Fix replication of profile updates to the right stream writer --- synapse/replication/http/profile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py index c5e5448f8ef..c967a45c556 100644 --- a/synapse/replication/http/profile.py +++ b/synapse/replication/http/profile.py @@ -61,6 +61,7 @@ def __init__(self, hs: "HomeServer"): @staticmethod async def _serialize_payload( # type: ignore[override] + user_id: str, requester_id: str, field_name: str, new_value: str, From 56a3647b8faac90f143c8a8aad2dd79eb025e6f7 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 29 May 2026 11:58:07 +0300 Subject: [PATCH 025/106] Remove unnecessary upgrade notes for profile updates stream writer --- docs/upgrade.md | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 8c994785659..44ab342d5a2 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -119,32 +119,6 @@ stacking them up. You can monitor the currently running background updates with # Upgrading to v1.152.0 -## Profile Updates Stream Writer Workers - -This version of Synapse adds a new `profile_updates` writer stream. The -following endpoints may now only be handled by either the main process, or a -worker that is designed a "profile_updates" writer. If you are already routing -the following endpoints to a worker: - -``` -/_matrix/client/(api/v1|r0|v3)/profile//(?) -/_matrix/client/unstable/uk.tcpip.msc4133/profile//(?) -``` - -those worker(s) need to be marked as a stream writer for the `profile_updates` -stream in the shared config, using the -[`stream_writers`](https://element-hq.github.io/synapse/v1.151/usage/configuration/config_documentation.html#stream_writers) -config option: - -```yaml -stream_writers: - profile_updates: worker1 -``` - -as well as included in the -[`instance_map`](https://element-hq.github.io/synapse/v1.151/usage/configuration/config_documentation.html#instance_map) -config option. - ## Workers which quarantine media must be stream writers A new [`quarantined_media_changes` stream writer](./workers.md#the-quarantined_media_changes-stream) is From 632c7ea42cac549c474395a85b7653651be9d2fa Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 29 May 2026 12:06:15 +0300 Subject: [PATCH 026/106] Add a docstring, remove premature optimization --- synapse/handlers/profile.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 5c4a54b9208..4a1c8388f71 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -102,23 +102,30 @@ def __init__(self, hs: "HomeServer"): ) self._worker_locks = hs.get_worker_locks_handler() - async def _notify_profile_update(self, user_id: UserID, stream_id: int) -> None: - room_ids = await self.store.get_rooms_for_user(user_id.to_string()) - if not room_ids: - return - - self._notifier.on_new_event( - StreamKeyType.PROFILE_UPDATES, stream_id, rooms=room_ids - ) - async def _record_profile_updates( self, user_id: UserID, updated_fields: list[str] ) -> None: + """ + Record user profile updates to our stream updates table. + + Args: + user_id: The user whose profile has had updates. + updated_fields: A list of the names of the fields that were updated. + + Returns: + None + """ if not self._msc4429_enabled or not updated_fields: return stream_id = await self.store.add_profile_updates(user_id, updated_fields) - await self._notify_profile_update(user_id, stream_id) + room_ids = await self.store.get_rooms_for_user(user_id.to_string()) + if not room_ids: + return + + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, stream_id, rooms=room_ids + ) async def get_profile(self, user_id: str, ignore_backoff: bool = True) -> JsonDict: """ From 92bfed7c188818b6cb08b87a055ab7af591b6df3 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 29 May 2026 12:40:49 +0300 Subject: [PATCH 027/106] Don't return null's in initial sync for profiles with no value for the asked field --- synapse/handlers/sync.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 35dac31cce0..dacba0027b6 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2170,14 +2170,15 @@ async def _generate_initial_sync_entry_for_profile_updates( per_user_updates: dict[str, JsonValue | None] = {} for field_name in profile_fields: - if field_name == ProfileFields.DISPLAYNAME: + if displayname and field_name == ProfileFields.DISPLAYNAME: per_user_updates[field_name] = displayname - elif field_name == ProfileFields.AVATAR_URL: + elif avatar_url and field_name == ProfileFields.AVATAR_URL: per_user_updates[field_name] = avatar_url - else: + elif custom_fields.get(field_name): per_user_updates[field_name] = custom_fields.get(field_name) - all_updates[other_user_id] = per_user_updates + if len(all_updates.keys()): + all_updates[other_user_id] = per_user_updates sync_result_builder.profile_updates = all_updates return From 4697e73880eb09d1bfe9991f7da52f22c5a1681a Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 29 May 2026 15:02:16 +0300 Subject: [PATCH 028/106] Simplify collecting profile field updates for sync Change `get_profile_data_for_users` to compile a friendly dictionary, so we don't need to special if/else case when collecting data for sync. The sync methods just treat every field identically, without displayname or avatar_url having any specific importance. --- synapse/handlers/sync.py | 51 ++++++++++------------- synapse/storage/databases/main/profile.py | 41 +++++++++--------- 2 files changed, 44 insertions(+), 48 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index dacba0027b6..86e48a4f940 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -26,6 +26,7 @@ Any, Mapping, Sequence, + cast, ) import attr @@ -37,7 +38,6 @@ EventContentFields, EventTypes, Membership, - ProfileFields, StickyEvent, ) from synapse.api.filtering import FilterCollection @@ -2159,29 +2159,27 @@ async def _generate_initial_sync_entry_for_profile_updates( profile_data_by_user = await self.store.get_profile_data_for_users(user_ids) - all_updates: dict[str, dict[str, JsonValue | None]] = {} + # Serialise the profile updates into the sync response format. + profile_updates: dict[str, dict[str, JsonValue | None]] = {} for other_user_id in user_ids: profile_data = profile_data_by_user.get(other_user_id) if profile_data is None: # Don't generate anything for users with no profile data + # in initial sync. continue - displayname, avatar_url, custom_fields = profile_data - - per_user_updates: dict[str, JsonValue | None] = {} + per_user_updates: dict[str, JsonValue] = {} for field_name in profile_fields: - if displayname and field_name == ProfileFields.DISPLAYNAME: - per_user_updates[field_name] = displayname - elif avatar_url and field_name == ProfileFields.AVATAR_URL: - per_user_updates[field_name] = avatar_url - elif custom_fields.get(field_name): - per_user_updates[field_name] = custom_fields.get(field_name) + if profile_data.get(field_name): + per_user_updates[field_name] = cast( + JsonValue, profile_data[field_name] + ) - if len(all_updates.keys()): - all_updates[other_user_id] = per_user_updates + if per_user_updates: + profile_updates[other_user_id] = per_user_updates - sync_result_builder.profile_updates = all_updates - return + if profile_updates: + sync_result_builder.profile_updates = profile_updates async def _generate_sync_entry_for_profile_updates( self, sync_result_builder: "SyncResultBuilder" @@ -2246,22 +2244,19 @@ async def _generate_sync_entry_for_profile_updates( # Serialise the profile updates into the sync response format. profile_updates: dict[str, dict[str, JsonValue | None]] = {} for other_user_id, fields in user_fields.items(): - displayname = None - avatar_url = None - custom_fields: JsonDict = {} - profile_data = profile_data_by_user.get(other_user_id) - if profile_data is not None: - displayname, avatar_url, custom_fields = profile_data + if profile_data is None: + # No profile data for this user, just return a blank dictionary + # in incremental sync, telling the clients to remove all profile + # information for this user. + profile_updates[other_user_id] = {} + continue - per_user_updates: dict[str, JsonValue | None] = {} + per_user_updates: dict[str, JsonValue] = {} for field_name in fields: - if field_name == ProfileFields.DISPLAYNAME: - per_user_updates[field_name] = displayname - elif field_name == ProfileFields.AVATAR_URL: - per_user_updates[field_name] = avatar_url - else: - per_user_updates[field_name] = custom_fields.get(field_name) + per_user_updates[field_name] = cast( + JsonValue, profile_data.get(field_name) + ) if per_user_updates: profile_updates[other_user_id] = per_user_updates diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index ae1279fbf30..d6e503fcfdb 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -27,7 +27,7 @@ from synapse.api.constants import ProfileFields from synapse.api.errors import Codes, StoreError from synapse.replication.tcp.streams._base import ProfileUpdatesStream -from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause +from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, @@ -408,32 +408,33 @@ def _get_profile_updates_for_fields_txn( async def get_profile_data_for_users( self, user_ids: Collection[str] - ) -> dict[str, tuple[str | None, str | None, JsonDict]]: + ) -> dict[str, dict[str, str | JsonDict | None]]: """Fetch displayname/avatar_url/custom fields for a list of users.""" if not user_ids: return {} - rows = cast( - list[tuple[str, str | None, str | None, object | None]], - await self.db_pool.simple_select_many_batch( - table="profiles", - column="full_user_id", - iterable=user_ids, - retcols=("full_user_id", "displayname", "avatar_url", "fields"), - desc="get_profile_data_for_users", - ), + rows = await self.db_pool.simple_select_many_batch( + table="profiles", + column="full_user_id", + iterable=user_ids, + retcols=("full_user_id", "displayname", "avatar_url", "fields"), + desc="get_profile_data_for_users", ) - results: dict[str, tuple[str | None, str | None, JsonDict]] = {} + results: dict[str, dict[str, str | JsonDict | None]] = {} for full_user_id, displayname, avatar_url, fields in rows: - if fields is None: - fields_dict: JsonDict = {} - elif isinstance(fields, (str, bytes, bytearray, memoryview)): - fields_dict = cast(JsonDict, db_to_json(fields)) - else: - fields_dict = cast(JsonDict, fields) - - results[full_user_id] = (displayname, avatar_url, fields_dict) + user_fields = fields + # The SQLite driver doesn't automatically convert JSON to + # Python objects + if isinstance(self.database_engine, Sqlite3Engine) and fields: + user_fields = json.loads(fields) + base_fields = { + ProfileFields.DISPLAYNAME: displayname, + ProfileFields.AVATAR_URL: avatar_url, + } + user_fields.update(base_fields) + + results[full_user_id] = user_fields return results From 3444aef480e62cc55c341dba481031413e3fec2b Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 1 Jun 2026 11:57:15 +0300 Subject: [PATCH 029/106] Improve docstrings of `get_updated_profile_updates` and `get_profile_updates_for_fields` --- synapse/storage/databases/main/profile.py | 28 ++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index d6e503fcfdb..df49083de48 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -336,9 +336,20 @@ def get_profile_updates_stream_id_generator(self) -> MultiWriterIdGenerator: return self._profile_updates_id_gen async def get_updated_profile_updates( - self, from_id: int, to_id: int, limit: int + self, *, from_id: int, to_id: int, limit: int ) -> list[tuple[int, str, str]]: - """Get profile updates that have changed, for the profile_updates stream.""" + """Get updates to profile updates between two stream IDs. + + Bounds: from_id < ... <= to_id + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + limit: The maximum number of rows to return + + Returns: + list of tuples representing stream_id, user_id and field_name + """ if from_id == to_id: return [] @@ -367,7 +378,18 @@ async def get_profile_updates_for_fields( to_id: int, field_names: Iterable[str], ) -> list[ProfileUpdate]: - """Get profile update markers for the given fields in a stream range.""" + """Get profile update markers for the given fields in a stream range. + + Bounds: from_id < ... <= to_id + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + field_names: List of field names to filter against. + + Returns: + list of ProfileUpdates update rows + """ if from_id == to_id: return [] From 4c2bbbce2016e589574233214f6ec64ea2ebe4a5 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 1 Jun 2026 12:19:05 +0300 Subject: [PATCH 030/106] Use a "many" insert for `add_profile_updates` --- synapse/storage/databases/main/profile.py | 33 ++++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index df49083de48..4e9c2e4f01e 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -477,19 +477,32 @@ def _add_profile_updates_txn(txn: LoggingTransaction) -> int: stream_ids = self._profile_updates_id_gen.get_next_mult_txn( txn, len(updated_fields) ) + values = [] + inserted_ts = self.clock.time_msec() for stream_id, field_name in zip(stream_ids, updated_fields): - self.db_pool.simple_insert_txn( - txn, - table="profile_updates", - values={ - "stream_id": stream_id, - "instance_name": self._instance_name, - "user_id": user_id_str, - "field_name": field_name, - "inserted_ts": self.clock.time_msec(), - }, + values.append( + [ + stream_id, + self._instance_name, + user_id_str, + field_name, + inserted_ts, + ] ) + self.db_pool.simple_insert_many_txn( + txn, + table="profile_updates", + keys=[ + "stream_id", + "instance_name", + "user_id", + "field_name", + "inserted_ts", + ], + values=values, + ) + return stream_ids[-1] return await self.db_pool.runInteraction( From 4cf5e098a5266b663937fa3234f27967c56fac8b Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 1 Jun 2026 12:22:51 +0300 Subject: [PATCH 031/106] Clarify `_generate_initial_sync_entry_for_profile_updates` is for local users only currently --- synapse/handlers/sync.py | 5 +---- synapse/storage/databases/main/profile.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 86e48a4f940..dc786f05da3 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2142,10 +2142,7 @@ async def _generate_initial_sync_entry_for_profile_updates( Build an initial sync entry for profile updates and attach it to the given `sync_result_builder`. - Note: Only the profile information for local users is returned. This is - to prevent fetching *too* many profiles in one request. Clients should - ideally instead first fetch profiles on-demand, then track updates for - all users via incremental `/sync`. + Note: Currently, only profile updates of local users are generated. Args: user_id: The Matrix ID of the user to generate the sync entry for. diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 4e9c2e4f01e..e70443fa773 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -431,7 +431,17 @@ def _get_profile_updates_for_fields_txn( async def get_profile_data_for_users( self, user_ids: Collection[str] ) -> dict[str, dict[str, str | JsonDict | None]]: - """Fetch displayname/avatar_url/custom fields for a list of users.""" + """Fetch displayname/avatar_url/custom fields for a list of users. + + Currently, this returns only local users as the `profiles` table only + tracks local users. + + Args: + user_ids: List of user IDs to filter against. + + Returns: + Dictionary of displayname/avatar_url/custom fields for a list of users. + """ if not user_ids: return {} From c082da40338242ff631ad6a8e8f6bc93bd198b67 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 1 Jun 2026 12:35:16 +0300 Subject: [PATCH 032/106] Allow a `msc4429_enabled` config alias outside of the experimental config --- synapse/config/experimental.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 28fe576b899..62d15117702 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -560,7 +560,12 @@ def read_config( self.msc4133_enabled: bool = experimental.get("msc4133_enabled", False) # MSC4429: Profile updates for legacy /sync - self.msc4429_enabled: bool = experimental.get("msc4429_enabled", False) + self.msc4429_enabled: bool = bool( + experimental.get("msc4429_enabled", False) + or + # Allow an alias outside the "experimental" section. + config.get("msc4429_enabled", False) + ) # MSC4143: Matrix RTC Transport using Livekit Backend self.msc4143_enabled: bool = experimental.get("msc4143_enabled", False) From aeeb9336a6a5032bc0a4ed28e9ebde7adcbbc7ca Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 1 Jun 2026 15:22:01 +0300 Subject: [PATCH 033/106] Restrict initial sync profiles to local users for now We don't have federated profile updates (yet), so use a cheaper database call to fetch only local users for the initial sync. This mirrors incremental sync as currently there are only local profiles being pushed into the profile updates stream table. --- synapse/handlers/sync.py | 4 ++-- synapse/storage/databases/main/roommember.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index dc786f05da3..029259e33ca 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2149,8 +2149,8 @@ async def _generate_initial_sync_entry_for_profile_updates( sync_result_builder: profile_fields: The list of field IDs to filter for. """ - user_ids = await self.store.get_users_who_share_room_with_user(user_id) - user_ids = {u for u in user_ids if self._is_mine_id(u)} + # Currently, limited to only local profiles, so filter remote servers out + user_ids = await self.store.get_local_users_who_share_room_with_user(user_id) if not user_ids: return diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 736f3e4c781..a645e4f3a59 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -982,6 +982,17 @@ async def get_users_who_share_room_with_user(self, user_id: str) -> set[str]: return user_who_share_room + async def get_local_users_who_share_room_with_user(self, user_id: str) -> set[str]: + """Returns the set of local users who share a room with `user_id`""" + room_ids = await self.get_rooms_for_user(user_id) + + user_who_share_room: set[str] = set() + for room_id in room_ids: + user_ids = await self.get_local_users_in_room(room_id) + user_who_share_room.update(user_ids) + + return user_who_share_room + @cached(cache_context=True, iterable=True) async def get_mutual_rooms_between_users( self, user_ids: frozenset[str], cache_context: _CacheContext From 54c299d09127316f66fd435f2c954b1e6b46f083 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 1 Jun 2026 15:24:16 +0300 Subject: [PATCH 034/106] Clarify newsfile that this pr is for local users only --- changelog.d/19556.feature | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.d/19556.feature b/changelog.d/19556.feature index 28ad5b06573..6422a6160d3 100644 --- a/changelog.d/19556.feature +++ b/changelog.d/19556.feature @@ -1 +1,2 @@ -Implement experimental support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). \ No newline at end of file +Implement experimental support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). +Currently limited to local users only for the sync results. \ No newline at end of file From 3cf7555f84fe58145e7df7e69166d62a54e0b96c Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 1 Jun 2026 16:01:30 +0300 Subject: [PATCH 035/106] Clarify profile updates sync response with a comment --- synapse/handlers/sync.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 029259e33ca..6920a60759e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -242,6 +242,7 @@ class SyncResult: next_batch: StreamToken presence: list[UserPresenceState] account_data: list[JsonDict] + # user ID -> {profile field -> value | null if unset } profile_updates: dict[str, dict[str, JsonValue | None]] joined: list[JoinedSyncResult] invited: list[InvitedSyncResult] @@ -2239,6 +2240,7 @@ async def _generate_sync_entry_for_profile_updates( ) # Serialise the profile updates into the sync response format. + # user ID -> {profile field -> value | null if unset } profile_updates: dict[str, dict[str, JsonValue | None]] = {} for other_user_id, fields in user_fields.items(): profile_data = profile_data_by_user.get(other_user_id) From d254167e193cce5ce9067f1bceb316b2af1cb6ef Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 1 Jun 2026 16:04:21 +0300 Subject: [PATCH 036/106] Clarify in `_generate_sync_entry_for_profile_updates` that only local profiles will be returned --- synapse/handlers/sync.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 6920a60759e..7733124c043 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2186,6 +2186,8 @@ async def _generate_sync_entry_for_profile_updates( Build a sync entry for profile updates and attach it to the given `sync_result_builder`. + Currently only local profiles updates will be included in the sync response. + Args: sync_result_builder: """ From e236ddb32b07505b2a0cf809fbe85212ccc5051c Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 2 Jun 2026 21:35:01 +0300 Subject: [PATCH 037/106] Add SyncProfileUpdatesTestCase For initial sync --- tests/handlers/test_sync.py | 167 ++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index d2b2523321b..a24ee9ed2e4 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -17,6 +17,7 @@ # [This file includes modifications made by New Vector Limited] # # +import json from http import HTTPStatus from typing import Collection, ContextManager from unittest.mock import AsyncMock, Mock, patch @@ -55,6 +56,7 @@ import tests.unittest import tests.utils from tests.test_utils.event_builders import make_test_pdu_event +from tests.unittest import override_config _request_key = 0 @@ -1152,6 +1154,171 @@ def generate_sync_config( ) +class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): + """Tests Sync Handler for profile updates.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + self.sync_handler = self.hs.get_sync_handler() + self.store = self.hs.get_datastores().main + self.user = self.register_user("user", "password") + self.tok = self.login("user", "password") + self.other_user = self.register_user("other_user", "password") + self.other_tok = self.login("other_user", "password") + self.joined_room = self.helper.create_room_as(self.user, tok=self.tok) + self.get_success( + self.store.set_profile_field( + user_id=UserID.from_string(self.user), + field_name="m.status", + new_value=json.dumps( + {"text": "Swimming in the Great Lakes!", "emoji": "🏊"} + ), + ) + ) + self.helper.join( + room=self.joined_room, user=self.other_user, tok=self.other_tok + ) + + def test_initial_sync_no_profile_updates_if_not_enabled(self) -> None: + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + self.user, + ), + request_key=generate_request_key(), + ) + ) + self.assertEqual(initial_result.profile_updates, {}) + + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + ), + request_key=generate_request_key(), + ) + ) + self.assertEqual( + initial_result.profile_updates, + {}, + ) + + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_initial_sync_responds_with_all_known_profiles(self) -> None: + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertEqual( + initial_result.profile_updates["@user:test"]["m.status"], + '{"text": "Swimming in the Great Lakes!", "emoji": "\\ud83c\\udfca"}', + ) + self.assertEqual( + initial_result.profile_updates["@user:test"]["displayname"], "user" + ) + self.assertEqual( + initial_result.profile_updates["@other_user:test"]["displayname"], + "other_user", + ) + self.assertCountEqual( + initial_result.profile_updates.keys(), + [ + "@other_user:test", + "@user:test", + ], + ) + + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( + self, + ) -> None: + """ + This test ensures lazy loading sync only returns profiles that we also have + events for in the sync response. The second room in this test has the most + recent events from "third_user" and thus we don't get the profile of + "other_user" down the line, who is in the the same rooms as the syncer, + but not in the second room. + """ + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + second_room = self.helper.create_room_as(self.user, tok=self.tok) + self.helper.join( + room=second_room, + user=third_user, + tok=third_tok, + ) + + requester = create_requester(self.user) + + self.helper.send_messages(room_id=second_room, num_events=10, tok=third_tok) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertEqual( + initial_result.profile_updates["@user:test"]["m.status"], + '{"text": "Swimming in the Great Lakes!", "emoji": "\\ud83c\\udfca"}', + ) + self.assertEqual( + initial_result.profile_updates["@user:test"]["displayname"], "user" + ) + self.assertEqual( + initial_result.profile_updates["@third_user:test"]["displayname"], + "third_user", + ) + self.assertCountEqual( + initial_result.profile_updates.keys(), + [ + "@third_user:test", + "@user:test", + ], + ) + + class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): """Tests Sync Handler state behavior when using `use_state_after.""" From 1ae91c87d05b5fa32becb2f64939f7856b3628a7 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 2 Jun 2026 21:36:00 +0300 Subject: [PATCH 038/106] Fix `get_profile_data_for_users` if user does not have custom fields --- synapse/storage/databases/main/profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index e70443fa773..0e96d1a9660 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -455,7 +455,7 @@ async def get_profile_data_for_users( results: dict[str, dict[str, str | JsonDict | None]] = {} for full_user_id, displayname, avatar_url, fields in rows: - user_fields = fields + user_fields = fields or {} # The SQLite driver doesn't automatically convert JSON to # Python objects if isinstance(self.database_engine, Sqlite3Engine) and fields: From 1a59c0ae0fe90e1ea52bccd4da85b3c7526e3d04 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 2 Jun 2026 21:38:16 +0300 Subject: [PATCH 039/106] Only return profiles for initial sync that have events when using lazy loading --- synapse/handlers/sync.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 7733124c043..50c3fd182ee 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1766,9 +1766,6 @@ async def generate_sync_result( if not sync_config.filter_collection.blocks_all_global_account_data(): await self._generate_sync_entry_for_account_data(sync_result_builder) - if self.hs_config.experimental.msc4429_enabled: - await self._generate_sync_entry_for_profile_updates(sync_result_builder) - # Presence data is included if the server has it enabled and not filtered out. include_presence_data = bool( self.hs_config.server.presence_enabled @@ -1864,6 +1861,12 @@ async def generate_sync_result( } ) + # Note, this needs to be after we collect `joined` sync results + # since we want to utilize the work we did to collect users into the + # lazy loading members cache + if self.hs_config.experimental.msc4429_enabled: + await self._generate_sync_entry_for_profile_updates(sync_result_builder) + logger.debug("Sync response calculation complete") return SyncResult( presence=sync_result_builder.presence, @@ -2152,6 +2155,15 @@ async def _generate_initial_sync_entry_for_profile_updates( """ # Currently, limited to only local profiles, so filter remote servers out user_ids = await self.store.get_local_users_who_share_room_with_user(user_id) + + sync_config = sync_result_builder.sync_config + lazy_load_members = sync_config.filter_collection.lazy_load_members() + if lazy_load_members: + # Only include members we've collected for lazy loading + cache_key = (sync_config.user.to_string(), sync_config.device_id) + cache = self.get_lazy_loaded_members_cache(cache_key) + user_ids = {user_id for user_id in user_ids if cache.get(user_id)} + if not user_ids: return From bd40add5d56ba237267b192a0f9d1a0afe02cb57 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 3 Jun 2026 11:57:27 +0300 Subject: [PATCH 040/106] Add some tests for profile handler updating profile_updates stream table and waking up notifiers --- tests/handlers/test_profile.py | 178 ++++++++++++++++++++++++++++++++- 1 file changed, 177 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 5152e8fc536..01dc21b87c4 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -31,7 +31,8 @@ from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer -from synapse.types import JsonDict, UserID +from synapse.storage.databases.main.profile import ProfileUpdate +from synapse.types import JsonDict, StreamKeyType, UserID from synapse.types.state import StateFilter from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -62,8 +63,10 @@ def register_query_handler( self.query_handlers[query_type] = handler self.mock_registry.register_query_handler = register_query_handler + self.mock_hs_notifier = Mock() hs = self.setup_test_homeserver( + notifier=self.mock_hs_notifier, federation_client=self.mock_federation, federation_server=Mock(), federation_registry=self.mock_registry, @@ -83,6 +86,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.frank_token = self.login(self.frank.localpart, "frankpassword") self.handler = hs.get_profile_handler() + self.on_new_event = self.mock_hs_notifier.on_new_event def test_get_my_name(self) -> None: self.get_success(self.store.set_profile_displayname(self.frank, "Frank")) @@ -161,6 +165,178 @@ def test_update_room_membership_on_set_displayname(self) -> None: ) self.assertEqual(membership[state_tuple].content["displayname"], "Frank Jr.") + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + def test_update_profile_does_not_update_stream_on_set_field_if_msc4429_not_enabled( + self, + field_name: str, + new_value: str, + ) -> None: + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + updates = self.get_success( + self.store.get_updated_profile_updates( + from_id=1, + to_id=2, + limit=1, + ) + ) + self.assertEqual(len(updates), 0) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + def test_update_profile_does_not_notify_notifier_on_set_field_if_msc4429_not_enabled( + self, + field_name: str, + new_value: str, + ) -> None: + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 0) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_update_profile_does_not_notify_notifier_on_set_field_if_user_not_in_rooms( + self, field_name: str, new_value: str + ) -> None: + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 0) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_update_profile_updates_stream_on_set_field( + self, field_name: str, new_value: str + ) -> None: + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + updates = self.get_success( + self.store.get_updated_profile_updates( + from_id=1, + to_id=2, + limit=1, + ) + ) + self.assertEqual(updates[0], (2, "@1234abcd:test", field_name)) + + fields_updates = self.get_success( + self.store.get_profile_updates_for_fields( + from_id=1, + to_id=2, + field_names=[field_name], + ) + ) + self.assertEqual( + fields_updates[0], + ProfileUpdate(stream_id=2, user_id="@1234abcd:test", field_name=field_name), + ) + + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value="", + ) + ) + delete_updates = self.get_success( + self.store.get_updated_profile_updates( + from_id=2, + to_id=3, + limit=1, + ) + ) + self.assertEqual(delete_updates[0], (3, "@1234abcd:test", field_name)) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_update_profile_notifies_notifier_on_set_field( + self, + field_name: str, + new_value: str, + ) -> None: + self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 1) + def test_background_update_room_membership_on_set_displayname(self) -> None: """Test that `set_displayname` returns immediately and that room membership updates are still done in background.""" From b61f748928a08806b9b9f5583c88ba794cefafd2 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 3 Jun 2026 18:08:36 +0300 Subject: [PATCH 041/106] Fix some broken tests due to sync token changes --- tests/rest/admin/test_room.py | 4 ++-- tests/rest/client/test_rooms.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index c4e4170c6f9..4deb3c29f41 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2549,7 +2549,7 @@ def test_timestamp_to_event(self) -> None: def test_topo_token_is_accepted(self) -> None: """Test Topo Token is accepted.""" - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), @@ -2563,7 +2563,7 @@ def test_topo_token_is_accepted(self) -> None: def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: """Test that stream token is accepted for forward pagination.""" - token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 10325c536a0..0538b03d7d0 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -2248,7 +2248,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.room_id = self.helper.create_room_as(self.user_id) def test_topo_token_is_accepted(self) -> None: - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) @@ -2259,7 +2259,7 @@ def test_topo_token_is_accepted(self) -> None: self.assertTrue("end" in channel.json_body) def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: - token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) From 36f0f64f62059950898dcffef3b687705344ae05 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 4 Jun 2026 11:26:50 +0300 Subject: [PATCH 042/106] Add "action" to profile updates stream This allows distinguishing between "a field was updated" or "something else happened". In this case, we want to know if the user left a room. Seeing this action while building the sync response we can then check if the user is still a member of the same rooms as the syncing user, and if not, we can tell the client to remove their profile. --- synapse/api/constants.py | 5 + synapse/handlers/profile.py | 25 ++++- synapse/handlers/sync.py | 28 ++++- synapse/replication/tcp/streams/_base.py | 7 +- synapse/storage/databases/main/profile.py | 63 +++++++---- .../main/delta/95/01_profile_updates.sql | 7 +- tests/handlers/test_profile.py | 19 +++- tests/handlers/test_sync.py | 102 ++++++++++++++++++ 8 files changed, 222 insertions(+), 34 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index eb9e6cc39b9..4d536b65107 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -383,6 +383,11 @@ class ProfileFields: AVATAR_URL: Final = "avatar_url" +class ProfileUpdateAction(enum.Enum): + LEFT_ROOM = "left_room" + UPDATE = "update" + + class StickyEventField(TypedDict): """ Dict content of the `sticky` part of an event. diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 4a1c8388f71..aede2256853 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -25,7 +25,7 @@ from twisted.internet.defer import CancelledError -from synapse.api.constants import ProfileFields +from synapse.api.constants import ProfileFields, ProfileUpdateAction from synapse.api.errors import ( AuthError, Codes, @@ -102,6 +102,8 @@ def __init__(self, hs: "HomeServer"): ) self._worker_locks = hs.get_worker_locks_handler() + hs.get_distributor().observe("user_left_room", self.user_left_room) + async def _record_profile_updates( self, user_id: UserID, updated_fields: list[str] ) -> None: @@ -118,7 +120,11 @@ async def _record_profile_updates( if not self._msc4429_enabled or not updated_fields: return - stream_id = await self.store.add_profile_updates(user_id, updated_fields) + stream_id = await self.store.add_profile_updates( + user_id=user_id, + updated_fields=updated_fields, + action=ProfileUpdateAction.UPDATE.value, + ) room_ids = await self.store.get_rooms_for_user(user_id.to_string()) if not room_ids: return @@ -412,6 +418,21 @@ async def set_avatar_url( if propagate: await self._update_join_states(requester, target_user) + async def _user_left_room(self, user_id: UserID) -> None: + await self.store.add_profile_updates( + user_id=user_id, + action=ProfileUpdateAction.LEFT_ROOM.value, + updated_fields=None, + ) + + def user_left_room(self, user: UserID, room_id: str) -> None: + if self.hs.is_mine_id(user.to_string()): + self.hs.run_as_background_process( + "profile._user_left_room", + self._user_left_room, + user_id=user, + ) + async def delete_profile_upon_deactivation( self, target_user: UserID, diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 50c3fd182ee..937f3823704 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -38,6 +38,7 @@ EventContentFields, EventTypes, Membership, + ProfileUpdateAction, StickyEvent, ) from synapse.api.filtering import FilterCollection @@ -2229,15 +2230,30 @@ async def _generate_sync_entry_for_profile_updates( if not updates: return - updated_user_ids = {update.user_id for update in updates} - shared_user_ids = await self.store.do_users_share_a_room( + updated_user_ids = { + update.user_id + for update in updates + if update.action == ProfileUpdateAction.UPDATE.value + } + shared_updated_user_ids = await self.store.do_users_share_a_room( user_id, updated_user_ids ) - shared_user_ids.add(user_id) + shared_updated_user_ids.add(user_id) + left_room_user_ids = { + update.user_id + for update in updates + if update.action == ProfileUpdateAction.LEFT_ROOM.value + } + shared_left_user_ids = await self.store.do_users_share_a_room( + user_id, left_room_user_ids + ) + no_longer_sharing_rooms_user_ids = set(left_room_user_ids) - set( + shared_left_user_ids + ) user_fields: dict[str, set[str]] = {} for update in updates: - if update.user_id not in shared_user_ids: + if not update.field_name or update.user_id not in shared_updated_user_ids: continue user_fields.setdefault(update.user_id, set()).add(update.field_name) @@ -2274,6 +2290,10 @@ async def _generate_sync_entry_for_profile_updates( if per_user_updates: profile_updates[other_user_id] = per_user_updates + for other_user_id in no_longer_sharing_rooms_user_ids: + # Return an empty dictionary to the client + profile_updates[other_user_id] = {} + if profile_updates: sync_result_builder.profile_updates = profile_updates diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index fab534a211d..92d10cc7849 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -772,6 +772,9 @@ class ProfileUpdatesStreamRow: user_id: UserID """The full user ID with the profile update.""" + action: str + """The action, either 'update' for a field update or 'left_room' if the user left a room, + see ProfileUpdateAction constant.""" field_name: str """The profile field that was updated, see https://spec.matrix.org/unstable/client-server-api/#profiles """ @@ -800,9 +803,9 @@ async def _update_function( ( stream_id, # These are the args to `ProfileUpdatesStreamRow` - (user_id, field_name), + (user_id, action, field_name), ) - for stream_id, user_id, field_name in updates + for stream_id, user_id, action, field_name in updates ] if not rows: diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 0e96d1a9660..8a9183ecd71 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -24,7 +24,7 @@ import attr from canonicaljson import encode_canonical_json -from synapse.api.constants import ProfileFields +from synapse.api.constants import ProfileFields, ProfileUpdateAction from synapse.api.errors import Codes, StoreError from synapse.replication.tcp.streams._base import ProfileUpdatesStream from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause @@ -52,7 +52,8 @@ class ProfileUpdate: stream_id: int user_id: str - field_name: str + field_name: str | None + action: str | None class ProfileWorkerStore(SQLBaseStore): @@ -337,7 +338,7 @@ def get_profile_updates_stream_id_generator(self) -> MultiWriterIdGenerator: async def get_updated_profile_updates( self, *, from_id: int, to_id: int, limit: int - ) -> list[tuple[int, str, str]]: + ) -> list[tuple[int, str, str, str | None]]: """Get updates to profile updates between two stream IDs. Bounds: from_id < ... <= to_id @@ -348,24 +349,24 @@ async def get_updated_profile_updates( limit: The maximum number of rows to return Returns: - list of tuples representing stream_id, user_id and field_name + list of tuples representing stream_id, user_id, action and field_name """ if from_id == to_id: return [] def _get_updated_profile_updates_txn( txn: LoggingTransaction, - ) -> list[tuple[int, str, str]]: + ) -> list[tuple[int, str, str, str | None]]: sql = """ SELECT - stream_id, user_id, field_name + stream_id, user_id, action, field_name FROM profile_updates WHERE ? < stream_id AND stream_id <= ? ORDER BY stream_id ASC LIMIT ? """ txn.execute(sql, (from_id, to_id, limit)) - return cast(list[tuple[int, str, str]], txn.fetchall()) + return cast(list[tuple[int, str, str, str | None]], txn.fetchall()) return await self.db_pool.runInteraction( "get_updated_profile_updates", _get_updated_profile_updates_txn @@ -404,20 +405,22 @@ def _get_profile_updates_for_fields_txn( txn.database_engine, "field_name", field_names ) sql = ( - "SELECT stream_id, user_id, field_name" + "SELECT stream_id, user_id, action, field_name" " FROM profile_updates" - f" WHERE ? < stream_id AND stream_id <= ? AND {clause}" + f" WHERE ? < stream_id AND stream_id <= ? AND ({clause}" + " OR action != ?) " " ORDER BY stream_id ASC" ) - txn.execute(sql, (from_id, to_id, *args)) - rows = cast(list[tuple[int, str, str]], txn.fetchall()) + txn.execute(sql, (from_id, to_id, ProfileUpdateAction.UPDATE.value, *args)) + rows = cast(list[tuple[int, str, str, str | None]], txn.fetchall()) updates: list[ProfileUpdate] = [] - for stream_id, user_id, field_name in rows: + for stream_id, user_id, action, field_name in rows: updates.append( ProfileUpdate( stream_id=stream_id, user_id=user_id, + action=action, field_name=field_name, ) ) @@ -473,33 +476,50 @@ async def get_profile_data_for_users( async def add_profile_updates( self, user_id: UserID, - updated_fields: list[str], + action: str, + updated_fields: list[str] | None, ) -> int: """Persist profile update markers and return the last stream ID.""" assert self._can_write_to_profile_updates + assert action in [action.value for action in ProfileUpdateAction] - if not updated_fields: + if action == ProfileUpdateAction.UPDATE.value and not updated_fields: return self._profile_updates_id_gen.get_current_token() + elif action == ProfileUpdateAction.LEFT_ROOM.value: + assert not updated_fields user_id_str = user_id.to_string() def _add_profile_updates_txn(txn: LoggingTransaction) -> int: - stream_ids = self._profile_updates_id_gen.get_next_mult_txn( - txn, len(updated_fields) - ) values = [] inserted_ts = self.clock.time_msec() - for stream_id, field_name in zip(stream_ids, updated_fields): + if updated_fields: + stream_ids = self._profile_updates_id_gen.get_next_mult_txn( + txn, len(updated_fields) + ) + for stream_id, field_name in zip(stream_ids, updated_fields): + values.append( + [ + stream_id, + self._instance_name, + user_id_str, + action, + field_name, + inserted_ts, + ] + ) + else: + stream_ids = [self._profile_updates_id_gen.get_next_txn(txn)] values.append( [ - stream_id, + stream_ids[0], self._instance_name, user_id_str, - field_name, + action, + None, inserted_ts, ] ) - self.db_pool.simple_insert_many_txn( txn, table="profile_updates", @@ -507,6 +527,7 @@ def _add_profile_updates_txn(txn: LoggingTransaction) -> int: "stream_id", "instance_name", "user_id", + "action", "field_name", "inserted_ts", ], diff --git a/synapse/storage/schema/main/delta/95/01_profile_updates.sql b/synapse/storage/schema/main/delta/95/01_profile_updates.sql index 557d01f9ba3..61112476c44 100644 --- a/synapse/storage/schema/main/delta/95/01_profile_updates.sql +++ b/synapse/storage/schema/main/delta/95/01_profile_updates.sql @@ -18,9 +18,14 @@ CREATE TABLE profile_updates ( -- The full user ID user_id TEXT NOT NULL, + + -- Profile action that has happened, see ProfileUpdateAction enum. + action TEXT NOT NULL, + -- Profile field name that has been updated, -- see https://spec.matrix.org/unstable/client-server-api/#profiles - field_name TEXT NOT NULL, + -- This is only required if "action" is "update" + field_name TEXT NULL, -- Unix timestamp for debugging purposes inserted_ts BIGINT NOT NULL diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 01dc21b87c4..1ff05120c4a 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -26,7 +26,7 @@ from twisted.internet.testing import MemoryReactor import synapse.types -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, ProfileUpdateAction from synapse.api.errors import AuthError, SynapseError from synapse.rest import admin from synapse.rest.client import login, room @@ -274,7 +274,10 @@ def test_update_profile_updates_stream_on_set_field( limit=1, ) ) - self.assertEqual(updates[0], (2, "@1234abcd:test", field_name)) + self.assertEqual( + updates[0], + (2, "@1234abcd:test", ProfileUpdateAction.UPDATE.value, field_name), + ) fields_updates = self.get_success( self.store.get_profile_updates_for_fields( @@ -285,7 +288,12 @@ def test_update_profile_updates_stream_on_set_field( ) self.assertEqual( fields_updates[0], - ProfileUpdate(stream_id=2, user_id="@1234abcd:test", field_name=field_name), + ProfileUpdate( + stream_id=2, + user_id="@1234abcd:test", + action=ProfileUpdateAction.UPDATE.value, + field_name=field_name, + ), ) self.get_success( @@ -303,7 +311,10 @@ def test_update_profile_updates_stream_on_set_field( limit=1, ) ) - self.assertEqual(delete_updates[0], (3, "@1234abcd:test", field_name)) + self.assertEqual( + delete_updates[0], + (3, "@1234abcd:test", ProfileUpdateAction.UPDATE.value, field_name), + ) @parameterized.expand( [ diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index a24ee9ed2e4..ae5b87a4adc 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1166,6 +1166,7 @@ class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: super().prepare(reactor, clock, hs) self.sync_handler = self.hs.get_sync_handler() + self.profile_handler = self.hs.get_profile_handler() self.store = self.hs.get_datastores().main self.user = self.register_user("user", "password") self.tok = self.login("user", "password") @@ -1318,6 +1319,107 @@ def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( ], ) + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_incremental_sync_sends_down_profile_updates( + self, + ) -> None: + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["m.status"], + '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', + ) + + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms( + self, + ) -> None: + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.helper.leave( + room=self.joined_room, user=self.other_user, tok=self.other_tok + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertEqual( + incremental_result.profile_updates["@other_user:test"], + {}, + ) + class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): """Tests Sync Handler state behavior when using `use_state_after.""" From 301e9c1ba7a10bf27f69421f01cc034d884f64d9 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 4 Jun 2026 13:10:29 +0300 Subject: [PATCH 043/106] ProfileUpdate.action should never be None --- synapse/storage/databases/main/profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 8a9183ecd71..9d726f29e47 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -52,8 +52,8 @@ class ProfileUpdate: stream_id: int user_id: str + action: str field_name: str | None - action: str | None class ProfileWorkerStore(SQLBaseStore): From 2d9f9430341ad3abf753dc0d83f3a94bbb13210a Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 4 Jun 2026 22:04:41 +0300 Subject: [PATCH 044/106] Collect lazy loaded members for profile updates in sync response from events --- synapse/handlers/sync.py | 45 ++++++++++++++++++----- tests/handlers/test_sync.py | 71 +++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 937f3823704..36319e4e61f 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2139,9 +2139,11 @@ async def _generate_sync_entry_for_account_data( async def _generate_initial_sync_entry_for_profile_updates( self, + *, user_id: str, sync_result_builder: "SyncResultBuilder", profile_fields: list[str], + include_users: set[str] | None, ) -> None: """ Build an initial sync entry for profile updates and attach it to the @@ -2153,17 +2155,16 @@ async def _generate_initial_sync_entry_for_profile_updates( user_id: The Matrix ID of the user to generate the sync entry for. sync_result_builder: profile_fields: The list of field IDs to filter for. + include_users: List of users profiles to include in the sync response, + for when we have calculated a list of users in our lazy loading + sync and want to only return those. """ # Currently, limited to only local profiles, so filter remote servers out user_ids = await self.store.get_local_users_who_share_room_with_user(user_id) - sync_config = sync_result_builder.sync_config - lazy_load_members = sync_config.filter_collection.lazy_load_members() - if lazy_load_members: - # Only include members we've collected for lazy loading - cache_key = (sync_config.user.to_string(), sync_config.device_id) - cache = self.get_lazy_loaded_members_cache(cache_key) - user_ids = {user_id for user_id in user_ids if cache.get(user_id)} + if include_users: + # Filter down to selected included users + user_ids = {user_id for user_id in user_ids if user_id in include_users} if not user_ids: return @@ -2213,9 +2214,33 @@ async def _generate_sync_entry_for_profile_updates( since_token = sync_result_builder.since_token now_token = sync_result_builder.now_token + sync_config = sync_result_builder.sync_config + lazy_load_members = sync_config.filter_collection.lazy_load_members() + include_users = None + if lazy_load_members: + # Collect members from the existing `sync_result_builder` data + include_users = set() + # invited + for invited in sync_result_builder.invited: + include_users.add(invited.invite.sender) + # joined + for joined in sync_result_builder.joined: + for timeline_event in joined.timeline.events: + include_users.add(timeline_event.event.sender) + # knocked + for knocked in sync_result_builder.knocked: + include_users.add(knocked.knock.sender) + # archived + for archived in sync_result_builder.archived: + for timeline_event in archived.timeline.events: + include_users.add(timeline_event.event.sender) + if since_token is None: await self._generate_initial_sync_entry_for_profile_updates( - user_id, sync_result_builder, profile_fields + user_id=user_id, + sync_result_builder=sync_result_builder, + profile_fields=profile_fields, + include_users=include_users, ) return @@ -2227,6 +2252,10 @@ async def _generate_sync_entry_for_profile_updates( to_id=now_token.profile_updates_key, field_names=profile_fields, ) + if include_users: + # Filter down to selected included users + updates = [update for update in updates if update.user_id in include_users] + if not updates: return diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index ae5b87a4adc..88ed55f873b 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1372,6 +1372,77 @@ def test_incremental_sync_sends_down_profile_updates( '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', ) + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_incremental_sync_sends_down_only_interesting_profile_updates_when_lazy_loading( + self, + ) -> None: + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + second_room = self.helper.create_room_as(self.user, tok=self.tok) + self.helper.join( + room=second_room, + user=third_user, + tok=third_tok, + ) + + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + ) + ) + self.helper.send_messages(room_id=second_room, num_events=10, tok=third_tok) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [ + "@third_user:test", + "@user:test", + ], + ) + @override_config({"experimental_features": {"msc4429_enabled": True}}) def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms( self, From e6c25e4930e94643d1494168ec4f044216eb9c5f Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 4 Jun 2026 22:29:05 +0300 Subject: [PATCH 045/106] Add `profile_updates_sequence` sequence to synapse_port_db.py --- synapse/_scripts/synapse_port_db.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 0b8a289d92d..f4f598d27c7 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -917,6 +917,10 @@ def alter_table(txn: LoggingTransaction) -> None: "quarantined_media_id_seq", [("quarantined_media_changes", "stream_id")], ) + await self._setup_sequence( + "profile_updates_sequence", + [("profile_updates", "stream_id")], + ) # Step 3. Get tables. self.progress.set_state("Fetching tables") From 28587d7907577926b7e44d6dcdb40c62513ec86b Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 5 Jun 2026 13:49:27 +0300 Subject: [PATCH 046/106] Fix the lazy loading test cases --- tests/handlers/test_sync.py | 49 +++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 88ed55f873b..87ede92e7aa 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1268,16 +1268,24 @@ def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( """ third_user = self.register_user("third_user", "password") third_tok = self.login("third_user", "password") - second_room = self.helper.create_room_as(self.user, tok=self.tok) self.helper.join( - room=second_room, + room=self.joined_room, user=third_user, tok=third_tok, ) requester = create_requester(self.user) - self.helper.send_messages(room_id=second_room, num_events=10, tok=third_tok) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + ) + ) + self.helper.send_messages(room_id=self.joined_room, num_events=1, tok=self.other_tok) + self.helper.send_messages(room_id=self.joined_room, num_events=10, tok=third_tok) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( requester, @@ -1300,22 +1308,12 @@ def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( request_key=generate_request_key(), ) ) - self.assertEqual( - initial_result.profile_updates["@user:test"]["m.status"], - '{"text": "Swimming in the Great Lakes!", "emoji": "\\ud83c\\udfca"}', - ) - self.assertEqual( - initial_result.profile_updates["@user:test"]["displayname"], "user" - ) - self.assertEqual( - initial_result.profile_updates["@third_user:test"]["displayname"], - "third_user", - ) + # Only third_user is returned, as lazy loading filters out the events from + # the other users self.assertCountEqual( initial_result.profile_updates.keys(), [ "@third_user:test", - "@user:test", ], ) @@ -1378,9 +1376,8 @@ def test_incremental_sync_sends_down_only_interesting_profile_updates_when_lazy_ ) -> None: third_user = self.register_user("third_user", "password") third_tok = self.login("third_user", "password") - second_room = self.helper.create_room_as(self.user, tok=self.tok) self.helper.join( - room=second_room, + room=self.joined_room, user=third_user, tok=third_tok, ) @@ -1411,7 +1408,20 @@ def test_incremental_sync_sends_down_only_interesting_profile_updates_when_lazy_ new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), ) ) - self.helper.send_messages(room_id=second_room, num_events=10, tok=third_tok) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value=json.dumps({"text": "On fire", "emoji": "🔥"}), + ) + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) incremental_result = self.get_success( self.sync_handler.wait_for_sync_for_user( requester, @@ -1435,11 +1445,12 @@ def test_incremental_sync_sends_down_only_interesting_profile_updates_when_lazy_ request_key=generate_request_key(), ) ) + # Only third_user is returned, as lazy loading filters out the events from + # other_user self.assertCountEqual( incremental_result.profile_updates.keys(), [ "@third_user:test", - "@user:test", ], ) From d02150fd84e0258e96f3d64d4074796a459b11df Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 5 Jun 2026 15:19:47 +0300 Subject: [PATCH 047/106] Remove code that accidentally came here from https://github.com/element-hq/synapse/pull/19542/changes#r2913827732 and was then removed in https://github.com/element-hq/synapse/pull/19398 --- synapse/handlers/profile.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index aede2256853..0b185f8fef7 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -465,27 +465,6 @@ async def delete_profile_upon_deactivation( profile_updates: list[tuple[str, JsonValue | None]] = [] current_profile: ProfileInfo | None = None - if not by_admin: - current_profile = await self.store.get_profileinfo(target_user) - if not self.hs.config.registration.enable_set_displayname: - if current_profile.display_name: - # SUSPICIOUS: It seems strange to block deactivation on this, - # though this is preserving previous behaviour. - raise SynapseError( - 400, - "Changing display name is disabled on this server", - Codes.FORBIDDEN, - ) - - if not self.hs.config.registration.enable_set_avatar_url: - if current_profile.avatar_url: - # SUSPICIOUS: It seems strange to block deactivation on this, - # though this is preserving previous behaviour. - raise SynapseError( - 400, - "Changing avatar is disabled on this server", - Codes.FORBIDDEN, - ) if self._msc4429_enabled: if current_profile is None: From 877074f38eaa8db4aabbfe1e09a8a49a8f4984ce Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 5 Jun 2026 15:30:23 +0300 Subject: [PATCH 048/106] Lint tests/handlers/test_sync.py --- tests/handlers/test_sync.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 87ede92e7aa..2ca66c7838e 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1284,8 +1284,12 @@ def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), ) ) - self.helper.send_messages(room_id=self.joined_room, num_events=1, tok=self.other_tok) - self.helper.send_messages(room_id=self.joined_room, num_events=10, tok=third_tok) + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( requester, From e90096ef571c55eca2da1147d7850a5422c4e1ed Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 5 Jun 2026 19:17:12 +0300 Subject: [PATCH 049/106] Use a "per user profile updates" tracking table This makes writes heavier when profile updates happen, but reduces the effort to produce an incremental sync response by not needing to look whether users share rooms. --- synapse/handlers/profile.py | 13 +++++ synapse/handlers/sync.py | 19 +++++-- synapse/storage/databases/main/profile.py | 68 +++++++++++++++++++++++ tests/handlers/test_profile.py | 47 ++++++++++++++++ 4 files changed, 141 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 0b185f8fef7..364a04ef7bd 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -129,6 +129,19 @@ async def _record_profile_updates( if not room_ids: return + users_who_share_rooms = ( + await self.store.get_local_users_who_share_room_with_user( + user_id.to_string() + ) + ) + # Remove ourselves from the user ID list + users_who_share_rooms.remove(user_id.to_string()) + if users_who_share_rooms: + await self.store.track_profile_updates_per_user( + stream_id=stream_id, + user_ids=users_who_share_rooms, + ) + self._notifier.on_new_event( StreamKeyType.PROFILE_UPDATES, stream_id, rooms=room_ids ) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 36319e4e61f..f5e3a47d54f 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2252,8 +2252,18 @@ async def _generate_sync_entry_for_profile_updates( to_id=now_token.profile_updates_key, field_names=profile_fields, ) + interesting_updates = await self.store.get_profile_updates_per_user_for_user( + from_id=since_token.profile_updates_key, + to_id=now_token.profile_updates_key, + user_id=user_id, + ) + # Only include updates we've got for us specifically + updates = [ + update for update in updates if update.stream_id in interesting_updates + ] + if include_users: - # Filter down to selected included users + # Further filter down to selected included users updates = [update for update in updates if update.user_id in include_users] if not updates: @@ -2264,10 +2274,7 @@ async def _generate_sync_entry_for_profile_updates( for update in updates if update.action == ProfileUpdateAction.UPDATE.value } - shared_updated_user_ids = await self.store.do_users_share_a_room( - user_id, updated_user_ids - ) - shared_updated_user_ids.add(user_id) + updated_user_ids.add(user_id) left_room_user_ids = { update.user_id for update in updates @@ -2282,7 +2289,7 @@ async def _generate_sync_entry_for_profile_updates( user_fields: dict[str, set[str]] = {} for update in updates: - if not update.field_name or update.user_id not in shared_updated_user_ids: + if not update.field_name or update.user_id not in updated_user_ids: continue user_fields.setdefault(update.user_id, set()).add(update.field_name) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 9d726f29e47..c7f8aea48f1 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -540,6 +540,74 @@ def _add_profile_updates_txn(txn: LoggingTransaction) -> int: "add_profile_updates", _add_profile_updates_txn ) + async def track_profile_updates_per_user( + self, + stream_id: int, + user_ids: set[str], + ) -> None: + """ + Create tracking rows for profile updater per target user interested in profile + updates for the user triggering one. + + Args: + stream_id: Stream ID referencing a `profile_updates` stream ID. + user_ids: A set of the full user IDs of the target users interested in + this change. + """ + + def _track_profile_updates_per_user_txn(txn: LoggingTransaction) -> None: + inserted_ts = self.clock.time_msec() + values = [[stream_id, user_id, inserted_ts] for user_id in user_ids] + self.db_pool.simple_insert_many_txn( + txn, + table="profile_updates_per_user", + keys=[ + "stream_id", + "user_id", + "inserted_ts", + ], + values=values, + ) + + return await self.db_pool.runInteraction( + "track_profile_updates_per_user", + _track_profile_updates_per_user_txn, + ) + + async def get_profile_updates_per_user_for_user( + self, *, from_id: int, to_id: int, user_id: str + ) -> list[int]: + """ + Get profile updates per user stream ID's for a particular user. + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + user_id: The full user ID to filter on + + Returns: + List of stream ID's. + """ + + def _get_profile_updates_per_user_for_user_txn( + txn: LoggingTransaction, + ) -> list[int]: + sql = """ + SELECT + stream_id + FROM profile_updates_per_user + WHERE + ? < stream_id AND stream_id <= ? AND user_id = ? + """ + txn.execute(sql, (from_id, to_id, user_id)) + rows = cast(list[tuple[int]], txn.fetchall()) + return [row[0] for row in rows] + + return await self.db_pool.runInteraction( + "get_profile_updates_per_user_for_user", + _get_profile_updates_per_user_for_user_txn, + ) + async def create_profile(self, user_id: UserID) -> None: """ Create a blank profile for a user. diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 1ff05120c4a..5267a6ab3b3 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -316,6 +316,53 @@ def test_update_profile_updates_stream_on_set_field( (3, "@1234abcd:test", ProfileUpdateAction.UPDATE.value, field_name), ) + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( + self, + ) -> None: + self.register_user("roger", "password") + roger_token = self.login("roger", "password") + self.register_user("millie", "password") + millie_token = self.login("millie", "password") + room_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + self.helper.join(room_id, "@roger:test", tok=roger_token) + self.helper.join(room_id, "@millie:test", tok=millie_token) + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="m.status", + new_value='{"text": "Holiday"}', + ) + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_per_user_for_user( + from_id=1, + to_id=2, + user_id="@roger:test", + ) + ) + self.assertEqual(per_user_updates, [2]) + per_user_updates = self.get_success( + self.store.get_profile_updates_per_user_for_user( + from_id=1, + to_id=2, + user_id="@millie:test", + ) + ) + self.assertEqual(per_user_updates, [2]) + per_user_updates = self.get_success( + self.store.get_profile_updates_per_user_for_user( + from_id=1, + to_id=2, + user_id=self.frank.to_string(), + ) + ) + self.assertEqual(per_user_updates, []) + @parameterized.expand( [ ["displayname", "Frank"], From 21d08a060c73789c5cebbade82d5b08c24e5196d Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 5 Jun 2026 19:37:05 +0300 Subject: [PATCH 050/106] Optimize calculating left room id's for incremental sync --- synapse/handlers/sync.py | 114 ++++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f5e3a47d54f..4dd0f34750d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2257,78 +2257,84 @@ async def _generate_sync_entry_for_profile_updates( to_id=now_token.profile_updates_key, user_id=user_id, ) - # Only include updates we've got for us specifically - updates = [ - update for update in updates if update.stream_id in interesting_updates - ] - if include_users: # Further filter down to selected included users updates = [update for update in updates if update.user_id in include_users] - if not updates: - return - - updated_user_ids = { + left_room_user_ids = { update.user_id for update in updates - if update.action == ProfileUpdateAction.UPDATE.value + if update.action == ProfileUpdateAction.LEFT_ROOM.value } - updated_user_ids.add(user_id) - left_room_user_ids = { + # Only include updates we've got for us specifically for field updates + updates = [ + update for update in updates if update.stream_id in interesting_updates + ] + updated_user_ids = { update.user_id for update in updates - if update.action == ProfileUpdateAction.LEFT_ROOM.value + if update.action == ProfileUpdateAction.UPDATE.value } - shared_left_user_ids = await self.store.do_users_share_a_room( - user_id, left_room_user_ids - ) - no_longer_sharing_rooms_user_ids = set(left_room_user_ids) - set( - shared_left_user_ids - ) + no_longer_sharing_rooms_user_ids: set[str] = set() + if left_room_user_ids: + shared_left_user_ids = await self.store.do_users_share_a_room( + user_id, left_room_user_ids + ) + no_longer_sharing_rooms_user_ids = set(left_room_user_ids) - set( + shared_left_user_ids + ) - user_fields: dict[str, set[str]] = {} - for update in updates: - if not update.field_name or update.user_id not in updated_user_ids: - continue - user_fields.setdefault(update.user_id, set()).add(update.field_name) - - # Note: there's a small race condition here where a profile update may - # occur between fetching `now_token` above and reaching this step. In - # that case, the profile information will be newer than `now_token`. - # This is fine, as users will generally always want the latest profile - # information. However, it does mean that on the next sync, the same - # profile update will come down a second time. - # - # Hopefully clients can just filter these out. - profile_data_by_user = await self.store.get_profile_data_for_users( - user_fields.keys() - ) + if not updated_user_ids and not left_room_user_ids: + return # Serialise the profile updates into the sync response format. # user ID -> {profile field -> value | null if unset } profile_updates: dict[str, dict[str, JsonValue | None]] = {} - for other_user_id, fields in user_fields.items(): - profile_data = profile_data_by_user.get(other_user_id) - if profile_data is None: - # No profile data for this user, just return a blank dictionary - # in incremental sync, telling the clients to remove all profile - # information for this user. - profile_updates[other_user_id] = {} - continue - per_user_updates: dict[str, JsonValue] = {} - for field_name in fields: - per_user_updates[field_name] = cast( - JsonValue, profile_data.get(field_name) - ) + # Process field updates + if updated_user_ids: + updated_user_ids.add(user_id) + user_fields: dict[str, set[str]] = {} + for update in updates: + if not update.field_name or update.user_id not in updated_user_ids: + continue + user_fields.setdefault(update.user_id, set()).add(update.field_name) + + # Note: there's a small race condition here where a profile update may + # occur between fetching `now_token` above and reaching this step. In + # that case, the profile information will be newer than `now_token`. + # This is fine, as users will generally always want the latest profile + # information. However, it does mean that on the next sync, the same + # profile update will come down a second time. + # + # Hopefully clients can just filter these out. + profile_data_by_user = await self.store.get_profile_data_for_users( + user_fields.keys() + ) - if per_user_updates: - profile_updates[other_user_id] = per_user_updates + for other_user_id, fields in user_fields.items(): + profile_data = profile_data_by_user.get(other_user_id) + if profile_data is None: + # No profile data for this user, just return a blank dictionary + # in incremental sync, telling the clients to remove all profile + # information for this user. + profile_updates[other_user_id] = {} + continue - for other_user_id in no_longer_sharing_rooms_user_ids: - # Return an empty dictionary to the client - profile_updates[other_user_id] = {} + per_user_updates: dict[str, JsonValue] = {} + for field_name in fields: + per_user_updates[field_name] = cast( + JsonValue, profile_data.get(field_name) + ) + + if per_user_updates: + profile_updates[other_user_id] = per_user_updates + + # Process left rooms + if no_longer_sharing_rooms_user_ids: + for other_user_id in no_longer_sharing_rooms_user_ids: + # Return an empty dictionary to the client + profile_updates[other_user_id] = {} if profile_updates: sync_result_builder.profile_updates = profile_updates From 029c4d81e4c618b9ccdb4e6262ec7b46ec275247 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Sun, 21 Jun 2026 22:39:27 +0100 Subject: [PATCH 051/106] Create `profile_updates_per_user` This table tracks which users should receive profile updates. It is intended as a write-heavy, cheap-read mechanism. At profile update time, the homeserver determines all the users that should receive the update. At the point of /sync, we then quickly read this table and check for any relevant updates for the syncing user. The timestamp field will be used to cull the table over time (so it doesn't grow indefinitely). --- .../schema/main/delta/95/01_profile_updates.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/storage/schema/main/delta/95/01_profile_updates.sql b/synapse/storage/schema/main/delta/95/01_profile_updates.sql index 61112476c44..fc31c48262f 100644 --- a/synapse/storage/schema/main/delta/95/01_profile_updates.sql +++ b/synapse/storage/schema/main/delta/95/01_profile_updates.sql @@ -33,3 +33,17 @@ CREATE TABLE profile_updates ( CREATE INDEX profile_updates_by_user ON profile_updates (user_id, stream_id); CREATE INDEX profile_updates_by_field ON profile_updates (field_name, stream_id); + +-- Track which local users should receive each profile update. +CREATE TABLE profile_updates_per_user ( + stream_id BIGINT NOT NULL, + + -- The full user ID of the local user that should receive the profile update. + user_id TEXT NOT NULL, + + -- Unix timestamp. Used to determine when to cull rows (to prevent the table + -- from growing indefinitely). + inserted_ts BIGINT NOT NULL +); + +CREATE INDEX profile_updates_per_user_by_user_stream ON profile_updates_per_user (user_id, stream_id); From 676ef4f819053498a8e6611bd77a4ea1e9fee3af Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Sun, 21 Jun 2026 22:40:52 +0100 Subject: [PATCH 052/106] Read from `profile_updates_per_user` when syncing No longer do we do the expensive lookup for which users share a room during /sync. --- synapse/api/filtering.py | 4 +- synapse/handlers/sync.py | 62 ++++++-------- synapse/storage/databases/main/profile.py | 100 +++++++++++++++++++++- 3 files changed, 129 insertions(+), 37 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 62f55416b5c..ee20e3d6c87 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -227,7 +227,7 @@ def __init__(self, hs: "HomeServer", filter_json: JsonMapping): self.event_fields = filter_json.get("event_fields", []) self.event_format = filter_json.get("event_format", "client") - self.profile_fields: list[str] = [] + self.profile_fields: set[str] = set() if hs.config.experimental.msc4429_enabled: profile_fields_filter = filter_json.get("org.matrix.msc4429.profile_fields") @@ -235,7 +235,7 @@ def __init__(self, hs: "HomeServer", filter_json: JsonMapping): ids = profile_fields_filter.get("ids", []) if ids is None: ids = [] - self.profile_fields = list(ids) + self.profile_fields = set(ids) def __repr__(self) -> str: return "" % (json.dumps(self._filter_json),) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 4dd0f34750d..19c63945c8f 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2142,7 +2142,7 @@ async def _generate_initial_sync_entry_for_profile_updates( *, user_id: str, sync_result_builder: "SyncResultBuilder", - profile_fields: list[str], + profile_fields: set[str], include_users: set[str] | None, ) -> None: """ @@ -2159,21 +2159,35 @@ async def _generate_initial_sync_entry_for_profile_updates( for when we have calculated a list of users in our lazy loading sync and want to only return those. """ - # Currently, limited to only local profiles, so filter remote servers out - user_ids = await self.store.get_local_users_who_share_room_with_user(user_id) + updates = await self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=sync_result_builder.now_token.profile_updates_key, + user_id=user_id, + field_names=profile_fields, + include_users=include_users, + ) - if include_users: - # Filter down to selected included users - user_ids = {user_id for user_id in user_ids if user_id in include_users} + user_fields: dict[str, set[str]] = {} + for update in updates: + if ( + update.action != ProfileUpdateAction.UPDATE.value + # TODO: When would field_name be None? + or update.field_name is None + ): + continue + + user_fields.setdefault(update.user_id, set()).add(update.field_name) - if not user_ids: + if not user_fields: return - profile_data_by_user = await self.store.get_profile_data_for_users(user_ids) + profile_data_by_user = await self.store.get_profile_data_for_users( + user_fields.keys() + ) # Serialise the profile updates into the sync response format. profile_updates: dict[str, dict[str, JsonValue | None]] = {} - for other_user_id in user_ids: + for other_user_id, fields in user_fields.items(): profile_data = profile_data_by_user.get(other_user_id) if profile_data is None: # Don't generate anything for users with no profile data @@ -2181,7 +2195,7 @@ async def _generate_initial_sync_entry_for_profile_updates( continue per_user_updates: dict[str, JsonValue] = {} - for field_name in profile_fields: + for field_name in fields: if profile_data.get(field_name): per_user_updates[field_name] = cast( JsonValue, profile_data[field_name] @@ -2247,42 +2261,23 @@ async def _generate_sync_entry_for_profile_updates( if since_token.profile_updates_key == now_token.profile_updates_key: return - updates = await self.store.get_profile_updates_for_fields( - from_id=since_token.profile_updates_key, - to_id=now_token.profile_updates_key, - field_names=profile_fields, - ) - interesting_updates = await self.store.get_profile_updates_per_user_for_user( + updates = await self.store.get_profile_updates_for_user_and_fields( from_id=since_token.profile_updates_key, to_id=now_token.profile_updates_key, user_id=user_id, + field_names=profile_fields, ) - if include_users: - # Further filter down to selected included users - updates = [update for update in updates if update.user_id in include_users] left_room_user_ids = { update.user_id for update in updates if update.action == ProfileUpdateAction.LEFT_ROOM.value } - # Only include updates we've got for us specifically for field updates - updates = [ - update for update in updates if update.stream_id in interesting_updates - ] updated_user_ids = { update.user_id for update in updates if update.action == ProfileUpdateAction.UPDATE.value } - no_longer_sharing_rooms_user_ids: set[str] = set() - if left_room_user_ids: - shared_left_user_ids = await self.store.do_users_share_a_room( - user_id, left_room_user_ids - ) - no_longer_sharing_rooms_user_ids = set(left_room_user_ids) - set( - shared_left_user_ids - ) if not updated_user_ids and not left_room_user_ids: return @@ -2293,7 +2288,6 @@ async def _generate_sync_entry_for_profile_updates( # Process field updates if updated_user_ids: - updated_user_ids.add(user_id) user_fields: dict[str, set[str]] = {} for update in updates: if not update.field_name or update.user_id not in updated_user_ids: @@ -2331,8 +2325,8 @@ async def _generate_sync_entry_for_profile_updates( profile_updates[other_user_id] = per_user_updates # Process left rooms - if no_longer_sharing_rooms_user_ids: - for other_user_id in no_longer_sharing_rooms_user_ids: + if left_room_user_ids: + for other_user_id in left_room_user_ids: # Return an empty dictionary to the client profile_updates[other_user_id] = {} diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index c7f8aea48f1..3a8a54da5e8 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -411,7 +411,7 @@ def _get_profile_updates_for_fields_txn( " OR action != ?) " " ORDER BY stream_id ASC" ) - txn.execute(sql, (from_id, to_id, ProfileUpdateAction.UPDATE.value, *args)) + txn.execute(sql, (from_id, to_id, *args, ProfileUpdateAction.UPDATE.value)) rows = cast(list[tuple[int, str, str, str | None]], txn.fetchall()) updates: list[ProfileUpdate] = [] @@ -431,6 +431,104 @@ def _get_profile_updates_for_fields_txn( "get_profile_updates_for_fields", _get_profile_updates_for_fields_txn ) + async def get_profile_updates_for_user_and_fields( + self, + *, + from_id: int, + to_id: int, + user_id: str, + field_names: set[str], + include_users: set[str] | None = None, + ) -> list[ProfileUpdate]: + """Get profile update markers for a user in a stream range. + + The returned profile update rows are restricted to those with a + corresponding `profile_updates_per_user` row for the syncing user. + + Bounds: from_id < ... <= to_id + + Args: + from_id: The starting stream ID (exclusive). + to_id: The ending stream ID (inclusive). + user_id: The full user ID to filter on. + field_names: Set of field names to filter update actions against. + include_users: If given, only include updates for these user IDs. + + Returns: + A list of ProfileUpdates update rows. + """ + if from_id == to_id: + return [] + + if len(field_names) == 0: + return [] + + if include_users is not None and len(include_users) == 0: + # All updates have been filtered out by lazy-loading. + return [] + + def _get_profile_updates_for_user_and_fields_txn( + txn: LoggingTransaction, + ) -> list[ProfileUpdate]: + field_clause, field_args = make_in_list_sql_clause( + txn.database_engine, "pu.field_name", field_names + ) + user_clause = "" + user_args: list[str] = [] + if include_users is not None: + # Filter out rows that aren't in `include_users`, if defined. + # This is only relevant when lazy-loading. + user_clause, user_args = make_in_list_sql_clause( + txn.database_engine, "pu.user_id", include_users + ) + user_clause = f"AND {user_clause}" + + # Retrieve profile updates where there's a corresponding row in + # `profile_updates_per_user` within the given `stream_id` bounds + # and the `user_id` and `field_names` match. + sql = f""" + SELECT pu.stream_id, pu.user_id, pu.action, pu.field_name + FROM profile_updates AS pu + INNER JOIN profile_updates_per_user AS puf + ON pu.stream_id = puf.stream_id + WHERE ? < pu.stream_id AND pu.stream_id <= ? + AND puf.user_id = ? + {user_clause} + AND ({field_clause} OR pu.action != ?) + ORDER BY pu.stream_id ASC + """ + + txn.execute( + sql, + ( + from_id, + to_id, + user_id, + *user_args, + *field_args, + ProfileUpdateAction.UPDATE.value, + ), + ) + rows = cast(list[tuple[int, str, str, str | None]], txn.fetchall()) + + updates: list[ProfileUpdate] = [] + for stream_id, updated_user_id, action, field_name in rows: + updates.append( + ProfileUpdate( + stream_id=stream_id, + user_id=updated_user_id, + action=action, + field_name=field_name, + ) + ) + + return updates + + return await self.db_pool.runInteraction( + "get_profile_updates_for_user_and_fields", + _get_profile_updates_for_user_and_fields_txn, + ) + async def get_profile_data_for_users( self, user_ids: Collection[str] ) -> dict[str, dict[str, str | JsonDict | None]]: From 4a3fc09856871e08b763e3af65789cc76ec9f1c9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Sun, 21 Jun 2026 22:53:29 +0100 Subject: [PATCH 053/106] fix the tests --- tests/handlers/test_sync.py | 39 ++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 2ca66c7838e..0ed08465d5d 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1217,7 +1217,16 @@ def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: ) @override_config({"experimental_features": {"msc4429_enabled": True}}) - def test_initial_sync_responds_with_all_known_profiles(self) -> None: + def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + ) + ) + requester = create_requester(self.user) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( @@ -1237,21 +1246,13 @@ def test_initial_sync_responds_with_all_known_profiles(self) -> None: ) ) self.assertEqual( - initial_result.profile_updates["@user:test"]["m.status"], - '{"text": "Swimming in the Great Lakes!", "emoji": "\\ud83c\\udfca"}', - ) - self.assertEqual( - initial_result.profile_updates["@user:test"]["displayname"], "user" - ) - self.assertEqual( - initial_result.profile_updates["@other_user:test"]["displayname"], - "other_user", + initial_result.profile_updates["@other_user:test"]["m.status"], + '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', ) self.assertCountEqual( initial_result.profile_updates.keys(), [ "@other_user:test", - "@user:test", ], ) @@ -1284,6 +1285,15 @@ def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), ) ) + # Check that lazy-loading filters out profile updates as well on initial sync. + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value=json.dumps({"text": "On fire", "emoji": "🔥"}), + ) + ) self.helper.send_messages( room_id=self.joined_room, num_events=1, tok=self.other_tok ) @@ -1375,7 +1385,7 @@ def test_incremental_sync_sends_down_profile_updates( ) @override_config({"experimental_features": {"msc4429_enabled": True}}) - def test_incremental_sync_sends_down_only_interesting_profile_updates_when_lazy_loading( + def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( self, ) -> None: third_user = self.register_user("third_user", "password") @@ -1449,11 +1459,12 @@ def test_incremental_sync_sends_down_only_interesting_profile_updates_when_lazy_ request_key=generate_request_key(), ) ) - # Only third_user is returned, as lazy loading filters out the events from - # other_user + # Lazy loading only filters initial sync profile updates. Incremental syncs + # should include all tracked profile updates for the syncing user. self.assertCountEqual( incremental_result.profile_updates.keys(), [ + "@other_user:test", "@third_user:test", ], ) From ab9c917ca24a3a16ddc3710fb6fed016b8fa0639 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Sun, 21 Jun 2026 22:55:20 +0100 Subject: [PATCH 054/106] Remove `test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms` We might just have clients do this instead (based on leave membership events). --- tests/handlers/test_sync.py | 48 ------------------------------------- 1 file changed, 48 deletions(-) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 0ed08465d5d..fdc9cb47263 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1469,54 +1469,6 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( ], ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) - def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms( - self, - ) -> None: - requester = create_requester(self.user) - initial_result = self.get_success( - self.sync_handler.wait_for_sync_for_user( - requester, - sync_config=generate_sync_config( - user_id=self.user, - filter_collection=FilterCollection( - hs=self.hs, - filter_json={ - "org.matrix.msc4429.profile_fields": { - "ids": ["m.status", "displayname", "avatar_url"] - } - }, - ), - ), - request_key=generate_request_key(), - ) - ) - self.helper.leave( - room=self.joined_room, user=self.other_user, tok=self.other_tok - ) - incremental_result = self.get_success( - self.sync_handler.wait_for_sync_for_user( - requester, - since_token=initial_result.next_batch, - sync_config=generate_sync_config( - user_id=self.user, - filter_collection=FilterCollection( - hs=self.hs, - filter_json={ - "org.matrix.msc4429.profile_fields": { - "ids": ["m.status", "displayname", "avatar_url"] - } - }, - ), - ), - request_key=generate_request_key(), - ) - ) - self.assertEqual( - incremental_result.profile_updates["@other_user:test"], - {}, - ) - class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): """Tests Sync Handler state behavior when using `use_state_after.""" From 27417cf0a81e75edbce1f50374ce6a71e515fece Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 22 Jun 2026 23:10:20 +0300 Subject: [PATCH 055/106] Remove mypy ignore --- synapse/replication/tcp/client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index c2155f05174..dc68f554460 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -273,10 +273,8 @@ async def on_rdata( user_ids_to_room_ids = await self.store.get_rooms_for_users( updated_user_ids ) - # Typing: `user_ids_to_room_ids.values` is a frozenset, but one - # can still iterate over it, hence the ignore. - for batched_room_ids in user_ids_to_room_ids.values(): # type: ignore[assignment] - room_ids.update(batched_room_ids) + for batched_user_ids_to_room_ids in user_ids_to_room_ids.values(): + room_ids.update(batched_user_ids_to_room_ids) if room_ids: self.notifier.on_new_event( From daa8d7a28dc05bb0105c688be7fb96e2c06c950f Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 23 Jun 2026 12:39:07 +0300 Subject: [PATCH 056/106] Ensure lazy loaded profile updates in sync contain full profiles for users in timeline --- synapse/handlers/sync.py | 17 +++++++++++++---- tests/handlers/test_sync.py | 23 +++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 19c63945c8f..f36a36b3d5d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2316,10 +2316,19 @@ async def _generate_sync_entry_for_profile_updates( continue per_user_updates: dict[str, JsonValue] = {} - for field_name in fields: - per_user_updates[field_name] = cast( - JsonValue, profile_data.get(field_name) - ) + if include_users and other_user_id in include_users: + # Include the full profile as this user has events in + # a lazy loaded sync response + for field_name in profile_data.keys(): + per_user_updates[field_name] = cast( + JsonValue, profile_data.get(field_name) + ) + else: + # Include only the diff + for field_name in fields: + per_user_updates[field_name] = cast( + JsonValue, profile_data.get(field_name) + ) if per_user_updates: profile_updates[other_user_id] = per_user_updates diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index fdc9cb47263..2c5fd215996 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1383,6 +1383,10 @@ def test_incremental_sync_sends_down_profile_updates( incremental_result.profile_updates["@other_user:test"]["m.status"], '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', ) + # We only send diffs in incremental sync for profile field updates + self.assertIsNone( + incremental_result.profile_updates["@other_user:test"].get("displayname"), + ) @override_config({"experimental_features": {"msc4429_enabled": True}}) def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( @@ -1468,6 +1472,25 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( "@third_user:test", ], ) + # This is a field upfate, so should be here + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["m.status"], + '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', + ) + # We don't have events for this user in this response, so their full profile + # is not included + self.assertIsNone( + incremental_result.profile_updates["@other_user:test"].get("displayname"), + ) + # This user has events in the timeline, thus their full profile is included + self.assertEqual( + incremental_result.profile_updates["@third_user:test"]["m.status"], + '{"text": "On fire", "emoji": "\\ud83d\\udd25"}', + ) + self.assertEqual( + incremental_result.profile_updates["@third_user:test"]["displayname"], + "third_user", + ) class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): From 5819dccd91221503466cc6fe64fe9076f011621b Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 23 Jun 2026 13:10:21 +0300 Subject: [PATCH 057/106] Use a cache to ensure we don't re-send the full profile fields of users in every incremental sync all the time when lazy loading --- synapse/handlers/sync.py | 59 +++++++++++++++++++++++++++++++++++-- tests/handlers/test_sync.py | 31 +++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f36a36b3d5d..bcf8a0dad95 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -107,10 +107,18 @@ # client for no more than 30 minutes. LAZY_LOADED_MEMBERS_CACHE_MAX_AGE = 30 * 60 * 1000 +# Store the cache that tracks which lazy-loaded profile fields have been sent to a given +# client for no more than 30 minutes. +LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE = 30 * 60 * 1000 + # Remember the last 100 members we sent to a client for the purposes of # avoiding redundantly sending the same lazy-loaded members to the client LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE = 100 +# Remember the last 100 profile field updates we sent to a client for the purposes of +# avoiding redundantly sending the same lazy-loaded full profiles to the client +LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_SIZE = 100 + SyncRequestKey = tuple[Any, ...] @@ -338,6 +346,17 @@ def __init__(self, hs: "HomeServer"): max_len=0, expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE, ) + # ExpiringCache((User, Device, Other User, Profile Field)) -> LruCache(user_id => field_value) + self.lazy_loaded_profile_fields_cache: ExpiringCache[ + tuple[str, str | None, str, str], LruCache[str, str] + ] = ExpiringCache( + cache_name="lazy_loaded_profile_fields_cache", + server_name=self.server_name, + hs=hs, + clock=self.clock, + max_len=0, + expiry_ms=LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE, + ) self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync @@ -1045,6 +1064,24 @@ def get_lazy_loaded_members_cache( logger.debug("found LruCache for %r", cache_key) return cache + def get_lazy_loaded_profile_fields_cache( + self, cache_key: tuple[str, str | None, str, str] + ) -> LruCache[str, str]: + cache: LruCache[str, str] | None = self.lazy_loaded_profile_fields_cache.get( + cache_key + ) + if cache is None: + logger.debug("creating LruCache for %r", cache_key) + cache = LruCache( + max_size=LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_SIZE, + clock=self.clock, + server_name=self.server_name, + ) + self.lazy_loaded_profile_fields_cache[cache_key] = cache + else: + logger.debug("found LruCache for %r", cache_key) + return cache + async def compute_state_delta( self, room_id: str, @@ -2318,13 +2355,29 @@ async def _generate_sync_entry_for_profile_updates( per_user_updates: dict[str, JsonValue] = {} if include_users and other_user_id in include_users: # Include the full profile as this user has events in - # a lazy loaded sync response + # a lazy loaded sync response, except for fields we've recently + # sent in a previous lazy loaded sync response for field_name in profile_data.keys(): - per_user_updates[field_name] = cast( - JsonValue, profile_data.get(field_name) + cache_key = ( + sync_config.user.to_string(), + sync_config.device_id, + other_user_id, + field_name, ) + cache = self.get_lazy_loaded_profile_fields_cache(cache_key) + # Only send the field if we haven't recently sent it + if not cache.get(field_name): + per_user_updates[field_name] = cast( + JsonValue, profile_data.get(field_name) + ) + # Update our cache + cache.set( + other_user_id, + cast(str, profile_data.get(field_name)), + ) else: # Include only the diff + # We don't use a cache here as changes are always sent for field_name in fields: per_user_updates[field_name] = cast( JsonValue, profile_data.get(field_name) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 2c5fd215996..a61eec9e0bd 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1492,6 +1492,37 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( "third_user", ) + # If we have more events from the third_user, and do another lazy sync, + # we don't expect the full profile to be sent again due to our cache + self.helper.send_messages(room_id=self.joined_room, num_events=1, tok=third_tok) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [], + ) + class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): """Tests Sync Handler state behavior when using `use_state_after.""" From 09e212537e97bf12e3395b84dc0524969050e6c5 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 23 Jun 2026 14:53:33 +0300 Subject: [PATCH 058/106] Fix incremental lazy sync to send down profiles for users without profile updates Remove the checking of any stream token changes and ensure we always collect profiles for users even if they have not done profile updates, if they have events in the timeline. Also fix the cache. --- synapse/handlers/sync.py | 55 ++++++++++++++++++------------- tests/handlers/test_sync.py | 64 +++++++++++++++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 24 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index bcf8a0dad95..0ebc6802ac0 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2295,9 +2295,6 @@ async def _generate_sync_entry_for_profile_updates( ) return - if since_token.profile_updates_key == now_token.profile_updates_key: - return - updates = await self.store.get_profile_updates_for_user_and_fields( from_id=since_token.profile_updates_key, to_id=now_token.profile_updates_key, @@ -2310,24 +2307,31 @@ async def _generate_sync_entry_for_profile_updates( for update in updates if update.action == ProfileUpdateAction.LEFT_ROOM.value } - updated_user_ids = { + users = set() + updated_users = { update.user_id for update in updates if update.action == ProfileUpdateAction.UPDATE.value } + # Add any users in the timeline, if we collected them due to lazy loading + if include_users: + users.update(include_users) + # Add users with updates + users.update(updated_users) - if not updated_user_ids and not left_room_user_ids: + if not users and not left_room_user_ids: return # Serialise the profile updates into the sync response format. # user ID -> {profile field -> value | null if unset } profile_updates: dict[str, dict[str, JsonValue | None]] = {} - # Process field updates - if updated_user_ids: + # Process field updates and users who have events in the sync response + if users: user_fields: dict[str, set[str]] = {} + # Set fields from updates for update in updates: - if not update.field_name or update.user_id not in updated_user_ids: + if not update.field_name or update.user_id not in users: continue user_fields.setdefault(update.user_id, set()).add(update.field_name) @@ -2339,11 +2343,9 @@ async def _generate_sync_entry_for_profile_updates( # profile update will come down a second time. # # Hopefully clients can just filter these out. - profile_data_by_user = await self.store.get_profile_data_for_users( - user_fields.keys() - ) + profile_data_by_user = await self.store.get_profile_data_for_users(users) - for other_user_id, fields in user_fields.items(): + for other_user_id in users: profile_data = profile_data_by_user.get(other_user_id) if profile_data is None: # No profile data for this user, just return a blank dictionary @@ -2366,19 +2368,28 @@ async def _generate_sync_entry_for_profile_updates( ) cache = self.get_lazy_loaded_profile_fields_cache(cache_key) # Only send the field if we haven't recently sent it - if not cache.get(field_name): - per_user_updates[field_name] = cast( - JsonValue, profile_data.get(field_name) - ) - # Update our cache - cache.set( - other_user_id, - cast(str, profile_data.get(field_name)), - ) + if cache.get(other_user_id) is None: + # If the field value is a None, don't send it down or + # set the cache unless we're sure it has become None due + # to a profile update, otherwise we'll just be sending the + # same field down in every single incremental lazy sync + # regardless of cache state + if ( + profile_data.get(field_name) is not None + or other_user_id in updated_users + ): + per_user_updates[field_name] = cast( + JsonValue, profile_data.get(field_name) + ) + # Update our cache + cache.set( + other_user_id, + cast(str, profile_data.get(field_name)), + ) else: # Include only the diff # We don't use a cache here as changes are always sent - for field_name in fields: + for field_name in user_fields.get(other_user_id, []): per_user_updates[field_name] = cast( JsonValue, profile_data.get(field_name) ) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index a61eec9e0bd..7e3d5c7aa58 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1492,9 +1492,69 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( "third_user", ) - # If we have more events from the third_user, and do another lazy sync, + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles( + self, + ) -> None: + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.helper.send_messages( + room_id=self.joined_room, + num_events=1, + tok=self.other_tok, + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + # Lazy loading incremental sync should include profiles from events + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [ + "@other_user:test", + ], + ) + + # If we have more events from the other_user, and do another lazy sync, # we don't expect the full profile to be sent again due to our cache - self.helper.send_messages(room_id=self.joined_room, num_events=1, tok=third_tok) + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) incremental_result = self.get_success( self.sync_handler.wait_for_sync_for_user( requester, From 339eee55cda138fdef4d0d9b2fe9d5d00089e37d Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 22 Jun 2026 14:41:06 +0100 Subject: [PATCH 059/106] WIP track when a user leaves a room in `profile_updates_per_user` --- synapse/handlers/profile.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 364a04ef7bd..296b7ed57d6 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -431,19 +431,49 @@ async def set_avatar_url( if propagate: await self._update_join_states(requester, target_user) - async def _user_left_room(self, user_id: UserID) -> None: - await self.store.add_profile_updates( + async def _user_left_room(self, user_id: UserID, room_id: str) -> None: + """ + A user left a room. We now: + + * Add a row to `profile_updates` stating the the user left a certain room. + * Check if this user no longer shares any rooms with certain users. + * Insert a row for each of those users into `profile_updates_per_user`. + * Now, when any of those users sync, the sync code will check + `profile_updates` and see that the user left a room. And thus a "clear + this user's profile" instruction will be sent down to the client. + + If that is the case, + """ + user_id_str = user_id.to_string() + stream_id = await self.store.add_profile_updates( user_id=user_id, action=ProfileUpdateAction.LEFT_ROOM.value, updated_fields=None, ) + users_in_left_room = set(await self.store.get_local_users_in_room(room_id)) + users_in_left_room.discard(user_id_str) + if not users_in_left_room: + return + + users_still_sharing_rooms = await self.store.do_users_share_a_room( + user_id_str, users_in_left_room + ) + + users_to_update = users_in_left_room - users_still_sharing_rooms + if users_to_update: + await self.store.track_profile_updates_per_user( + stream_id=stream_id, + user_ids=users_to_update, + ) + def user_left_room(self, user: UserID, room_id: str) -> None: if self.hs.is_mine_id(user.to_string()): self.hs.run_as_background_process( "profile._user_left_room", self._user_left_room, user_id=user, + room_id=room_id, ) async def delete_profile_upon_deactivation( From 2dbaec329b3570a93bae1b85f13de6bb2f70bbce Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 23 Jun 2026 15:37:35 +0300 Subject: [PATCH 060/106] Send down a null when user has left all shared rooms Restore test and switch from {} to null. --- synapse/handlers/sync.py | 12 ++++----- tests/handlers/test_sync.py | 53 ++++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 0ebc6802ac0..1f31b537868 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -252,7 +252,7 @@ class SyncResult: presence: list[UserPresenceState] account_data: list[JsonDict] # user ID -> {profile field -> value | null if unset } - profile_updates: dict[str, dict[str, JsonValue | None]] + profile_updates: dict[str, dict[str, JsonValue | None] | None] joined: list[JoinedSyncResult] invited: list[InvitedSyncResult] knocked: list[KnockedSyncResult] @@ -2223,7 +2223,7 @@ async def _generate_initial_sync_entry_for_profile_updates( ) # Serialise the profile updates into the sync response format. - profile_updates: dict[str, dict[str, JsonValue | None]] = {} + profile_updates: dict[str, dict[str, JsonValue | None] | None] = {} for other_user_id, fields in user_fields.items(): profile_data = profile_data_by_user.get(other_user_id) if profile_data is None: @@ -2324,7 +2324,7 @@ async def _generate_sync_entry_for_profile_updates( # Serialise the profile updates into the sync response format. # user ID -> {profile field -> value | null if unset } - profile_updates: dict[str, dict[str, JsonValue | None]] = {} + profile_updates: dict[str, dict[str, JsonValue | None] | None] = {} # Process field updates and users who have events in the sync response if users: @@ -2351,7 +2351,7 @@ async def _generate_sync_entry_for_profile_updates( # No profile data for this user, just return a blank dictionary # in incremental sync, telling the clients to remove all profile # information for this user. - profile_updates[other_user_id] = {} + profile_updates[other_user_id] = None continue per_user_updates: dict[str, JsonValue] = {} @@ -2401,7 +2401,7 @@ async def _generate_sync_entry_for_profile_updates( if left_room_user_ids: for other_user_id in left_room_user_ids: # Return an empty dictionary to the client - profile_updates[other_user_id] = {} + profile_updates[other_user_id] = None if profile_updates: sync_result_builder.profile_updates = profile_updates @@ -3441,7 +3441,7 @@ class SyncResultBuilder: presence: list[UserPresenceState] = attr.Factory(list) account_data: list[JsonDict] = attr.Factory(list) - profile_updates: dict[str, dict[str, JsonValue | None]] = attr.Factory(dict) + profile_updates: dict[str, dict[str, JsonValue | None] | None] = attr.Factory(dict) joined: list[JoinedSyncResult] = attr.Factory(list) invited: list[InvitedSyncResult] = attr.Factory(list) knocked: list[KnockedSyncResult] = attr.Factory(list) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 7e3d5c7aa58..fbec22ab9b8 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1245,6 +1245,7 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: request_key=generate_request_key(), ) ) + assert initial_result.profile_updates["@other_user:test"] is not None self.assertEqual( initial_result.profile_updates["@other_user:test"]["m.status"], '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', @@ -1379,6 +1380,7 @@ def test_incremental_sync_sends_down_profile_updates( request_key=generate_request_key(), ) ) + assert incremental_result.profile_updates["@other_user:test"] is not None self.assertEqual( incremental_result.profile_updates["@other_user:test"]["m.status"], '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', @@ -1472,7 +1474,8 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( "@third_user:test", ], ) - # This is a field upfate, so should be here + assert incremental_result.profile_updates["@other_user:test"] is not None + # This is a field update, so should be here self.assertEqual( incremental_result.profile_updates["@other_user:test"]["m.status"], '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', @@ -1482,6 +1485,7 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( self.assertIsNone( incremental_result.profile_updates["@other_user:test"].get("displayname"), ) + assert incremental_result.profile_updates["@third_user:test"] is not None # This user has events in the timeline, thus their full profile is included self.assertEqual( incremental_result.profile_updates["@third_user:test"]["m.status"], @@ -1583,6 +1587,53 @@ def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles( [], ) + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms( + self, + ) -> None: + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.helper.leave( + room=self.joined_room, user=self.other_user, tok=self.other_tok + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertIsNone( + incremental_result.profile_updates["@other_user:test"], + ) + class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): """Tests Sync Handler state behavior when using `use_state_after.""" From 8504386b1f99e91dd75263fdcb8c815ec553cbd9 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 23 Jun 2026 16:44:39 +0300 Subject: [PATCH 061/106] Send down full profile of user if user has joined during incremental sync Also refactor the "user left" to not listen to the "user_left_room" dispatch hook since we only want to do this once, thus call the profile handler immediately in the instance responsible for receiving the event. --- synapse/api/constants.py | 1 + synapse/handlers/profile.py | 44 +++++++++++++----- synapse/handlers/room_member.py | 16 +++++++ synapse/handlers/sync.py | 16 ++++++- tests/handlers/test_sync.py | 80 +++++++++++++++++++++++++++++++++ 5 files changed, 143 insertions(+), 14 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 44c84252fc7..80505da6d1e 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -411,6 +411,7 @@ class ProfileFields: class ProfileUpdateAction(enum.Enum): + JOINED_ROOM = "joined_room" LEFT_ROOM = "left_room" UPDATE = "update" diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 296b7ed57d6..ba5a0ceab4d 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -102,8 +102,6 @@ def __init__(self, hs: "HomeServer"): ) self._worker_locks = hs.get_worker_locks_handler() - hs.get_distributor().observe("user_left_room", self.user_left_room) - async def _record_profile_updates( self, user_id: UserID, updated_fields: list[str] ) -> None: @@ -431,18 +429,18 @@ async def set_avatar_url( if propagate: await self._update_join_states(requester, target_user) - async def _user_left_room(self, user_id: UserID, room_id: str) -> None: + async def user_left_room(self, user_id: UserID, room_id: str) -> None: """ A user left a room. We now: - * Add a row to `profile_updates` stating the the user left a certain room. + * Add a row to `profile_updates` stating that the user left a room. * Check if this user no longer shares any rooms with certain users. * Insert a row for each of those users into `profile_updates_per_user`. * Now, when any of those users sync, the sync code will check `profile_updates` and see that the user left a room. And thus a "clear this user's profile" instruction will be sent down to the client. - If that is the case, + If that is the case, """ user_id_str = user_id.to_string() stream_id = await self.store.add_profile_updates( @@ -467,13 +465,35 @@ async def _user_left_room(self, user_id: UserID, room_id: str) -> None: user_ids=users_to_update, ) - def user_left_room(self, user: UserID, room_id: str) -> None: - if self.hs.is_mine_id(user.to_string()): - self.hs.run_as_background_process( - "profile._user_left_room", - self._user_left_room, - user_id=user, - room_id=room_id, + async def user_joined_room(self, user_id: UserID, room_id: str) -> None: + """ + A user joined a room. We now: + + * Add a row to `profile_updates` stating that the user joined a room. + * Get list of users in that room. + * Insert a row for each of those users into `profile_updates_per_user`. + * Now, when any of those users sync, the sync code will check + `profile_updates` and see that the user joined a room. Thus, we can + include the users full profile in the case that we need to do so. + + If that is the case, + """ + user_id_str = user_id.to_string() + stream_id = await self.store.add_profile_updates( + user_id=user_id, + action=ProfileUpdateAction.JOINED_ROOM.value, + updated_fields=None, + ) + + users_in_room = set(await self.store.get_local_users_in_room(room_id)) + users_in_room.discard(user_id_str) + if not users_in_room: + return + + if users_in_room: + await self.store.track_profile_updates_per_user( + stream_id=stream_id, + user_ids=users_in_room, ) async def delete_profile_upon_deactivation( diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 5152d0b522e..2ba29fbe485 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -526,6 +526,14 @@ async def _local_membership_update( ) if prev_member_event.membership == Membership.JOIN: await self._user_left_room(target, room_id) + # Notify the profile handler. We only want to do this once + # in a multi-worker setup, so we can't listen on the dispatched + # event above. + await self.profile_handler.user_left_room(target, room_id) + elif event.membership == Membership.JOIN: + # Notify the profile handler. We only want to do this once + # in a multi-worker setup, so we can't dispatch a hook to all workers. + await self.profile_handler.user_joined_room(target, room_id) break except PartialStateConflictError as e: @@ -1541,6 +1549,14 @@ async def send_membership_event( prev_member_event = await self.store.get_event(prev_member_event_id) if prev_member_event.membership == Membership.JOIN: await self._user_left_room(target_user, room_id) + # Notify the profile handler. We only want to do this once + # in a multi-worker setup, so we can't listen on the dispatched + # event above. + await self.profile_handler.user_left_room(target_user, room_id) + elif event.membership == Membership.JOIN: + # Notify the profile handler. We only want to do this once + # in a multi-worker setup, so we can't dispatch a hook to all workers. + await self.profile_handler.user_joined_room(target_user, room_id) async def _can_guest_join(self, partial_current_state_ids: StateMap[str]) -> bool: """ diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 1f31b537868..eb9b186775b 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2307,6 +2307,11 @@ async def _generate_sync_entry_for_profile_updates( for update in updates if update.action == ProfileUpdateAction.LEFT_ROOM.value } + joined_room_user_ids = { + update.user_id + for update in updates + if update.action == ProfileUpdateAction.JOINED_ROOM.value + } users = set() updated_users = { update.user_id @@ -2318,6 +2323,8 @@ async def _generate_sync_entry_for_profile_updates( users.update(include_users) # Add users with updates users.update(updated_users) + # Add any newly joined users + users.update(joined_room_user_ids) if not users and not left_room_user_ids: return @@ -2387,9 +2394,14 @@ async def _generate_sync_entry_for_profile_updates( cast(str, profile_data.get(field_name)), ) else: - # Include only the diff + # Include only the diff, unless the user recently joined # We don't use a cache here as changes are always sent - for field_name in user_fields.get(other_user_id, []): + fields = ( + list(profile_data.keys()) + if other_user_id in joined_room_user_ids + else user_fields.get(other_user_id, []) + ) + for field_name in fields: per_user_updates[field_name] = cast( JsonValue, profile_data.get(field_name) ) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index fbec22ab9b8..4ee74a47210 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1634,6 +1634,86 @@ def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_room incremental_result.profile_updates["@other_user:test"], ) + @override_config({"experimental_features": {"msc4429_enabled": True}}) + def test_incremental_sync_sends_down_full_profile_for_users_who_have_joined( + self, + ) -> None: + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + assert incremental_result.profile_updates["@third_user:test"] is not None + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [third_user], + ) + self.assertEqual( + incremental_result.profile_updates["@third_user:test"]["displayname"], + "third_user", + ) + self.assertIsNone( + incremental_result.profile_updates["@third_user:test"]["avatar_url"], + ) + class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): """Tests Sync Handler state behavior when using `use_state_after.""" From 50e788b9d65c9fbe9f3db1d98107fc0a68cf3c27 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 23 Jun 2026 16:49:08 +0300 Subject: [PATCH 062/106] Add foreign key to `profile_updates_per_user.stream_id` --- synapse/storage/schema/main/delta/95/01_profile_updates.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/schema/main/delta/95/01_profile_updates.sql b/synapse/storage/schema/main/delta/95/01_profile_updates.sql index fc31c48262f..1c487bbe95b 100644 --- a/synapse/storage/schema/main/delta/95/01_profile_updates.sql +++ b/synapse/storage/schema/main/delta/95/01_profile_updates.sql @@ -36,7 +36,7 @@ CREATE INDEX profile_updates_by_field ON profile_updates (field_name, stream_id) -- Track which local users should receive each profile update. CREATE TABLE profile_updates_per_user ( - stream_id BIGINT NOT NULL, + stream_id BIGINT NOT NULL REFERENCES profile_updates (stream_id), -- The full user ID of the local user that should receive the profile update. user_id TEXT NOT NULL, From ddb0c52fde792c99592ab702c1955c7a7119d799 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 23 Jun 2026 15:24:52 +0100 Subject: [PATCH 063/106] Actually run MSC4429 complement tests in CI I only just noticed that the Complement tests weren't actually running in CI... --- scripts-dev/complement.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index cca87d42a9e..d8e553f446f 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -286,6 +286,7 @@ main() { ./tests/msc4155 ./tests/msc4306 ./tests/msc4222 + ./tests/msc4429 ) # Export the list of test packages as a space-separated environment variable, so other From d3aacee2354b29fd6ce0b4c9f7666c8ba66ae9a8 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 24 Jun 2026 13:32:19 +0300 Subject: [PATCH 064/106] Fix `test_update_profile_set_field_writes_to_per_user_profile_tracking_table` test And remove unused `get_profile_updates_per_user_for_user` function --- synapse/storage/databases/main/profile.py | 34 ----------- tests/handlers/test_profile.py | 69 +++++++++++++++++++---- 2 files changed, 57 insertions(+), 46 deletions(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 3a8a54da5e8..1bbc1593f63 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -672,40 +672,6 @@ def _track_profile_updates_per_user_txn(txn: LoggingTransaction) -> None: _track_profile_updates_per_user_txn, ) - async def get_profile_updates_per_user_for_user( - self, *, from_id: int, to_id: int, user_id: str - ) -> list[int]: - """ - Get profile updates per user stream ID's for a particular user. - - Args: - from_id: The starting stream ID (exclusive) - to_id: The ending stream ID (inclusive) - user_id: The full user ID to filter on - - Returns: - List of stream ID's. - """ - - def _get_profile_updates_per_user_for_user_txn( - txn: LoggingTransaction, - ) -> list[int]: - sql = """ - SELECT - stream_id - FROM profile_updates_per_user - WHERE - ? < stream_id AND stream_id <= ? AND user_id = ? - """ - txn.execute(sql, (from_id, to_id, user_id)) - rows = cast(list[tuple[int]], txn.fetchall()) - return [row[0] for row in rows] - - return await self.db_pool.runInteraction( - "get_profile_updates_per_user_for_user", - _get_profile_updates_per_user_for_user_txn, - ) - async def create_profile(self, user_id: UserID) -> None: """ Create a blank profile for a user. diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 5267a6ab3b3..00b0aca565f 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -339,29 +339,74 @@ def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( ) ) per_user_updates = self.get_success( - self.store.get_profile_updates_per_user_for_user( - from_id=1, - to_id=2, + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, user_id="@roger:test", + field_names={"m.status"}, ) ) - self.assertEqual(per_user_updates, [2]) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=4, + user_id="@millie:test", + action="joined_room", + field_name=None, + ), + ProfileUpdate( + stream_id=5, + user_id=self.frank.to_string(), + action="update", + field_name="m.status", + ), + ], + ) per_user_updates = self.get_success( - self.store.get_profile_updates_per_user_for_user( - from_id=1, - to_id=2, + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, user_id="@millie:test", + field_names={"m.status"}, ) ) - self.assertEqual(per_user_updates, [2]) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=5, + user_id=self.frank.to_string(), + action="update", + field_name="m.status", + ), + ], + ) per_user_updates = self.get_success( - self.store.get_profile_updates_per_user_for_user( - from_id=1, - to_id=2, + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, user_id=self.frank.to_string(), + field_names={"m.status"}, ) ) - self.assertEqual(per_user_updates, []) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=3, + user_id="@roger:test", + action="joined_room", + field_name=None, + ), + ProfileUpdate( + stream_id=4, + user_id="@millie:test", + action="joined_room", + field_name=None, + ), + ], + ) @parameterized.expand( [ From 7c728ec61c52528529d91bd59d7987636eee054d Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 24 Jun 2026 13:45:13 +0300 Subject: [PATCH 065/106] Optimize profile handler `user_joined_room` and `user_left_room` Save a db query if there are no users to add profile updates for. --- synapse/handlers/profile.py | 31 ++++++++++++++++--------------- tests/handlers/test_profile.py | 10 +++++----- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index ba5a0ceab4d..d04b3129c6b 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -443,11 +443,6 @@ async def user_left_room(self, user_id: UserID, room_id: str) -> None: If that is the case, """ user_id_str = user_id.to_string() - stream_id = await self.store.add_profile_updates( - user_id=user_id, - action=ProfileUpdateAction.LEFT_ROOM.value, - updated_fields=None, - ) users_in_left_room = set(await self.store.get_local_users_in_room(room_id)) users_in_left_room.discard(user_id_str) @@ -460,6 +455,12 @@ async def user_left_room(self, user_id: UserID, room_id: str) -> None: users_to_update = users_in_left_room - users_still_sharing_rooms if users_to_update: + stream_id = await self.store.add_profile_updates( + user_id=user_id, + action=ProfileUpdateAction.LEFT_ROOM.value, + updated_fields=None, + ) + await self.store.track_profile_updates_per_user( stream_id=stream_id, user_ids=users_to_update, @@ -479,22 +480,22 @@ async def user_joined_room(self, user_id: UserID, room_id: str) -> None: If that is the case, """ user_id_str = user_id.to_string() - stream_id = await self.store.add_profile_updates( - user_id=user_id, - action=ProfileUpdateAction.JOINED_ROOM.value, - updated_fields=None, - ) users_in_room = set(await self.store.get_local_users_in_room(room_id)) users_in_room.discard(user_id_str) if not users_in_room: return - if users_in_room: - await self.store.track_profile_updates_per_user( - stream_id=stream_id, - user_ids=users_in_room, - ) + stream_id = await self.store.add_profile_updates( + user_id=user_id, + action=ProfileUpdateAction.JOINED_ROOM.value, + updated_fields=None, + ) + + await self.store.track_profile_updates_per_user( + stream_id=stream_id, + user_ids=users_in_room, + ) async def delete_profile_upon_deactivation( self, diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 00b0aca565f..ab9893cbb13 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -350,13 +350,13 @@ def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( per_user_updates, [ ProfileUpdate( - stream_id=4, + stream_id=3, user_id="@millie:test", action="joined_room", field_name=None, ), ProfileUpdate( - stream_id=5, + stream_id=4, user_id=self.frank.to_string(), action="update", field_name="m.status", @@ -375,7 +375,7 @@ def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( per_user_updates, [ ProfileUpdate( - stream_id=5, + stream_id=4, user_id=self.frank.to_string(), action="update", field_name="m.status", @@ -394,13 +394,13 @@ def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( per_user_updates, [ ProfileUpdate( - stream_id=3, + stream_id=2, user_id="@roger:test", action="joined_room", field_name=None, ), ProfileUpdate( - stream_id=4, + stream_id=3, user_id="@millie:test", action="joined_room", field_name=None, From 05634f90b48f5cacb1c24b59a83838a7c9e19ef6 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 24 Jun 2026 16:29:32 +0300 Subject: [PATCH 066/106] Ensure `room_member` handler writes profile updates to the right profile update stream writer --- synapse/handlers/room_member.py | 57 ++++++++++++++++++++++++++-- synapse/replication/http/profile.py | 59 +++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 2ba29fbe485..956e020a463 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -53,6 +53,7 @@ from synapse.logging import opentracing from synapse.logging.opentracing import SynapseTags, set_tag, tag_args, trace from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions +from synapse.replication.http.profile import ReplicationProfileUserRoomMembershipChange from synapse.replication.http.push import ReplicationCopyPusherRestServlet from synapse.storage.databases.main.state_deltas import StateDelta from synapse.storage.invite_rule import InviteRule @@ -205,6 +206,15 @@ def __init__(self, hs: "HomeServer"): ) self._push_writer = hs.config.worker.writers.push_rules[0] self._copy_push_client = ReplicationCopyPusherRestServlet.make_client(hs) + self._is_profile_worker = ( + hs.get_instance_name() in hs.config.worker.writers.profile_updates + ) + self._profile_updates_writer_instance = ( + self.hs.config.worker.writers.profile_updates[0] + ) + self._profile_user_room_membership_change_client = ( + ReplicationProfileUserRoomMembershipChange.make_client(self.hs) + ) def _on_user_joined_room(self, event_id: str, room_id: str) -> None: """Notify the rate limiter that a room join has occurred. @@ -529,11 +539,32 @@ async def _local_membership_update( # Notify the profile handler. We only want to do this once # in a multi-worker setup, so we can't listen on the dispatched # event above. - await self.profile_handler.user_left_room(target, room_id) + if self._is_profile_worker: + await self.profile_handler.user_left_room( + target, room_id + ) + else: + # Offload to the right worker via http replication + await self._profile_user_room_membership_change_client( + instance_name=self._profile_updates_writer_instance, + user_id=target.to_string(), + room_id=room_id, + membership=Membership.LEAVE, + ) + elif event.membership == Membership.JOIN: # Notify the profile handler. We only want to do this once # in a multi-worker setup, so we can't dispatch a hook to all workers. - await self.profile_handler.user_joined_room(target, room_id) + if self._is_profile_worker: + await self.profile_handler.user_joined_room(target, room_id) + else: + # Offload to the right worker via http replication + await self._profile_user_room_membership_change_client( + instance_name=self._profile_updates_writer_instance, + user_id=target.to_string(), + room_id=room_id, + membership=Membership.JOIN, + ) break except PartialStateConflictError as e: @@ -1552,11 +1583,29 @@ async def send_membership_event( # Notify the profile handler. We only want to do this once # in a multi-worker setup, so we can't listen on the dispatched # event above. - await self.profile_handler.user_left_room(target_user, room_id) + if self._is_profile_worker: + await self.profile_handler.user_left_room(target_user, room_id) + else: + # Offload to the right worker via http replication + await self._profile_user_room_membership_change_client( + instance_name=self._profile_updates_writer_instance, + user_id=target_user.to_string(), + room_id=room_id, + membership=Membership.LEAVE, + ) elif event.membership == Membership.JOIN: # Notify the profile handler. We only want to do this once # in a multi-worker setup, so we can't dispatch a hook to all workers. - await self.profile_handler.user_joined_room(target_user, room_id) + if self._is_profile_worker: + await self.profile_handler.user_joined_room(target_user, room_id) + else: + # Offload to the right worker via http replication + await self._profile_user_room_membership_change_client( + instance_name=self._profile_updates_writer_instance, + user_id=target_user.to_string(), + room_id=room_id, + membership=Membership.JOIN, + ) async def _can_guest_join(self, partial_current_state_ids: StateMap[str]) -> bool: """ diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py index c967a45c556..21a79226b67 100644 --- a/synapse/replication/http/profile.py +++ b/synapse/replication/http/profile.py @@ -18,6 +18,7 @@ from twisted.web.server import Request +from synapse.api.constants import Membership from synapse.http.server import HttpServer from synapse.replication.http._base import ReplicationEndpoint from synapse.types import JsonDict, UserID, create_requester @@ -101,5 +102,63 @@ async def _handle_request( # type: ignore[override] return (200, {}) +class ReplicationProfileUserRoomMembershipChange(ReplicationEndpoint): + """Store user profile update action regarding membership changes. + + The POST looks like: + + POST /_synapse/replication/profile_user_room_membership_change/ + + { + "room_id": "!1234:domain.tld", + "membership": "join | leave" + } + + 200 OK + + {} + """ + + NAME = "profile_user_room_membership_change" + PATH_ARGS = ("user_id",) + METHOD = "POST" + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self._profile_handler = hs.get_profile_handler() + + @staticmethod + async def _serialize_payload( # type: ignore[override] + user_id: str, + room_id: str, + membership: str, + ) -> JsonDict: + assert membership in (Membership.JOIN, Membership.LEAVE) + return { + "room_id": room_id, + "membership": membership, + } + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_id: str + ) -> tuple[int, JsonDict]: + assert content["membership"] in (Membership.JOIN, Membership.LEAVE) + if content["membership"] == Membership.JOIN: + await self._profile_handler.user_joined_room( + user_id=UserID.from_string(user_id), + room_id=content["room_id"], + ) + else: + await self._profile_handler.user_left_room( + user_id=UserID.from_string(user_id), + room_id=content["room_id"], + ) + + return (200, {}) + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: ReplicationProfileSetFieldValue(hs).register(http_server) + ReplicationProfileUserRoomMembershipChange(hs).register(http_server) From b657d2f8db272cf95312d789990ae1584157192c Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 24 Jun 2026 19:11:41 +0300 Subject: [PATCH 067/106] Add a prune job for the profile updates stream tables Taken and adapted from https://github.com/element-hq/synapse/pull/19473 Didn't add the "safety" table due to afaik that has more relevance with the device updates, and not so much here for the profile updates. --- synapse/storage/databases/main/profile.py | 140 ++++++++++++++++++ .../main/delta/95/01_profile_updates.sql | 5 +- tests/storage/test_profile.py | 118 +++++++++++++++ 3 files changed, 262 insertions(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 1bbc1593f63..7185c88ad87 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -19,6 +19,7 @@ # # import json +import logging from typing import TYPE_CHECKING, Collection, Iterable, cast import attr @@ -26,6 +27,7 @@ from synapse.api.constants import ProfileFields, ProfileUpdateAction from synapse.api.errors import Codes, StoreError +from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.replication.tcp.streams._base import ProfileUpdatesStream from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause from synapse.storage.database import ( @@ -37,14 +39,24 @@ from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import JsonDict, JsonValue, UserID +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer +logger = logging.getLogger(__name__) # The number of bytes that the serialized profile can have. MAX_PROFILE_SIZE = 65536 +# Prunes entries out of the `profile_updates` and `profile_updates_per_user` tables +# that are more than this old. +PRUNE_PROFILE_UPDATES_AGE = Duration(days=30) + +# The number of rows to delete at once when pruning old entries out of the +# `profile_updates` and `profile_updates_per_user` tables. +PRUNE_PROFILE_UPDATES_BATCH_SIZE = 1000 + @attr.s(slots=True, frozen=True, auto_attribs=True) class ProfileUpdate: @@ -95,6 +107,11 @@ def __init__( sequence_name="profile_updates_sequence", writers=hs.config.worker.writers.profile_updates, ) + if hs.config.worker.run_background_tasks: + self.clock.looping_call( + self._prune_profile_updates, + Duration(hours=1), + ) async def populate_full_user_id_profiles( self, progress: JsonDict, batch_size: int @@ -927,6 +944,129 @@ async def delete_profile(self, user_id: UserID) -> None: keyvalues={"full_user_id": user_id.to_string()}, ) + @wrap_as_background_process("prune_profile_updates") + async def _prune_profile_updates(self) -> None: + """Delete old entries out of the `profile_updates` and + `profile_updates_per_user` tables, so that the tables don't grow indefinitely. + """ + prune_before_ts = self.clock.time_msec() - PRUNE_PROFILE_UPDATES_AGE.as_millis() + cutoff_sql = """ + SELECT stream_id FROM profile_updates + WHERE inserted_ts <= ? + ORDER BY inserted_ts DESC + LIMIT 1 + """ + + def get_prune_before_stream_id_txn(txn: LoggingTransaction) -> int | None: + txn.execute(cutoff_sql, (prune_before_ts,)) + row = txn.fetchone() + return row[0] if row else None + + prune_before_stream_id = await self.db_pool.runInteraction( + "prune_profile_updates_get_stream_id", + get_prune_before_stream_id_txn, + ) + + if prune_before_stream_id is None: + return + + # Get the max stream ID in the table so we avoid deleting it. We need + # to keep the latest row so that we can calculate the maximum stream ID + # used. + max_stream_id = await self.db_pool.simple_select_one_onecol( + table="profile_updates", + keyvalues={}, + retcol="MAX(stream_id)", + desc="prune_profile_updates_get_max_stream_id", + ) + if prune_before_stream_id >= max_stream_id: + prune_before_stream_id = max_stream_id - 1 + + logger.debug( + "Pruning profile_updates before stream ID %d (timestamp %d)", + prune_before_stream_id, + prune_before_ts, + ) + # Now delete all rows with stream_id less than the + # prune_before_stream_id. + # + # We also delete in batches to avoid massive churn when initially + # clearing out all the old entries. + # + # We set a minimum stream ID so that when we delete in batches the + # database doesn't have to scan through all the (dead) tuples that were just + # deleted to find the next batch to delete. + + # The minimum stream ID to delete in the next batch, c.f. comment above. + # We default to 0 here as that is less than all possible stream IDs. + min_stream_id = 0 + + def prune_profile_updates_txn(txn: LoggingTransaction) -> int: + nonlocal min_stream_id + + assert table in ("profile_updates", "profile_updates_per_user") + delete_sql = """ + DELETE FROM %s + WHERE stream_id IN ( + SELECT stream_id FROM %s + WHERE ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC + LIMIT ? + ) + RETURNING stream_id + """ % (table, table) + txn.execute( + delete_sql, + ( + min_stream_id, + prune_before_stream_id, + PRUNE_PROFILE_UPDATES_BATCH_SIZE, + ), + ) + + # We can't use rowcount as that is incorrect on SQLite when using + # RETURNING. + num_deleted = 0 + for row in txn: + num_deleted += 1 + min_stream_id = max(min_stream_id, row[0]) + + return num_deleted + + # Do this twice, first for the per_user table, then for the main table + for table in ("profile_updates_per_user", "profile_updates"): + progress_num_rows_deleted = 0 + while True: + batch_deleted = await self.db_pool.runInteraction( + f"prune_{table}", + prune_profile_updates_txn, + ) + + finished = batch_deleted < PRUNE_PROFILE_UPDATES_BATCH_SIZE + + progress_num_rows_deleted += batch_deleted + + # Periodically report progress in the logs. We do this either when + # we've deleted a significant number of rows or when we've finished + # deleting all rows in this round. + if finished or progress_num_rows_deleted > 10000: + logger.info( + "Pruned %d rows from %s", + progress_num_rows_deleted, + table, + ) + progress_num_rows_deleted = 0 + + if finished: + break + + # Sleep for a short time to avoid hammering the database too much if + # there are a lot of rows to delete. + await self.clock.sleep(Duration(milliseconds=100)) + + # Reset the minimum stream id for our next table + min_stream_id = 0 + class ProfileStore(ProfileWorkerStore): pass diff --git a/synapse/storage/schema/main/delta/95/01_profile_updates.sql b/synapse/storage/schema/main/delta/95/01_profile_updates.sql index 1c487bbe95b..3e56300e0fc 100644 --- a/synapse/storage/schema/main/delta/95/01_profile_updates.sql +++ b/synapse/storage/schema/main/delta/95/01_profile_updates.sql @@ -27,12 +27,14 @@ CREATE TABLE profile_updates ( -- This is only required if "action" is "update" field_name TEXT NULL, - -- Unix timestamp for debugging purposes + -- Unix timestamp. Used to determine when to cull rows (to prevent the table + -- from growing indefinitely). inserted_ts BIGINT NOT NULL ); CREATE INDEX profile_updates_by_user ON profile_updates (user_id, stream_id); CREATE INDEX profile_updates_by_field ON profile_updates (field_name, stream_id); +CREATE INDEX profile_updates_inserted_ts ON profile_updates (inserted_ts); -- Track which local users should receive each profile update. CREATE TABLE profile_updates_per_user ( @@ -47,3 +49,4 @@ CREATE TABLE profile_updates_per_user ( ); CREATE INDEX profile_updates_per_user_by_user_stream ON profile_updates_per_user (user_id, stream_id); +CREATE INDEX profile_updates_per_user_inserted_ts ON profile_updates_per_user (inserted_ts); diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index dbaf2986975..d02c7e6388d 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -18,14 +18,19 @@ # [This file includes modifications made by New Vector Limited] # # +import itertools +from unittest.mock import patch from twisted.internet.testing import MemoryReactor +from synapse.api.constants import ProfileUpdateAction from synapse.server import HomeServer from synapse.storage.database import LoggingTransaction +from synapse.storage.databases.main.profile import PRUNE_PROFILE_UPDATES_AGE from synapse.storage.engines import PostgresEngine from synapse.types import UserID from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest @@ -132,3 +137,116 @@ def f(txn: LoggingTransaction) -> None: ) self.assertEqual(len(res), len(expected_values)) self.assertEqual(res, expected_values) + + @patch("synapse.storage.databases.main.profile.PRUNE_PROFILE_UPDATES_BATCH_SIZE", 5) + def test_prune_profile_updates(self) -> None: + """Test that old entries in the `profile_updates` and `profile_updates_per_user` + tables are pruned properly.""" + + # Create a generator for field names so we can easily create many unique + # field names without having to keep track of the count ourselves. + field_name_gen = (f"field{i}" for i in itertools.count()) + + def get_profile_updates_status() -> tuple[int, str]: + """Helper function to get the count of entries in the + `profile_updates` table.""" + return self.get_success( + self.store.db_pool.simple_select_one( + table="profile_updates", + keyvalues={}, + retcols=("COUNT(*)", "MIN(field_name)"), + ) + ) + + def get_profile_updates_per_user_status() -> tuple[int]: + """Helper function to get the count of entries in the + `profile_updates_per_user` table.""" + return self.get_success( + self.store.db_pool.simple_select_one( + table="profile_updates_per_user", + keyvalues={}, + retcols=("COUNT(*)",), + ) + ) + + # First add some entries + for _ in range(10): + stream_id = self.get_success( + self.store.add_profile_updates( + user_id=UserID.from_string("@user:test"), + updated_fields=[next(field_name_gen)], + action=ProfileUpdateAction.UPDATE.value, + ) + ) + self.get_success( + self.store.track_profile_updates_per_user( + stream_id=stream_id, + user_ids={"@alice:test", "@bob:test"}, + ) + ) + + # Advance the reactor a while, but not long enough to trigger pruning. + self.reactor.advance(Duration(hours=1).as_secs()) + + # The `profile_updates_per_user` table should now have 10 * 2 entries. + per_user_count = get_profile_updates_per_user_status() + self.assertEqual(per_user_count[0], 20) + # The `profile_updates` table should have 10 entries. + # and the minimum field name should be `field0`. + updates_count, min_field_name = get_profile_updates_status() + self.assertEqual(updates_count, 10) + self.assertEqual(min_field_name, "field0") + + # Now we add some more entries + for _ in range(10): + stream_id = self.get_success( + self.store.add_profile_updates( + user_id=UserID.from_string("@user:test"), + updated_fields=[next(field_name_gen)], + action=ProfileUpdateAction.UPDATE.value, + ) + ) + self.get_success( + self.store.track_profile_updates_per_user( + stream_id=stream_id, + user_ids={"@alice:test", "@bob:test"}, + ) + ) + + # Advance the reactor a while more, so that the first batch of entries is + # now old enough to be pruned. + self.reactor.advance( + (PRUNE_PROFILE_UPDATES_AGE - Duration(minutes=30)).as_secs() + ) + + # Advance repeatedly a bit so that the pruning process can run to completion. + for _ in range(10): + self.reactor.advance(Duration(milliseconds=110).as_secs()) + + # Check that the old entries have been pruned, and the new entries are still there. + # The `profile_updates_per_user` table should now have 10 * 2 entries. + per_user_count = get_profile_updates_per_user_status() + self.assertEqual(per_user_count[0], 20) + # The `profile_updates` table should have 10 entries. + # and the minimum field name should be `field10`. + updates_count, min_field_name = get_profile_updates_status() + self.assertEqual(updates_count, 10) + self.assertEqual(min_field_name, "field10") + + # We should always keep the most recent entries, even if they are old enough to be pruned. + self.reactor.advance( + (PRUNE_PROFILE_UPDATES_AGE + Duration(minutes=30)).as_secs() + ) + + # Advance repeatedly a bit so that the pruning process can run to completion. + for _ in range(10): + self.reactor.advance(Duration(milliseconds=110).as_secs()) + + # The `profile_updates_per_user` table should now have 2 entries. + per_user_count = get_profile_updates_per_user_status() + self.assertEqual(per_user_count[0], 2) + # The `profile_updates` table should have 1 entry. + # and the minimum field name should be `field19`. + updates_count, min_field_name = get_profile_updates_status() + self.assertEqual(updates_count, 1) + self.assertEqual(min_field_name, "field19") From 3e4301ad01ef2f00386302fd75a2a0ef3118805f Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 25 Jun 2026 15:26:43 +0300 Subject: [PATCH 068/106] Remove unnecessary if --- synapse/api/filtering.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index ee20e3d6c87..3f4d6d40b7a 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -233,8 +233,6 @@ def __init__(self, hs: "HomeServer", filter_json: JsonMapping): if isinstance(profile_fields_filter, Mapping): ids = profile_fields_filter.get("ids", []) - if ids is None: - ids = [] self.profile_fields = set(ids) def __repr__(self) -> str: From 5f761ab5b47f257fa5b3c7f5120a67e3a64127fa Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 25 Jun 2026 17:39:42 +0300 Subject: [PATCH 069/106] Use set for uniqueness --- synapse/handlers/profile.py | 14 +++++++------- synapse/storage/databases/main/profile.py | 2 +- tests/storage/test_profile.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index d04b3129c6b..4a47df2fdb4 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -103,14 +103,14 @@ def __init__(self, hs: "HomeServer"): self._worker_locks = hs.get_worker_locks_handler() async def _record_profile_updates( - self, user_id: UserID, updated_fields: list[str] + self, user_id: UserID, updated_fields: set[str] ) -> None: """ Record user profile updates to our stream updates table. Args: user_id: The user whose profile has had updates. - updated_fields: A list of the names of the fields that were updated. + updated_fields: A set of the names of the fields that were updated. Returns: None @@ -300,7 +300,7 @@ async def set_displayname( await self.store.set_profile_displayname(target_user, displayname_to_set) await self._record_profile_updates( target_user, - [ProfileFields.DISPLAYNAME], + {ProfileFields.DISPLAYNAME}, ) profile = await self.store.get_profileinfo(target_user) @@ -413,7 +413,7 @@ async def set_avatar_url( await self.store.set_profile_avatar_url(target_user, avatar_url_to_set) await self._record_profile_updates( target_user, - [ProfileFields.AVATAR_URL], + {ProfileFields.AVATAR_URL}, ) profile = await self.store.get_profileinfo(target_user) @@ -545,7 +545,7 @@ async def delete_profile_upon_deactivation( await self.store.delete_profile(target_user) await self._record_profile_updates( - target_user, [field_name for field_name, _value in profile_updates] + target_user, {field_name for field_name, _value in profile_updates} ) await self._third_party_rules.on_profile_update( @@ -746,7 +746,7 @@ async def set_profile_field( raise AuthError(403, "Cannot set another user's profile") await self.store.set_profile_field(target_user, field_name, new_value) - await self._record_profile_updates(target_user, [field_name]) + await self._record_profile_updates(target_user, {field_name}) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) @@ -782,7 +782,7 @@ async def delete_profile_field( raise AuthError(400, "Cannot set another user's profile") await self.store.delete_profile_field(target_user, field_name) - await self._record_profile_updates(target_user, [field_name]) + await self._record_profile_updates(target_user, {field_name}) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 7185c88ad87..605bfacf3a8 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -592,7 +592,7 @@ async def add_profile_updates( self, user_id: UserID, action: str, - updated_fields: list[str] | None, + updated_fields: set[str] | None, ) -> int: """Persist profile update markers and return the last stream ID.""" assert self._can_write_to_profile_updates diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index d02c7e6388d..f672c39a972 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -174,7 +174,7 @@ def get_profile_updates_per_user_status() -> tuple[int]: stream_id = self.get_success( self.store.add_profile_updates( user_id=UserID.from_string("@user:test"), - updated_fields=[next(field_name_gen)], + updated_fields={next(field_name_gen)}, action=ProfileUpdateAction.UPDATE.value, ) ) @@ -202,7 +202,7 @@ def get_profile_updates_per_user_status() -> tuple[int]: stream_id = self.get_success( self.store.add_profile_updates( user_id=UserID.from_string("@user:test"), - updated_fields=[next(field_name_gen)], + updated_fields={next(field_name_gen)}, action=ProfileUpdateAction.UPDATE.value, ) ) From bf10187a1dbe94fa6e170d5ba1f9de1184d30c8e Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 25 Jun 2026 18:35:40 +0300 Subject: [PATCH 070/106] Docstring for ProfileUpdateAction --- synapse/api/constants.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 80505da6d1e..952beb02571 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -411,9 +411,22 @@ class ProfileFields: class ProfileUpdateAction(enum.Enum): + """ + Enum representing the action of a row in the profile updates stream tables. + """ + JOINED_ROOM = "joined_room" + """ + User joined a room. + """ LEFT_ROOM = "left_room" + """ + User left a room. + """ UPDATE = "update" + """ + User updated a profile field. + """ class StickyEventField(TypedDict): From 585590bbd9d21d710afb85e45f70e5d14581e93a Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 25 Jun 2026 18:46:58 +0300 Subject: [PATCH 071/106] Protect against stream being rewound --- synapse/storage/databases/main/profile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 605bfacf3a8..7c41c4562c8 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -368,7 +368,7 @@ async def get_updated_profile_updates( Returns: list of tuples representing stream_id, user_id, action and field_name """ - if from_id == to_id: + if from_id >= to_id: return [] def _get_updated_profile_updates_txn( @@ -408,7 +408,7 @@ async def get_profile_updates_for_fields( Returns: list of ProfileUpdates update rows """ - if from_id == to_id: + if from_id >= to_id: return [] field_names = list(field_names) @@ -474,7 +474,7 @@ async def get_profile_updates_for_user_and_fields( Returns: A list of ProfileUpdates update rows. """ - if from_id == to_id: + if from_id >= to_id: return [] if len(field_names) == 0: From e278dfdb035db264634ffb0daa5b50e9d3526f19 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 25 Jun 2026 21:43:49 +0300 Subject: [PATCH 072/106] Restore initial sync behaviour to return full profiles of users currently sharing rooms --- synapse/handlers/sync.py | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index eb9b186775b..ce09ebf3574 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2196,35 +2196,25 @@ async def _generate_initial_sync_entry_for_profile_updates( for when we have calculated a list of users in our lazy loading sync and want to only return those. """ - updates = await self.store.get_profile_updates_for_user_and_fields( - from_id=0, - to_id=sync_result_builder.now_token.profile_updates_key, - user_id=user_id, - field_names=profile_fields, - include_users=include_users, - ) - - user_fields: dict[str, set[str]] = {} - for update in updates: - if ( - update.action != ProfileUpdateAction.UPDATE.value - # TODO: When would field_name be None? - or update.field_name is None - ): - continue + # Currently, limited to only local profiles, so filter remote servers out + user_ids = await self.store.get_local_users_who_share_room_with_user(user_id) + if include_users: + # Filter down to selected included users + user_ids = {user_id for user_id in user_ids if user_id in include_users} - user_fields.setdefault(update.user_id, set()).add(update.field_name) + # Remove ourselves + # FIXME?: Should `get_local_users_who_share_room_with_user` even return + # ourselves? + user_ids.discard(user_id) - if not user_fields: + if not user_ids: return - profile_data_by_user = await self.store.get_profile_data_for_users( - user_fields.keys() - ) + profile_data_by_user = await self.store.get_profile_data_for_users(user_ids) # Serialise the profile updates into the sync response format. profile_updates: dict[str, dict[str, JsonValue | None] | None] = {} - for other_user_id, fields in user_fields.items(): + for other_user_id in user_ids: profile_data = profile_data_by_user.get(other_user_id) if profile_data is None: # Don't generate anything for users with no profile data @@ -2232,7 +2222,7 @@ async def _generate_initial_sync_entry_for_profile_updates( continue per_user_updates: dict[str, JsonValue] = {} - for field_name in fields: + for field_name in profile_fields: if profile_data.get(field_name): per_user_updates[field_name] = cast( JsonValue, profile_data[field_name] From 8bace9f97ba56386250bc5a477fefafb4096f5fb Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 25 Jun 2026 22:29:11 +0300 Subject: [PATCH 073/106] Fix some types in various places regarding profile field values I believe the correct type is `JsonValue | dict[str, JsonValue]`, except deep down in the displayname/avatar_url store methods. Removes a dubious cast by correctly casting in `get_profile_data_for_users` --- synapse/handlers/profile.py | 14 +++++++++--- synapse/handlers/sync.py | 22 ++++++++++-------- synapse/replication/http/profile.py | 4 ++-- synapse/rest/client/profile.py | 4 +++- synapse/storage/databases/main/profile.py | 27 ++++++++++++++++------- 5 files changed, 48 insertions(+), 23 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 4a47df2fdb4..c9203b915f0 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -632,7 +632,7 @@ async def check_avatar_size_and_mime_type(self, mxc: str) -> bool: async def get_profile_field( self, target_user: UserID, field_name: str - ) -> JsonValue: + ) -> JsonValue | dict[str, JsonValue]: """ Fetch a user's profile from the database for local users and over federation for remote users. @@ -676,12 +676,16 @@ async def set_field( target_user: UserID, requester: Requester, field_name: str, - new_value: str, + new_value: JsonValue | dict[str, JsonValue], by_admin: bool = False, propagate: bool = False, ) -> None: """Wrapper function for setting any profile field for a user.""" if field_name == ProfileFields.DISPLAYNAME: + if not isinstance(new_value, str): + raise SynapseError( + 400, "'displayname' must be a string", errcode=Codes.INVALID_PARAM + ) await self.set_displayname( target_user=target_user, requester=requester, @@ -690,6 +694,10 @@ async def set_field( propagate=propagate, ) elif field_name == ProfileFields.AVATAR_URL: + if not isinstance(new_value, str): + raise SynapseError( + 400, "'avatar_url' must be a string", errcode=Codes.INVALID_PARAM + ) await self.set_avatar_url( target_user=target_user, requester=requester, @@ -721,7 +729,7 @@ async def set_profile_field( target_user: UserID, requester: Requester, field_name: str, - new_value: JsonValue, + new_value: JsonValue | dict[str, JsonValue], *, by_admin: bool = False, ) -> None: diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ce09ebf3574..f349d93ae06 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -252,7 +252,7 @@ class SyncResult: presence: list[UserPresenceState] account_data: list[JsonDict] # user ID -> {profile field -> value | null if unset } - profile_updates: dict[str, dict[str, JsonValue | None] | None] + profile_updates: dict[str, dict[str, JsonValue | dict[str, JsonValue]] | None] joined: list[JoinedSyncResult] invited: list[InvitedSyncResult] knocked: list[KnockedSyncResult] @@ -2213,7 +2213,9 @@ async def _generate_initial_sync_entry_for_profile_updates( profile_data_by_user = await self.store.get_profile_data_for_users(user_ids) # Serialise the profile updates into the sync response format. - profile_updates: dict[str, dict[str, JsonValue | None] | None] = {} + profile_updates: dict[ + str, dict[str, JsonValue | dict[str, JsonValue]] | None + ] = {} for other_user_id in user_ids: profile_data = profile_data_by_user.get(other_user_id) if profile_data is None: @@ -2221,12 +2223,10 @@ async def _generate_initial_sync_entry_for_profile_updates( # in initial sync. continue - per_user_updates: dict[str, JsonValue] = {} + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} for field_name in profile_fields: if profile_data.get(field_name): - per_user_updates[field_name] = cast( - JsonValue, profile_data[field_name] - ) + per_user_updates[field_name] = profile_data[field_name] if per_user_updates: profile_updates[other_user_id] = per_user_updates @@ -2321,7 +2321,9 @@ async def _generate_sync_entry_for_profile_updates( # Serialise the profile updates into the sync response format. # user ID -> {profile field -> value | null if unset } - profile_updates: dict[str, dict[str, JsonValue | None] | None] = {} + profile_updates: dict[ + str, dict[str, JsonValue | dict[str, JsonValue]] | None + ] = {} # Process field updates and users who have events in the sync response if users: @@ -2351,7 +2353,7 @@ async def _generate_sync_entry_for_profile_updates( profile_updates[other_user_id] = None continue - per_user_updates: dict[str, JsonValue] = {} + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} if include_users and other_user_id in include_users: # Include the full profile as this user has events in # a lazy loaded sync response, except for fields we've recently @@ -3443,7 +3445,9 @@ class SyncResultBuilder: presence: list[UserPresenceState] = attr.Factory(list) account_data: list[JsonDict] = attr.Factory(list) - profile_updates: dict[str, dict[str, JsonValue | None] | None] = attr.Factory(dict) + profile_updates: dict[str, dict[str, JsonValue | dict[str, JsonValue]] | None] = ( + attr.Factory(dict) + ) joined: list[JoinedSyncResult] = attr.Factory(list) invited: list[InvitedSyncResult] = attr.Factory(list) knocked: list[KnockedSyncResult] = attr.Factory(list) diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py index 21a79226b67..a692836489f 100644 --- a/synapse/replication/http/profile.py +++ b/synapse/replication/http/profile.py @@ -21,7 +21,7 @@ from synapse.api.constants import Membership from synapse.http.server import HttpServer from synapse.replication.http._base import ReplicationEndpoint -from synapse.types import JsonDict, UserID, create_requester +from synapse.types import JsonDict, JsonValue, UserID, create_requester if TYPE_CHECKING: from synapse.server import HomeServer @@ -65,7 +65,7 @@ async def _serialize_payload( # type: ignore[override] user_id: str, requester_id: str, field_name: str, - new_value: str, + new_value: JsonValue | dict[str, JsonValue], by_admin: bool = False, propagate: bool = False, authenticated_entity: str | None = None, diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 7cd4678bc06..1767b13eb6f 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -150,7 +150,9 @@ async def on_GET( await self.profile_handler.check_profile_query_allowed(user, requester_user) if field_name == ProfileFields.DISPLAYNAME: - field_value: JsonValue = await self.profile_handler.get_displayname(user) + field_value: ( + JsonValue | dict[str, JsonValue] + ) = await self.profile_handler.get_displayname(user) elif field_name == ProfileFields.AVATAR_URL: field_value = await self.profile_handler.get_avatar_url(user) else: diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 7c41c4562c8..f1141f90fba 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -265,7 +265,9 @@ async def get_profile_avatar_url(self, user_id: UserID) -> str | None: desc="get_profile_avatar_url", ) - async def get_profile_field(self, user_id: UserID, field_name: str) -> JsonValue: + async def get_profile_field( + self, user_id: UserID, field_name: str + ) -> JsonValue | dict[str, JsonValue]: """ Get a custom profile field for a user. @@ -277,7 +279,9 @@ async def get_profile_field(self, user_id: UserID, field_name: str) -> JsonValue The string value if the field exists, otherwise raises 404. """ - def get_profile_field(txn: LoggingTransaction) -> JsonValue: + def get_profile_field( + txn: LoggingTransaction, + ) -> JsonValue | dict[str, JsonValue]: # This will error if field_name has double quotes in it, but that's not # possible due to the grammar. field_path = f'$."{field_name}"' @@ -295,7 +299,9 @@ def get_profile_field(txn: LoggingTransaction) -> JsonValue: # Test exists first since value being None is used for both # missing and a null JSON value. - exists, value = cast(tuple[bool, JsonValue], txn.fetchone()) + exists, value = cast( + tuple[bool, JsonValue | dict[str, JsonValue]], txn.fetchone() + ) if not exists: raise StoreError(404, "No row found") return value @@ -312,7 +318,9 @@ def get_profile_field(txn: LoggingTransaction) -> JsonValue: ) # If value_type is None, then the value did not exist. - value_type, value = cast(tuple[str | None, JsonValue], txn.fetchone()) + value_type, value = cast( + tuple[str | None, JsonValue | dict[str, JsonValue]], txn.fetchone() + ) if not value_type: raise StoreError(404, "No row found") # If value_type is object or array, then need to deserialize the JSON. @@ -548,7 +556,7 @@ def _get_profile_updates_for_user_and_fields_txn( async def get_profile_data_for_users( self, user_ids: Collection[str] - ) -> dict[str, dict[str, str | JsonDict | None]]: + ) -> dict[str, dict[str, JsonValue | dict[str, JsonValue]]]: """Fetch displayname/avatar_url/custom fields for a list of users. Currently, this returns only local users as the `profiles` table only @@ -571,7 +579,7 @@ async def get_profile_data_for_users( desc="get_profile_data_for_users", ) - results: dict[str, dict[str, str | JsonDict | None]] = {} + results: dict[str, dict[str, JsonValue | dict[str, JsonValue]]] = {} for full_user_id, displayname, avatar_url, fields in rows: user_fields = fields or {} # The SQLite driver doesn't automatically convert JSON to @@ -708,7 +716,7 @@ def _check_profile_size( txn: LoggingTransaction, user_id: UserID, new_field_name: str, - new_value: JsonValue, + new_value: JsonValue | dict[str, JsonValue], ) -> None: # For each entry there are 4 quotes (2 each for key and value), 1 colon, # and 1 comma. @@ -835,7 +843,10 @@ def set_profile_avatar_url(txn: LoggingTransaction) -> None: ) async def set_profile_field( - self, user_id: UserID, field_name: str, new_value: JsonValue + self, + user_id: UserID, + field_name: str, + new_value: JsonValue | dict[str, JsonValue], ) -> None: """ Set a custom profile field for a user. From 7a60657dd11c27e54ca9c242d90f7a36950f8ee3 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 26 Jun 2026 13:25:53 +0300 Subject: [PATCH 074/106] Clean up recording profile updates on changes Instead of using replication when setting field values, remove that replication, keep writing the field via the profile handler in the current instance, and then dispatch recording the profile update over replication, if needed. This is cleaner as it ensures we don't need to wrap various places outside the profile handler that set things like the displayname in the "if profile worker else replication" logic. --- synapse/handlers/profile.py | 51 +++++++++++++-- synapse/replication/http/profile.py | 99 +++++++++++------------------ synapse/rest/client/profile.py | 71 +++++---------------- 3 files changed, 98 insertions(+), 123 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index c9203b915f0..8bc724ad35c 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -34,6 +34,7 @@ StoreError, SynapseError, ) +from synapse.replication.http.profile import ReplicationProfileRecordFieldUpdates from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia from synapse.storage.roommember import ProfileInfo from synapse.types import ( @@ -101,8 +102,17 @@ def __init__(self, hs: "HomeServer"): self._update_join_states_task, UPDATE_JOIN_STATES_ACTION_NAME ) self._worker_locks = hs.get_worker_locks_handler() + self._is_profile_worker = ( + hs.get_instance_name() in hs.config.worker.writers.profile_updates + ) + self._record_profile_updates_client = ( + ReplicationProfileRecordFieldUpdates.make_client(self.hs) + ) + self._profile_updates_writer_instance = ( + self.hs.config.worker.writers.profile_updates[0] + ) - async def _record_profile_updates( + async def record_profile_updates( self, user_id: UserID, updated_fields: set[str] ) -> None: """ @@ -298,7 +308,7 @@ async def set_displayname( ) await self.store.set_profile_displayname(target_user, displayname_to_set) - await self._record_profile_updates( + await self._dispatch_record_profile_updates( target_user, {ProfileFields.DISPLAYNAME}, ) @@ -411,7 +421,7 @@ async def set_avatar_url( ) await self.store.set_profile_avatar_url(target_user, avatar_url_to_set) - await self._record_profile_updates( + await self._dispatch_record_profile_updates( target_user, {ProfileFields.AVATAR_URL}, ) @@ -544,7 +554,9 @@ async def delete_profile_upon_deactivation( profile_updates.append((field_name, None)) await self.store.delete_profile(target_user) - await self._record_profile_updates( + + # Record profile updates for the profile update stream + await self._dispatch_record_profile_updates( target_user, {field_name for field_name, _value in profile_updates} ) @@ -555,6 +567,33 @@ async def delete_profile_upon_deactivation( deactivation=True, ) + async def _dispatch_record_profile_updates( + self, user_id: UserID, updated_fields: set[str] + ) -> None: + """ + Dispatch the recording of profile updates, either directly via the current + instance, if we're a profile worker, otherwise push via replication. + + Args: + user_id: The user whose profile has had updates. + updated_fields: A set of the names of the fields that were updated. + + Returns: + None + """ + if self._is_profile_worker: + await self.record_profile_updates( + user_id, + updated_fields, + ) + else: + # Offload to the right worker via http replication + await self._record_profile_updates_client( + instance_name=self._profile_updates_writer_instance, + user_id=user_id.to_string(), + updated_fields=updated_fields, + ) + @cached() async def check_avatar_size_and_mime_type(self, mxc: str) -> bool: """Check that the size and content type of the avatar at the given MXC URI are @@ -754,7 +793,7 @@ async def set_profile_field( raise AuthError(403, "Cannot set another user's profile") await self.store.set_profile_field(target_user, field_name, new_value) - await self._record_profile_updates(target_user, {field_name}) + await self._dispatch_record_profile_updates(target_user, {field_name}) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) @@ -790,7 +829,7 @@ async def delete_profile_field( raise AuthError(400, "Cannot set another user's profile") await self.store.delete_profile_field(target_user, field_name) - await self._record_profile_updates(target_user, {field_name}) + await self._dispatch_record_profile_updates(target_user, {field_name}) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py index a692836489f..8b27b088733 100644 --- a/synapse/replication/http/profile.py +++ b/synapse/replication/http/profile.py @@ -21,7 +21,7 @@ from synapse.api.constants import Membership from synapse.http.server import HttpServer from synapse.replication.http._base import ReplicationEndpoint -from synapse.types import JsonDict, JsonValue, UserID, create_requester +from synapse.types import JsonDict, UserID if TYPE_CHECKING: from synapse.server import HomeServer @@ -29,20 +29,16 @@ logger = logging.getLogger(__name__) -class ReplicationProfileSetFieldValue(ReplicationEndpoint): - """Set profile field for a user. +class ReplicationProfileUserRoomMembershipChange(ReplicationEndpoint): + """Store user profile update action regarding membership changes. The POST looks like: - POST /_synapse/replication/profile_set_field_value/ + POST /_synapse/replication/profile_user_room_membership_change/ { - "requester_id": "@user:domain.tld", - "field_name": "displayname", - "new_value": "User Display Name", - "by_admin": False, - "propagate": False, - "authenticated_entity": "@admin:domain.tld", + "room_id": "!1234:domain.tld", + "membership": "join | leave" } 200 OK @@ -50,7 +46,7 @@ class ReplicationProfileSetFieldValue(ReplicationEndpoint): {} """ - NAME = "profile_set_field_value" + NAME = "profile_user_room_membership_change" PATH_ARGS = ("user_id",) METHOD = "POST" CACHE = False @@ -63,55 +59,42 @@ def __init__(self, hs: "HomeServer"): @staticmethod async def _serialize_payload( # type: ignore[override] user_id: str, - requester_id: str, - field_name: str, - new_value: JsonValue | dict[str, JsonValue], - by_admin: bool = False, - propagate: bool = False, - authenticated_entity: str | None = None, + room_id: str, + membership: str, ) -> JsonDict: + assert membership in (Membership.JOIN, Membership.LEAVE) return { - "requester_id": requester_id, - "field_name": field_name, - "new_value": new_value, - "by_admin": by_admin, - "propagate": propagate, - "authenticated_entity": authenticated_entity, + "room_id": room_id, + "membership": membership, } async def _handle_request( # type: ignore[override] self, request: Request, content: JsonDict, user_id: str ) -> tuple[int, JsonDict]: - # Create a requester object with potentially an authenticated_entity, - # ie an admin who has done the request on behalf of the user. - requester = create_requester( - user_id=user_id, - authenticated_entity=content["authenticated_entity"] - if content["by_admin"] - else None, - ) - await self._profile_handler.set_field( - target_user=UserID.from_string(user_id), - requester=requester, - field_name=content["field_name"], - new_value=content["new_value"], - by_admin=content["by_admin"], - propagate=content["propagate"], - ) + assert content["membership"] in (Membership.JOIN, Membership.LEAVE) + if content["membership"] == Membership.JOIN: + await self._profile_handler.user_joined_room( + user_id=UserID.from_string(user_id), + room_id=content["room_id"], + ) + else: + await self._profile_handler.user_left_room( + user_id=UserID.from_string(user_id), + room_id=content["room_id"], + ) return (200, {}) -class ReplicationProfileUserRoomMembershipChange(ReplicationEndpoint): - """Store user profile update action regarding membership changes. +class ReplicationProfileRecordFieldUpdates(ReplicationEndpoint): + """Record user profile field updates for the profile updates stream. The POST looks like: - POST /_synapse/replication/profile_user_room_membership_change/ + POST /_synapse/replication/profile_record_field_updates/ { - "room_id": "!1234:domain.tld", - "membership": "join | leave" + "updated_fields": ["list", "of", "fields"] } 200 OK @@ -119,7 +102,7 @@ class ReplicationProfileUserRoomMembershipChange(ReplicationEndpoint): {} """ - NAME = "profile_user_room_membership_change" + NAME = "profile_record_field_updates" PATH_ARGS = ("user_id",) METHOD = "POST" CACHE = False @@ -132,33 +115,25 @@ def __init__(self, hs: "HomeServer"): @staticmethod async def _serialize_payload( # type: ignore[override] user_id: str, - room_id: str, - membership: str, + updated_fields: set[str], ) -> JsonDict: - assert membership in (Membership.JOIN, Membership.LEAVE) + assert len(updated_fields) > 0 return { - "room_id": room_id, - "membership": membership, + "updated_fields": list(updated_fields), } async def _handle_request( # type: ignore[override] self, request: Request, content: JsonDict, user_id: str ) -> tuple[int, JsonDict]: - assert content["membership"] in (Membership.JOIN, Membership.LEAVE) - if content["membership"] == Membership.JOIN: - await self._profile_handler.user_joined_room( - user_id=UserID.from_string(user_id), - room_id=content["room_id"], - ) - else: - await self._profile_handler.user_left_room( - user_id=UserID.from_string(user_id), - room_id=content["room_id"], - ) + assert len(content["updated_fields"]) > 0 + await self._profile_handler.record_profile_updates( + user_id=UserID.from_string(user_id), + updated_fields=set(content["updated_fields"]), + ) return (200, {}) def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - ReplicationProfileSetFieldValue(hs).register(http_server) ReplicationProfileUserRoomMembershipChange(hs).register(http_server) + ReplicationProfileRecordFieldUpdates(hs).register(http_server) diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 1767b13eb6f..daab2c66518 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -35,7 +35,6 @@ parse_json_object_from_request, ) from synapse.http.site import SynapseRequest -from synapse.replication.http.profile import ReplicationProfileSetFieldValue from synapse.rest.client._base import client_patterns from synapse.types import JsonDict, JsonValue, UserID from synapse.util.stringutils import is_namedspaced_grammar @@ -210,33 +209,14 @@ async def on_PUT( Codes.USER_ACCOUNT_SUSPENDED, ) - if self._is_profile_worker: - await self.profile_handler.set_field( - target_user=user, - requester=requester, - field_name=field_name, - new_value=new_value, - by_admin=is_admin, - propagate=propagate, - ) - else: - # Offload to the right worker via http replication - set_profile_data_client = ReplicationProfileSetFieldValue.make_client( - self.hs - ) - profile_updates_writer_instance = ( - self.hs.config.worker.writers.profile_updates[0] - ) - await set_profile_data_client( - instance_name=profile_updates_writer_instance, - user_id=user.to_string(), - requester_id=requester.user.to_string(), - field_name=field_name, - new_value=new_value, - by_admin=is_admin, - propagate=propagate, - authenticated_entity=requester.authenticated_entity, - ) + await self.profile_handler.set_field( + target_user=user, + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=is_admin, + propagate=propagate, + ) return 200, {} @@ -282,33 +262,14 @@ async def on_DELETE( Codes.USER_ACCOUNT_SUSPENDED, ) - if self._is_profile_worker: - await self.profile_handler.set_field( - target_user=user, - requester=requester, - field_name=field_name, - new_value="", - by_admin=is_admin, - propagate=propagate, - ) - else: - # Offload to the right worker via http replication - set_profile_data_client = ReplicationProfileSetFieldValue.make_client( - self.hs - ) - profile_updates_writer_instance = ( - self.hs.config.worker.writers.profile_updates[0] - ) - await set_profile_data_client( - instance_name=profile_updates_writer_instance, - user_id=user.to_string(), - requester_id=requester.user.to_string(), - field_name=field_name, - new_value="", - by_admin=is_admin, - propagate=propagate, - authenticated_entity=requester.authenticated_entity, - ) + await self.profile_handler.set_field( + target_user=user, + requester=requester, + field_name=field_name, + new_value="", + by_admin=is_admin, + propagate=propagate, + ) return 200, {} From f453e03c8c4860715eeb4e8e8332edc3bc0ba1aa Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 26 Jun 2026 13:29:48 +0300 Subject: [PATCH 075/106] Remove more useless casts --- synapse/handlers/sync.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f349d93ae06..f3169bb863d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2377,8 +2377,8 @@ async def _generate_sync_entry_for_profile_updates( profile_data.get(field_name) is not None or other_user_id in updated_users ): - per_user_updates[field_name] = cast( - JsonValue, profile_data.get(field_name) + per_user_updates[field_name] = profile_data.get( + field_name ) # Update our cache cache.set( @@ -2394,9 +2394,7 @@ async def _generate_sync_entry_for_profile_updates( else user_fields.get(other_user_id, []) ) for field_name in fields: - per_user_updates[field_name] = cast( - JsonValue, profile_data.get(field_name) - ) + per_user_updates[field_name] = profile_data.get(field_name) if per_user_updates: profile_updates[other_user_id] = per_user_updates From b2246de4e3752021071cb6bef66b80822f8f8dfd Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 26 Jun 2026 16:57:33 +0300 Subject: [PATCH 076/106] Ensure we don't call `_dispatch_record_profile_updates` with an empty list of fields --- synapse/handlers/profile.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 8bc724ad35c..70ae1bcca84 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -556,9 +556,10 @@ async def delete_profile_upon_deactivation( await self.store.delete_profile(target_user) # Record profile updates for the profile update stream - await self._dispatch_record_profile_updates( - target_user, {field_name for field_name, _value in profile_updates} - ) + if len(profile_updates): + await self._dispatch_record_profile_updates( + target_user, {field_name for field_name, _value in profile_updates} + ) await self._third_party_rules.on_profile_update( target_user.to_string(), From b459512725ff1112b0931c629f9fc5551b747e82 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 26 Jun 2026 17:57:57 +0300 Subject: [PATCH 077/106] Use `include_profile_updates_in_sync` config item instead of an experimental flag + alias. The requirement for this comes from a customer commitment. --- changelog.d/19556.feature | 4 ++-- .../complement/conf/workers-shared-extra.yaml.j2 | 4 ++-- synapse/api/filtering.py | 2 +- synapse/config/experimental.py | 8 -------- synapse/config/server.py | 5 +++++ synapse/handlers/profile.py | 2 +- synapse/handlers/sync.py | 9 +++++---- synapse/rest/client/sync.py | 2 +- synapse/rest/client/versions.py | 2 +- tests/handlers/test_profile.py | 8 ++++---- tests/handlers/test_sync.py | 16 ++++++++-------- 11 files changed, 30 insertions(+), 32 deletions(-) diff --git a/changelog.d/19556.feature b/changelog.d/19556.feature index 6422a6160d3..bcb6c5c983c 100644 --- a/changelog.d/19556.feature +++ b/changelog.d/19556.feature @@ -1,2 +1,2 @@ -Implement experimental support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). -Currently limited to local users only for the sync results. \ No newline at end of file +Add optional support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). +Currently defaults to not enabled, and is limited to local users only for the sync results. \ No newline at end of file diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index b596b58daa6..99ba9e622a5 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -15,6 +15,8 @@ enable_registration_without_verification: true bcrypt_rounds: 4 url_preview_enabled: true url_preview_ip_range_blacklist: [] +# MSC4299 Profile updates down legacy /sync +include_profile_updates_in_sync: true ## Registration ## @@ -141,8 +143,6 @@ experimental_features: msc4354_enabled: true # `/sync` `state_after` msc4222_enabled: true - # Profile updates down legacy /sync - msc4429_enabled: true server_notices: system_mxid_localpart: _server diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 3f4d6d40b7a..ebb8ce5b543 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -228,7 +228,7 @@ def __init__(self, hs: "HomeServer", filter_json: JsonMapping): self.event_format = filter_json.get("event_format", "client") self.profile_fields: set[str] = set() - if hs.config.experimental.msc4429_enabled: + if hs.config.server.include_profile_updates_in_sync: profile_fields_filter = filter_json.get("org.matrix.msc4429.profile_fields") if isinstance(profile_fields_filter, Mapping): diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 5f7592385f2..120b31cee58 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -564,14 +564,6 @@ def read_config( # MSC4133: Custom profile fields self.msc4133_enabled: bool = experimental.get("msc4133_enabled", False) - # MSC4429: Profile updates for legacy /sync - self.msc4429_enabled: bool = bool( - experimental.get("msc4429_enabled", False) - or - # Allow an alias outside the "experimental" section. - config.get("msc4429_enabled", False) - ) - # MSC4143: Matrix RTC Transport using Livekit Backend self.msc4143_enabled: bool = experimental.get("msc4143_enabled", False) diff --git a/synapse/config/server.py b/synapse/config/server.py index ca94c224ea5..99acc7d3f5e 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -545,6 +545,11 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: " 'allow_public_rooms_over_federation' is set." ) + # Whether to support MSC4299 profile updates down legacy /sync + self.include_profile_updates_in_sync = config.get( + "include_profile_updates_in_sync", False, + ) + # Check if the legacy "restrict_public_rooms_to_local_users" flag is set. This # flag is now obsolete but we need to check it for backward-compatibility. if config.get("restrict_public_rooms_to_local_users", False): diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 70ae1bcca84..da146ecfe50 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -78,7 +78,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.hs = hs self._notifier = hs.get_notifier() - self._msc4429_enabled = hs.config.experimental.msc4429_enabled + self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync self.federation = hs.get_federation_client() hs.get_federation_registry().register_query_handler( diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f3169bb863d..b30fc2adb7f 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1899,10 +1899,11 @@ async def generate_sync_result( } ) - # Note, this needs to be after we collect `joined` sync results - # since we want to utilize the work we did to collect users into the - # lazy loading members cache - if self.hs_config.experimental.msc4429_enabled: + # Note, this needs to be after we collect `joined`, `invited`, `knocked` and + # `archived` sync results since we want to utilize the work we did to collect + # events in those responses as a basis for which users to include profiles + # for when lazy loading. + if self.hs_config.server.include_profile_updates_in_sync: await self._generate_sync_entry_for_profile_updates(sync_result_builder) logger.debug("Sync response calculation complete") diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index ecaa015a1c6..042f3091e09 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -124,7 +124,7 @@ def __init__(self, hs: "HomeServer"): self._event_serializer = hs.get_event_client_serializer() self._msc2654_enabled = hs.config.experimental.msc2654_enabled self._msc3773_enabled = hs.config.experimental.msc3773_enabled - self._msc4429_enabled = hs.config.experimental.msc4429_enabled + self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync self._json_filter_cache: LruCache[str, bool] = LruCache( max_size=1000, diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 962e208bac7..9956f4a4896 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -197,7 +197,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: "uk.tcpip.msc4133": self.config.experimental.msc4133_enabled, "uk.tcpip.msc4133.stable": True, # MSC4429: Profile updates for legacy /sync. - "org.matrix.msc4429": self.config.experimental.msc4429_enabled, + "org.matrix.msc4429": self.config.server.include_profile_updates_in_sync, # MSC4155: Invite filtering "org.matrix.msc4155": self.config.experimental.msc4155_enabled, # MSC4306: Support for thread subscriptions diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index ab9893cbb13..ca21ff3c9e7 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -229,7 +229,7 @@ def test_update_profile_does_not_notify_notifier_on_set_field_if_msc4429_not_ena ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], ] ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_update_profile_does_not_notify_notifier_on_set_field_if_user_not_in_rooms( self, field_name: str, new_value: str ) -> None: @@ -255,7 +255,7 @@ def test_update_profile_does_not_notify_notifier_on_set_field_if_user_not_in_roo ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], ] ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_update_profile_updates_stream_on_set_field( self, field_name: str, new_value: str ) -> None: @@ -316,7 +316,7 @@ def test_update_profile_updates_stream_on_set_field( (3, "@1234abcd:test", ProfileUpdateAction.UPDATE.value, field_name), ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( self, ) -> None: @@ -415,7 +415,7 @@ def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], ] ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_update_profile_notifies_notifier_on_set_field( self, field_name: str, diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 4ee74a47210..b69d16295c7 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1199,7 +1199,7 @@ def test_initial_sync_no_profile_updates_if_not_enabled(self) -> None: ) self.assertEqual(initial_result.profile_updates, {}) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: requester = create_requester(self.user) initial_result = self.get_success( @@ -1216,7 +1216,7 @@ def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: {}, ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: self.get_success( self.profile_handler.set_field( @@ -1257,7 +1257,7 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: ], ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( self, ) -> None: @@ -1332,7 +1332,7 @@ def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( ], ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_sends_down_profile_updates( self, ) -> None: @@ -1390,7 +1390,7 @@ def test_incremental_sync_sends_down_profile_updates( incremental_result.profile_updates["@other_user:test"].get("displayname"), ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( self, ) -> None: @@ -1496,7 +1496,7 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( "third_user", ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles( self, ) -> None: @@ -1587,7 +1587,7 @@ def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles( [], ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms( self, ) -> None: @@ -1634,7 +1634,7 @@ def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_room incremental_result.profile_updates["@other_user:test"], ) - @override_config({"experimental_features": {"msc4429_enabled": True}}) + @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_sends_down_full_profile_for_users_who_have_joined( self, ) -> None: From 4a874c4687c2bf6a7e87f51291b46f3874ce6234 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 26 Jun 2026 18:09:02 +0300 Subject: [PATCH 078/106] Remove invalid docs on `profile_updates` stream `PUT` and `DELETE` can go to any supported worker, replication will handle the stream updates to the right worker. --- docs/workers.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/workers.md b/docs/workers.md index 160e13df7ff..6259e3ac7f1 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -585,14 +585,6 @@ configured as stream writer for the `quarantined_media_changes` stream: ^/_synapse/admin/v1/quarantine_media/.*$ -#### The `profile_updates` stream - -The `profile_updates` stream supports multiple writers. The following endpoints -can be handled by any worker, but PUT and DELETE should be routed directly to one of the -workers configured as stream writer for the `profile_updates` stream: - - ^/_matrix/client/(api/v1|r0|v3|unstable)/profile/.*/[^/]+$ - #### Restrict outbound federation traffic to a specific set of workers From 088f23065adc3fa642923e0c5bf711e526461290 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 26 Jun 2026 18:10:25 +0300 Subject: [PATCH 079/106] Gate more things behind msc4429 being enabled --- synapse/handlers/profile.py | 7 +++ synapse/handlers/room_member.py | 67 ++++++++++++++++------------- synapse/replication/http/profile.py | 5 ++- 3 files changed, 46 insertions(+), 33 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index da146ecfe50..4acf5c598c5 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -452,6 +452,8 @@ async def user_left_room(self, user_id: UserID, room_id: str) -> None: If that is the case, """ + if not self._msc4429_enabled: + return user_id_str = user_id.to_string() users_in_left_room = set(await self.store.get_local_users_in_room(room_id)) @@ -489,6 +491,8 @@ async def user_joined_room(self, user_id: UserID, room_id: str) -> None: If that is the case, """ + if not self._msc4429_enabled: + return user_id_str = user_id.to_string() users_in_room = set(await self.store.get_local_users_in_room(room_id)) @@ -582,6 +586,9 @@ async def _dispatch_record_profile_updates( Returns: None """ + if not self._msc4429_enabled: + return + if self._is_profile_worker: await self.record_profile_updates( user_id, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 956e020a463..e34fc79d4ac 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -206,6 +206,7 @@ def __init__(self, hs: "HomeServer"): ) self._push_writer = hs.config.worker.writers.push_rules[0] self._copy_push_client = ReplicationCopyPusherRestServlet.make_client(hs) + self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync self._is_profile_worker = ( hs.get_instance_name() in hs.config.worker.writers.profile_updates ) @@ -536,23 +537,24 @@ async def _local_membership_update( ) if prev_member_event.membership == Membership.JOIN: await self._user_left_room(target, room_id) - # Notify the profile handler. We only want to do this once - # in a multi-worker setup, so we can't listen on the dispatched - # event above. - if self._is_profile_worker: - await self.profile_handler.user_left_room( - target, room_id - ) - else: - # Offload to the right worker via http replication - await self._profile_user_room_membership_change_client( - instance_name=self._profile_updates_writer_instance, - user_id=target.to_string(), - room_id=room_id, - membership=Membership.LEAVE, - ) - - elif event.membership == Membership.JOIN: + if self._msc4429_enabled: + # Notify the profile handler. We only want to do this once + # in a multi-worker setup, so we can't listen on the dispatched + # event above. + if self._is_profile_worker: + await self.profile_handler.user_left_room( + target, room_id + ) + else: + # Offload to the right worker via http replication + await self._profile_user_room_membership_change_client( + instance_name=self._profile_updates_writer_instance, + user_id=target.to_string(), + room_id=room_id, + membership=Membership.LEAVE, + ) + + elif self._msc4429_enabled and event.membership == Membership.JOIN: # Notify the profile handler. We only want to do this once # in a multi-worker setup, so we can't dispatch a hook to all workers. if self._is_profile_worker: @@ -1580,20 +1582,23 @@ async def send_membership_event( prev_member_event = await self.store.get_event(prev_member_event_id) if prev_member_event.membership == Membership.JOIN: await self._user_left_room(target_user, room_id) - # Notify the profile handler. We only want to do this once - # in a multi-worker setup, so we can't listen on the dispatched - # event above. - if self._is_profile_worker: - await self.profile_handler.user_left_room(target_user, room_id) - else: - # Offload to the right worker via http replication - await self._profile_user_room_membership_change_client( - instance_name=self._profile_updates_writer_instance, - user_id=target_user.to_string(), - room_id=room_id, - membership=Membership.LEAVE, - ) - elif event.membership == Membership.JOIN: + if self._msc4429_enabled: + # Notify the profile handler. We only want to do this once + # in a multi-worker setup, so we can't listen on the dispatched + # event above. + if self._is_profile_worker: + await self.profile_handler.user_left_room( + target_user, room_id + ) + else: + # Offload to the right worker via http replication + await self._profile_user_room_membership_change_client( + instance_name=self._profile_updates_writer_instance, + user_id=target_user.to_string(), + room_id=room_id, + membership=Membership.LEAVE, + ) + elif self._msc4429_enabled and event.membership == Membership.JOIN: # Notify the profile handler. We only want to do this once # in a multi-worker setup, so we can't dispatch a hook to all workers. if self._is_profile_worker: diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py index 8b27b088733..2ebae2c0078 100644 --- a/synapse/replication/http/profile.py +++ b/synapse/replication/http/profile.py @@ -135,5 +135,6 @@ async def _handle_request( # type: ignore[override] def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - ReplicationProfileUserRoomMembershipChange(hs).register(http_server) - ReplicationProfileRecordFieldUpdates(hs).register(http_server) + if hs.config.server.include_profile_updates_in_sync: + ReplicationProfileUserRoomMembershipChange(hs).register(http_server) + ReplicationProfileRecordFieldUpdates(hs).register(http_server) From c12623e769eccca8ea670c344942cab6c7ec61d5 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 26 Jun 2026 18:15:30 +0300 Subject: [PATCH 080/106] Lint server config --- synapse/config/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index 99acc7d3f5e..ffd4ef5ab89 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -547,7 +547,8 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # Whether to support MSC4299 profile updates down legacy /sync self.include_profile_updates_in_sync = config.get( - "include_profile_updates_in_sync", False, + "include_profile_updates_in_sync", + False, ) # Check if the legacy "restrict_public_rooms_to_local_users" flag is set. This From 1fc8d4c6cd352ce26979afe5d631ae620f659591 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 29 Jun 2026 12:39:46 +0300 Subject: [PATCH 081/106] Move profile updates SQL to latest delta We shouldn't need a delta bump for this feature as far as I can tell. --- .../05_profile_updates.sql} | 14 +++++++------- .../05_profile_updates_seq.sql.postgres} | 0 2 files changed, 7 insertions(+), 7 deletions(-) rename synapse/storage/schema/main/delta/{95/01_profile_updates.sql => 94/05_profile_updates.sql} (71%) rename synapse/storage/schema/main/delta/{95/01_profile_updates_seq.sql.postgres => 94/05_profile_updates_seq.sql.postgres} (100%) diff --git a/synapse/storage/schema/main/delta/95/01_profile_updates.sql b/synapse/storage/schema/main/delta/94/05_profile_updates.sql similarity index 71% rename from synapse/storage/schema/main/delta/95/01_profile_updates.sql rename to synapse/storage/schema/main/delta/94/05_profile_updates.sql index 3e56300e0fc..3d2653648bb 100644 --- a/synapse/storage/schema/main/delta/95/01_profile_updates.sql +++ b/synapse/storage/schema/main/delta/94/05_profile_updates.sql @@ -12,7 +12,7 @@ -- . -- Track updates to profile fields for MSC4429 legacy /sync. -CREATE TABLE profile_updates ( +CREATE TABLE IF NOT EXISTS profile_updates ( stream_id BIGINT NOT NULL PRIMARY KEY, instance_name TEXT NOT NULL, @@ -32,12 +32,12 @@ CREATE TABLE profile_updates ( inserted_ts BIGINT NOT NULL ); -CREATE INDEX profile_updates_by_user ON profile_updates (user_id, stream_id); -CREATE INDEX profile_updates_by_field ON profile_updates (field_name, stream_id); -CREATE INDEX profile_updates_inserted_ts ON profile_updates (inserted_ts); +CREATE INDEX IF NOT EXISTS profile_updates_by_user ON profile_updates (user_id, stream_id); +CREATE INDEX IF NOT EXISTS profile_updates_by_field ON profile_updates (field_name, stream_id); +CREATE INDEX IF NOT EXISTS profile_updates_inserted_ts ON profile_updates (inserted_ts); -- Track which local users should receive each profile update. -CREATE TABLE profile_updates_per_user ( +CREATE TABLE IF NOT EXISTS profile_updates_per_user ( stream_id BIGINT NOT NULL REFERENCES profile_updates (stream_id), -- The full user ID of the local user that should receive the profile update. @@ -48,5 +48,5 @@ CREATE TABLE profile_updates_per_user ( inserted_ts BIGINT NOT NULL ); -CREATE INDEX profile_updates_per_user_by_user_stream ON profile_updates_per_user (user_id, stream_id); -CREATE INDEX profile_updates_per_user_inserted_ts ON profile_updates_per_user (inserted_ts); +CREATE INDEX IF NOT EXISTS profile_updates_per_user_by_user_stream ON profile_updates_per_user (user_id, stream_id); +CREATE INDEX IF NOT EXISTS profile_updates_per_user_inserted_ts ON profile_updates_per_user (inserted_ts); diff --git a/synapse/storage/schema/main/delta/95/01_profile_updates_seq.sql.postgres b/synapse/storage/schema/main/delta/94/05_profile_updates_seq.sql.postgres similarity index 100% rename from synapse/storage/schema/main/delta/95/01_profile_updates_seq.sql.postgres rename to synapse/storage/schema/main/delta/94/05_profile_updates_seq.sql.postgres From bf3e1339fff667cf4cb690fd00e7b4844d13d2e2 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 29 Jun 2026 13:01:03 +0300 Subject: [PATCH 082/106] Fix `SCHEMA_VERSION` in __init__.py --- synapse/storage/schema/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 8e3b13c5315..3495dce866a 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -19,7 +19,7 @@ # # -SCHEMA_VERSION = 95 # remember to update the list below when updating +SCHEMA_VERSION = 94 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -175,7 +175,6 @@ Changes in SCHEMA_VERSION = 94 - Add `recheck` column (boolean, default true) to the `redactions` table. - MSC4242: Add state DAG tables. -Changes in SCHEMA_VERSION = 95 - MSC4429: Track updates to user profile fields via a new stream. """ From 60ed1758dc8662a6b8175ae965f4259b0824d68c Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 29 Jun 2026 16:00:00 +0300 Subject: [PATCH 083/106] When user has left all rooms shared with a user, clear any previous profile updates This ensures we don't accidentally leak any profile updates to users no longer sharing rooms. --- synapse/handlers/profile.py | 11 ++- synapse/storage/databases/main/profile.py | 64 +++++++++++++ tests/handlers/test_profile.py | 108 ++++++++++++++++++++++ 3 files changed, 182 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 4acf5c598c5..5ea99648ef6 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -446,6 +446,9 @@ async def user_left_room(self, user_id: UserID, room_id: str) -> None: * Add a row to `profile_updates` stating that the user left a room. * Check if this user no longer shares any rooms with certain users. * Insert a row for each of those users into `profile_updates_per_user`. + * Remove any previous profile update stream rows concerning + this user. This is done to stop leaking any updates to users who no longer + share a room. * Now, when any of those users sync, the sync code will check `profile_updates` and see that the user left a room. And thus a "clear this user's profile" instruction will be sent down to the client. @@ -467,12 +470,18 @@ async def user_left_room(self, user_id: UserID, room_id: str) -> None: users_to_update = users_in_left_room - users_still_sharing_rooms if users_to_update: + # First clear any old profile updates for these users + await self.store.clear_profile_updates_for_user( + user_id=user_id, + users_to_remove=users_to_update, + ) + + # Record our leave stream_id = await self.store.add_profile_updates( user_id=user_id, action=ProfileUpdateAction.LEFT_ROOM.value, updated_fields=None, ) - await self.store.track_profile_updates_per_user( stream_id=stream_id, user_ids=users_to_update, diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index f1141f90fba..1475ee994ca 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -955,6 +955,70 @@ async def delete_profile(self, user_id: UserID) -> None: keyvalues={"full_user_id": user_id.to_string()}, ) + async def clear_profile_updates_for_user( + self, user_id: UserID, users_to_remove: set[str] + ) -> None: + """ + Clear all the ProfileUpdateAction.UPDATE rows from the + `profile_updates_per_user` table from a particular user for + a list of target users. + + This does not remove the stream ID row from `profile_updates` as it is + likely other per user rows may refer to it. Our automatic pruning of old + stream ID's will kick in later and clean up potential orphan `profile_updates` + table rows. + + Args: + user_id: The user's ID. + users_to_remove: List of users to remove per user rows for. + + Returns: + None + """ + assert self._can_write_to_profile_updates + if not users_to_remove: + return + + def _clear_profile_updates_for_user_txn( + txn: LoggingTransaction, + ) -> None: + # Delete profile updates where there's a corresponding row in + # `profile_updates_per_user`. + sql = """ + SELECT stream_id FROM profile_updates + WHERE user_id = ? AND action = ? + """ + + txn.execute(sql, (user_id.to_string(), ProfileUpdateAction.UPDATE.value)) + res = txn.fetchall() + if not res: + return + + stream_ids = [row[0] for row in res] + + user_clause, user_args = make_in_list_sql_clause( + txn.database_engine, + "user_id", + users_to_remove, + ) + stream_id_clause, stream_id_args = make_in_list_sql_clause( + txn.database_engine, + "stream_id", + stream_ids, + ) + sql = f""" + DELETE FROM profile_updates_per_user + WHERE {user_clause} + AND {stream_id_clause} + """ + params = user_args + stream_id_args + txn.execute(sql, (*params,)) + + await self.db_pool.runInteraction( + "clear_profile_updates_for_user", + _clear_profile_updates_for_user_txn, + ) + @wrap_as_background_process("prune_profile_updates") async def _prune_profile_updates(self) -> None: """Delete old entries out of the `profile_updates` and diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index ca21ff3c9e7..ae59d394cb1 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -408,6 +408,114 @@ def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( ], ) + @override_config({"include_profile_updates_in_sync": True}) + def test_previous_profile_updates_stream_rows_cleared_if_no_longer_sharing_a_room( + self, + ) -> None: + self.register_user("roger", "password") + roger_token = self.login("roger", "password") + self.register_user("millie", "password") + millie_token = self.login("millie", "password") + room_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + room_with_millie_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + self.helper.join(room_id, "@roger:test", tok=roger_token) + self.helper.join(room_with_millie_id, "@millie:test", tok=millie_token) + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="m.status", + new_value='{"text": "Holiday"}', + ) + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@roger:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=4, + user_id=self.frank.to_string(), + action="update", + field_name="m.status", + ), + ], + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@millie:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=4, + user_id=self.frank.to_string(), + action="update", + field_name="m.status", + ), + ], + ) + + # Leave room and verify only the "left room" exists for roger + self.helper.leave(room_id, self.frank.to_string(), tok=self.frank_token) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@roger:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=5, + user_id=self.frank.to_string(), + action="left_room", + field_name=None, + ), + ], + ) + + # Sanity check we didn't clear any rows for millie + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@millie:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=4, + user_id=self.frank.to_string(), + action="update", + field_name="m.status", + ), + ], + ) + @parameterized.expand( [ ["displayname", "Frank"], From 2377165ebdca0be021b1e5f6f076e9db569295b9 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 29 Jun 2026 16:33:36 +0300 Subject: [PATCH 084/106] Add test to prove initial sync doesn't include users updates who don't share a room --- tests/handlers/test_sync.py | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index b69d16295c7..cef15b46562 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1257,6 +1257,53 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: ], ) + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_initial_sync_does_not_include_untracked_users_profile_updates( + self, is_lazy: bool + ) -> None: + third_user = self.register_user("third_user", "password") + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + ) + ) + + requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertIsNone(initial_result.profile_updates.get(third_user)) + @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( self, From 496728df15c62d39103224517948249fb83cfee0 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 29 Jun 2026 21:48:11 +0300 Subject: [PATCH 085/106] Add docstrings to tests Plus make a few tests actually do something sane.. --- tests/handlers/test_profile.py | 19 ++++++++++++++ tests/handlers/test_sync.py | 45 +++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index ae59d394cb1..82c1e7c4350 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -177,6 +177,8 @@ def test_update_profile_does_not_update_stream_on_set_field_if_msc4429_not_enabl field_name: str, new_value: str, ) -> None: + """Test that profile updates don't get recorded in the profile updates stream + if MSC4429 is not enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -206,6 +208,8 @@ def test_update_profile_does_not_notify_notifier_on_set_field_if_msc4429_not_ena field_name: str, new_value: str, ) -> None: + """Test that profile updates do not cause the profile updates stream notifier + to wake up if MSC4429 is not enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -233,6 +237,8 @@ def test_update_profile_does_not_notify_notifier_on_set_field_if_msc4429_not_ena def test_update_profile_does_not_notify_notifier_on_set_field_if_user_not_in_rooms( self, field_name: str, new_value: str ) -> None: + """Test that profile updates do not cause the profile updates stream notifier + to wake up if the user is not in any rooms, if MSC4429 is enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -259,6 +265,8 @@ def test_update_profile_does_not_notify_notifier_on_set_field_if_user_not_in_roo def test_update_profile_updates_stream_on_set_field( self, field_name: str, new_value: str ) -> None: + """Test that profile updates get recorded in the profile updates stream if + MSC4429 is enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -320,6 +328,8 @@ def test_update_profile_updates_stream_on_set_field( def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( self, ) -> None: + """Test that profiles updates get recorded in the 'per user' profile updates + stream tracking table, if MSC4429 is enabled.""" self.register_user("roger", "password") roger_token = self.login("roger", "password") self.register_user("millie", "password") @@ -412,6 +422,13 @@ def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( def test_previous_profile_updates_stream_rows_cleared_if_no_longer_sharing_a_room( self, ) -> None: + """Test that previous profile update stream rows are removed for a user if + the user no longer shares rooms with another user, if MSC4429 is enabled. + + This test ensures that when a user leaves a room, we clear all old profile + update rows of users who the user no longer shares rooms with, to avoid + leaking any further profile field updates from those users. + """ self.register_user("roger", "password") roger_token = self.login("roger", "password") self.register_user("millie", "password") @@ -529,6 +546,8 @@ def test_update_profile_notifies_notifier_on_set_field( field_name: str, new_value: str, ) -> None: + """Test that profile updates wake up the profile updates stream on profile + field updates, if MSC4429 is enabled.""" self.helper.create_room_as( room_creator=self.frank.to_string(), tok=self.frank_token, diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index cef15b46562..f72cf54956a 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1187,6 +1187,17 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: ) def test_initial_sync_no_profile_updates_if_not_enabled(self) -> None: + """Test that without MSC4429 enabled the initial sync response does not + contain any profile updates.""" + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + ) + ) + requester = create_requester(self.user) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( @@ -1201,6 +1212,17 @@ def test_initial_sync_no_profile_updates_if_not_enabled(self) -> None: @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: + """Test that with MSC4429 enabled the initial sync response does not + contain any profile updates, if fields are not filtered for.""" + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + ) + ) + requester = create_requester(self.user) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( @@ -1218,6 +1240,8 @@ def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: + """Test that with MSC4429 enabled the initial sync response does + contain profile updates for users who share rooms.""" self.get_success( self.profile_handler.set_field( target_user=UserID.from_string(self.other_user), @@ -1267,6 +1291,8 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: def test_initial_sync_does_not_include_untracked_users_profile_updates( self, is_lazy: bool ) -> None: + """Test that with MSC4429 enabled the initial sync response does not + contain profile updates for users who do not share rooms.""" third_user = self.register_user("third_user", "password") self.get_success( self.profile_handler.set_field( @@ -1308,7 +1334,9 @@ def test_initial_sync_does_not_include_untracked_users_profile_updates( def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( self, ) -> None: - """ + """Test that with MSC4429 enabled the initial sync lazy loading response does + contain profile updates for events in the timeline. + This test ensures lazy loading sync only returns profiles that we also have events for in the sync response. The second room in this test has the most recent events from "third_user" and thus we don't get the profile of @@ -1383,6 +1411,8 @@ def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( def test_incremental_sync_sends_down_profile_updates( self, ) -> None: + """Test that with MSC4429 enabled the incremental sync response does + contain profile updates.""" requester = create_requester(self.user) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( @@ -1441,6 +1471,10 @@ def test_incremental_sync_sends_down_profile_updates( def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( self, ) -> None: + """Test that with MSC4429 enabled the incremental sync lazy loading response + does contain profile updates even if the user would be filtered out by lazy + loading. + """ third_user = self.register_user("third_user", "password") third_tok = self.login("third_user", "password") self.helper.join( @@ -1547,6 +1581,9 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles( self, ) -> None: + """Test that with MSC4429 enabled the incremental sync lazy loading response + filters out unchanged profiles we have recently sent to the client. + """ requester = create_requester(self.user) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( @@ -1638,6 +1675,9 @@ def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles( def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms( self, ) -> None: + """Test that with MSC4429 enabled the incremental sync response + includes a 'null' for users who are no longer sharing rooms. + """ requester = create_requester(self.user) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( @@ -1685,6 +1725,9 @@ def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_room def test_incremental_sync_sends_down_full_profile_for_users_who_have_joined( self, ) -> None: + """Test that with MSC4429 enabled the incremental sync response + includes the full profile of a user who has joined a room with the syncing user. + """ requester = create_requester(self.user) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( From 5064349eecbb206657d1b91ae8529697f4a90c6c Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 29 Jun 2026 21:56:58 +0300 Subject: [PATCH 086/106] Don't delete custom profile field when setting an empty string --- synapse/handlers/profile.py | 24 +++++++----------------- synapse/rest/client/profile.py | 24 ++++++++++++++++-------- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 5ea99648ef6..f27c0a475da 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -762,23 +762,13 @@ async def set_field( propagate=propagate, ) else: - # For custom fields, we need to call a separate delete method - # for empty strings. - if new_value == "": - await self.delete_profile_field( - target_user=target_user, - requester=requester, - field_name=field_name, - by_admin=by_admin, - ) - else: - await self.set_profile_field( - target_user=target_user, - requester=requester, - field_name=field_name, - new_value=new_value, - by_admin=by_admin, - ) + await self.set_profile_field( + target_user=target_user, + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=by_admin, + ) async def set_profile_field( self, diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index daab2c66518..93a5b102d11 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -262,14 +262,22 @@ async def on_DELETE( Codes.USER_ACCOUNT_SUSPENDED, ) - await self.profile_handler.set_field( - target_user=user, - requester=requester, - field_name=field_name, - new_value="", - by_admin=is_admin, - propagate=propagate, - ) + if field_name in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL): + await self.profile_handler.set_field( + target_user=user, + requester=requester, + field_name=field_name, + new_value="", + by_admin=is_admin, + propagate=propagate, + ) + else: + await self.profile_handler.delete_profile_field( + target_user=user, + requester=requester, + field_name=field_name, + by_admin=is_admin, + ) return 200, {} From eb5aa2129a8ec67476e74fc0e28b5f19dc2476c4 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 11:22:57 +0300 Subject: [PATCH 087/106] Ensure both initial and incremental syncs include our own profile and updates Otherwise if the user has multiple clients doing incremental sync, they wont get their own profile updates to their other clients. For initial because it should probably match the incremental behaviour + it avoids clients needing to look up the profile separately potentially. --- synapse/handlers/profile.py | 11 ++-- synapse/handlers/sync.py | 5 -- synapse/storage/databases/main/profile.py | 2 +- synapse/storage/databases/main/roommember.py | 5 +- tests/handlers/test_profile.py | 6 ++ tests/handlers/test_sync.py | 68 +++++++++++++++++++- 6 files changed, 82 insertions(+), 15 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index f27c0a475da..05e59c18187 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -142,13 +142,10 @@ async def record_profile_updates( user_id.to_string() ) ) - # Remove ourselves from the user ID list - users_who_share_rooms.remove(user_id.to_string()) - if users_who_share_rooms: - await self.store.track_profile_updates_per_user( - stream_id=stream_id, - user_ids=users_who_share_rooms, - ) + await self.store.track_profile_updates_per_user( + stream_id=stream_id, + user_ids=users_who_share_rooms, + ) self._notifier.on_new_event( StreamKeyType.PROFILE_UPDATES, stream_id, rooms=room_ids diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b30fc2adb7f..17f0467deb3 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2203,11 +2203,6 @@ async def _generate_initial_sync_entry_for_profile_updates( # Filter down to selected included users user_ids = {user_id for user_id in user_ids if user_id in include_users} - # Remove ourselves - # FIXME?: Should `get_local_users_who_share_room_with_user` even return - # ourselves? - user_ids.discard(user_id) - if not user_ids: return diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 1475ee994ca..b8a35314fda 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -670,7 +670,7 @@ async def track_profile_updates_per_user( ) -> None: """ Create tracking rows for profile updater per target user interested in profile - updates for the user triggering one. + updates for the user triggering one, including themselves. Args: stream_id: Stream ID referencing a `profile_updates` stream ID. diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index a645e4f3a59..8c69266d510 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -983,7 +983,10 @@ async def get_users_who_share_room_with_user(self, user_id: str) -> set[str]: return user_who_share_room async def get_local_users_who_share_room_with_user(self, user_id: str) -> set[str]: - """Returns the set of local users who share a room with `user_id`""" + """Returns the set of local users who share a room with `user_id`. + + This also includes the `user_id` themselves. + """ room_ids = await self.get_rooms_for_user(user_id) user_who_share_room: set[str] = set() diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 82c1e7c4350..0bf940d24b7 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -415,6 +415,12 @@ def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( action="joined_room", field_name=None, ), + ProfileUpdate( + stream_id=4, + user_id=self.frank.to_string(), + action="update", + field_name="m.status", + ), ], ) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index f72cf54956a..61cf0b02374 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1241,7 +1241,8 @@ def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: """Test that with MSC4429 enabled the initial sync response does - contain profile updates for users who share rooms.""" + contain profile updates for users who share rooms, including our + syncing user.""" self.get_success( self.profile_handler.set_field( target_user=UserID.from_string(self.other_user), @@ -1269,6 +1270,7 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: request_key=generate_request_key(), ) ) + assert initial_result.profile_updates[self.user] is not None assert initial_result.profile_updates["@other_user:test"] is not None self.assertEqual( initial_result.profile_updates["@other_user:test"]["m.status"], @@ -1277,6 +1279,7 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: self.assertCountEqual( initial_result.profile_updates.keys(), [ + self.user, "@other_user:test", ], ) @@ -1804,6 +1807,69 @@ def test_incremental_sync_sends_down_full_profile_for_users_who_have_joined( incremental_result.profile_updates["@third_user:test"]["avatar_url"], ) + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_includes_own_profile_updates(self, is_lazy: bool) -> None: + """Test that with MSC4429 enabled the incremental sync response includes + ones own profile updates.""" + requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.user), + requester=requester, + field_name="m.status", + new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + assert incremental_result.profile_updates["@user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@user:test"]["m.status"], + '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', + ) + class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): """Tests Sync Handler state behavior when using `use_state_after.""" From ecd890a7afcd86e61d9af67bcbc14fd12bf0674b Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 16:01:40 +0300 Subject: [PATCH 088/106] Fix MSC4429 referring comment Co-authored-by: Olivier 'reivilibre' --- docker/complement/conf/workers-shared-extra.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 99ba9e622a5..64a36522fa4 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -15,7 +15,7 @@ enable_registration_without_verification: true bcrypt_rounds: 4 url_preview_enabled: true url_preview_ip_range_blacklist: [] -# MSC4299 Profile updates down legacy /sync +# MSC4429 Profile updates down legacy /sync include_profile_updates_in_sync: true ## Registration ## From 6a5adb9c04740f9d8d65c4dfdba49fa66892b98c Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 17:23:22 +0300 Subject: [PATCH 089/106] Remove extra `ids` definition Co-authored-by: Olivier 'reivilibre' --- synapse/api/filtering.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index ebb8ce5b543..cbae9133c66 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -232,8 +232,7 @@ def __init__(self, hs: "HomeServer", filter_json: JsonMapping): profile_fields_filter = filter_json.get("org.matrix.msc4429.profile_fields") if isinstance(profile_fields_filter, Mapping): - ids = profile_fields_filter.get("ids", []) - self.profile_fields = set(ids) + self.profile_fields = set(profile_fields_filter.get("ids", [])) def __repr__(self) -> str: return "" % (json.dumps(self._filter_json),) From 45e9a018af1d94e149357b77e1bea5125404780b Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 17:32:08 +0300 Subject: [PATCH 090/106] Don't unpact `ProfileUpdateAction` unnecessarily early --- synapse/handlers/profile.py | 6 +++--- synapse/storage/databases/main/profile.py | 11 +++++------ tests/storage/test_profile.py | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 05e59c18187..3b857d6b12e 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -131,7 +131,7 @@ async def record_profile_updates( stream_id = await self.store.add_profile_updates( user_id=user_id, updated_fields=updated_fields, - action=ProfileUpdateAction.UPDATE.value, + action=ProfileUpdateAction.UPDATE, ) room_ids = await self.store.get_rooms_for_user(user_id.to_string()) if not room_ids: @@ -476,7 +476,7 @@ async def user_left_room(self, user_id: UserID, room_id: str) -> None: # Record our leave stream_id = await self.store.add_profile_updates( user_id=user_id, - action=ProfileUpdateAction.LEFT_ROOM.value, + action=ProfileUpdateAction.LEFT_ROOM, updated_fields=None, ) await self.store.track_profile_updates_per_user( @@ -508,7 +508,7 @@ async def user_joined_room(self, user_id: UserID, room_id: str) -> None: stream_id = await self.store.add_profile_updates( user_id=user_id, - action=ProfileUpdateAction.JOINED_ROOM.value, + action=ProfileUpdateAction.JOINED_ROOM, updated_fields=None, ) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index b8a35314fda..136594c0724 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -599,16 +599,15 @@ async def get_profile_data_for_users( async def add_profile_updates( self, user_id: UserID, - action: str, + action: ProfileUpdateAction, updated_fields: set[str] | None, ) -> int: """Persist profile update markers and return the last stream ID.""" assert self._can_write_to_profile_updates - assert action in [action.value for action in ProfileUpdateAction] - if action == ProfileUpdateAction.UPDATE.value and not updated_fields: + if action == ProfileUpdateAction.UPDATE and not updated_fields: return self._profile_updates_id_gen.get_current_token() - elif action == ProfileUpdateAction.LEFT_ROOM.value: + elif action == ProfileUpdateAction.LEFT_ROOM: assert not updated_fields user_id_str = user_id.to_string() @@ -626,7 +625,7 @@ def _add_profile_updates_txn(txn: LoggingTransaction) -> int: stream_id, self._instance_name, user_id_str, - action, + action.value, field_name, inserted_ts, ] @@ -638,7 +637,7 @@ def _add_profile_updates_txn(txn: LoggingTransaction) -> int: stream_ids[0], self._instance_name, user_id_str, - action, + action.value, None, inserted_ts, ] diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index f672c39a972..6f96c1c4b05 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -175,7 +175,7 @@ def get_profile_updates_per_user_status() -> tuple[int]: self.store.add_profile_updates( user_id=UserID.from_string("@user:test"), updated_fields={next(field_name_gen)}, - action=ProfileUpdateAction.UPDATE.value, + action=ProfileUpdateAction.UPDATE, ) ) self.get_success( @@ -203,7 +203,7 @@ def get_profile_updates_per_user_status() -> tuple[int]: self.store.add_profile_updates( user_id=UserID.from_string("@user:test"), updated_fields={next(field_name_gen)}, - action=ProfileUpdateAction.UPDATE.value, + action=ProfileUpdateAction.UPDATE, ) ) self.get_success( From 6730ad061022a34fc20a39497ceece1241f8bc93 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 17:35:08 +0300 Subject: [PATCH 091/106] Add `str` to ProfileUpdateAction types --- synapse/api/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 952beb02571..ffc5b831b33 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -410,7 +410,7 @@ class ProfileFields: AVATAR_URL: Final = "avatar_url" -class ProfileUpdateAction(enum.Enum): +class ProfileUpdateAction(str, enum.Enum): """ Enum representing the action of a row in the profile updates stream tables. """ From 28563f0921ddbcd27bb79f6bb6d78ce3e1b92ebd Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 18:17:09 +0300 Subject: [PATCH 092/106] Make `ProfileUpdateAction` docstrings more descriptive on what the different enum values mean. --- synapse/api/constants.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index ffc5b831b33..a5ee617344b 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -413,19 +413,50 @@ class ProfileFields: class ProfileUpdateAction(str, enum.Enum): """ Enum representing the action of a row in the profile updates stream tables. + These actions are used to determine how profiles, and what data is included in the + sync responses, depending on field updates and room membership changes. """ JOINED_ROOM = "joined_room" """ - User joined a room. + This profile update row action represents a user joining a room. + + When gathering an incremental sync non-lazy response for profile updates, + we always include the full profile of users who have joined a room the syncing + user is a member of, where full profile means all the current profile values the + client asked for, regardless of whether they have changed recently. This ensures + that clients have profile re-populated for any users who have recently left + shared rooms. + + A scenario example would be as follows: + + * Alice leaves a room with Bob + * Bob's client clears all profile fields from Alice + * Alice joins a room with Bob + * Bob's client does an incremental non-lazy sync + + At the end of the flow Bob should receive all the profile fields the client + is interested in, not just the potential diff, which non-lazy incremental sync + normally includes. This update action currently has no meaning for sync responses + that are not incremental and non-lazy. """ LEFT_ROOM = "left_room" """ - User left a room. + This profile update row action represents a user leaving a room. + + Clients will want to know when they no longer share rooms with a user. This + profile action row allows the sync code to deliver a `null` response for those + profiles, so clients can clear their cache containing the users profile data + they are no longer interested in. """ UPDATE = "update" """ - User updated a profile field. + This profile update row action represents a user updating a profile field. + + Depending on the type of sync (initial/incremental, lazy/non-lazy), either the + diff of profile field updates or all the current profile fields are included + in the sync response. In the latter case the profile update action row signifies + a change, but the client may still get fields that have not changed. """ From f6c1db70dd16992d8d5369606b483dd37690c79f Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 18:22:25 +0300 Subject: [PATCH 093/106] Fix `StreamKeyType.__str__` --- synapse/types/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index a9481fb413f..ef99d575546 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -1391,9 +1391,9 @@ def __str__(self) -> str: f"typing: {self.typing_key}, receipt: {self.receipt_key}, " f"account_data: {self.account_data_key}, push_rules: {self.push_rules_key}, " f"to_device: {self.to_device_key}, device_list: {self.device_list_key}, " - f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key}," + f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key}, " f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key}, " - f"quarantined_media: {self.quarantined_media_key})" + f"quarantined_media: {self.quarantined_media_key}), " f"profile_updates: {self.profile_updates_key})" ) From 180c6a2746ce0f4af42f31f1fa0ea8062c361ae2 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 18:24:41 +0300 Subject: [PATCH 094/106] tuples in `_track_profile_updates_per_user_txn` --- synapse/storage/databases/main/profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 136594c0724..19731163e7a 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -679,7 +679,7 @@ async def track_profile_updates_per_user( def _track_profile_updates_per_user_txn(txn: LoggingTransaction) -> None: inserted_ts = self.clock.time_msec() - values = [[stream_id, user_id, inserted_ts] for user_id in user_ids] + values = [(stream_id, user_id, inserted_ts) for user_id in user_ids] self.db_pool.simple_insert_many_txn( txn, table="profile_updates_per_user", From e74046a5835afe2a92a288fe1ad62b5ba292d53a Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 18:28:40 +0300 Subject: [PATCH 095/106] Remove confusing wording --- synapse/storage/databases/main/profile.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 19731163e7a..750bfd78d19 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -981,8 +981,6 @@ async def clear_profile_updates_for_user( def _clear_profile_updates_for_user_txn( txn: LoggingTransaction, ) -> None: - # Delete profile updates where there's a corresponding row in - # `profile_updates_per_user`. sql = """ SELECT stream_id FROM profile_updates WHERE user_id = ? AND action = ? From 4271ffa937b9844e32cfea1f451d4141e26b22bb Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 18:40:11 +0300 Subject: [PATCH 096/106] Add primary key to `profile_updates_per_user` --- synapse/storage/schema/main/delta/94/05_profile_updates.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/storage/schema/main/delta/94/05_profile_updates.sql b/synapse/storage/schema/main/delta/94/05_profile_updates.sql index 3d2653648bb..b612ceea861 100644 --- a/synapse/storage/schema/main/delta/94/05_profile_updates.sql +++ b/synapse/storage/schema/main/delta/94/05_profile_updates.sql @@ -38,6 +38,9 @@ CREATE INDEX IF NOT EXISTS profile_updates_inserted_ts ON profile_updates (inser -- Track which local users should receive each profile update. CREATE TABLE IF NOT EXISTS profile_updates_per_user ( + id $%AUTO_INCREMENT_PRIMARY_KEY%$, + + -- Stream ID reference to `profile_updates` stream_id BIGINT NOT NULL REFERENCES profile_updates (stream_id), -- The full user ID of the local user that should receive the profile update. From 4934fca9467bc73d681e81bb88e4955e69983bf7 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 19:03:29 +0300 Subject: [PATCH 097/106] Clean up some SQL for easier reading --- synapse/storage/databases/main/profile.py | 46 ++++++++++++----------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 750bfd78d19..66011912a00 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -1003,13 +1003,14 @@ def _clear_profile_updates_for_user_txn( "stream_id", stream_ids, ) - sql = f""" + txn.execute( + f""" DELETE FROM profile_updates_per_user WHERE {user_clause} AND {stream_id_clause} - """ - params = user_args + stream_id_args - txn.execute(sql, (*params,)) + """, + (*user_args, *stream_id_args), + ) await self.db_pool.runInteraction( "clear_profile_updates_for_user", @@ -1022,15 +1023,17 @@ async def _prune_profile_updates(self) -> None: `profile_updates_per_user` tables, so that the tables don't grow indefinitely. """ prune_before_ts = self.clock.time_msec() - PRUNE_PROFILE_UPDATES_AGE.as_millis() - cutoff_sql = """ - SELECT stream_id FROM profile_updates - WHERE inserted_ts <= ? - ORDER BY inserted_ts DESC - LIMIT 1 - """ def get_prune_before_stream_id_txn(txn: LoggingTransaction) -> int | None: - txn.execute(cutoff_sql, (prune_before_ts,)) + txn.execute( + """ + SELECT stream_id FROM profile_updates + WHERE inserted_ts <= ? + ORDER BY inserted_ts DESC + LIMIT 1 + """, + (prune_before_ts,), + ) row = txn.fetchone() return row[0] if row else None @@ -1077,18 +1080,17 @@ def prune_profile_updates_txn(txn: LoggingTransaction) -> int: nonlocal min_stream_id assert table in ("profile_updates", "profile_updates_per_user") - delete_sql = """ - DELETE FROM %s - WHERE stream_id IN ( - SELECT stream_id FROM %s - WHERE ? < stream_id AND stream_id <= ? - ORDER BY stream_id ASC - LIMIT ? - ) - RETURNING stream_id - """ % (table, table) txn.execute( - delete_sql, + f""" + DELETE FROM {table} + WHERE stream_id IN ( + SELECT stream_id FROM {table} + WHERE ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC + LIMIT ? + ) + RETURNING stream_id + """, ( min_stream_id, prune_before_stream_id, From c7f3b79e75f1405e9a18d6995be3b310f7713dc9 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 1 Jul 2026 19:05:39 +0300 Subject: [PATCH 098/106] Use `deleted_stream_id` instead of indexing on `row` --- synapse/storage/databases/main/profile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 66011912a00..173381e5fed 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -1101,9 +1101,9 @@ def prune_profile_updates_txn(txn: LoggingTransaction) -> int: # We can't use rowcount as that is incorrect on SQLite when using # RETURNING. num_deleted = 0 - for row in txn: + for (deleted_stream_id,) in txn: num_deleted += 1 - min_stream_id = max(min_stream_id, row[0]) + min_stream_id = max(min_stream_id, deleted_stream_id) return num_deleted From e68560a8a24f68b9b51aefd4db69bcd2f6fe8313 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 2 Jul 2026 22:21:00 +0300 Subject: [PATCH 099/106] Don't dump to json string in tests --- tests/handlers/test_sync.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 61cf0b02374..cd59fc6147a 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -17,7 +17,6 @@ # [This file includes modifications made by New Vector Limited] # # -import json from http import HTTPStatus from typing import Collection, ContextManager from unittest.mock import AsyncMock, Mock, patch @@ -1177,9 +1176,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store.set_profile_field( user_id=UserID.from_string(self.user), field_name="m.status", - new_value=json.dumps( - {"text": "Swimming in the Great Lakes!", "emoji": "🏊"} - ), + new_value={"text": "Swimming in the Great Lakes!", "emoji": "🏊"}, ) ) self.helper.join( @@ -1194,7 +1191,7 @@ def test_initial_sync_no_profile_updates_if_not_enabled(self) -> None: target_user=UserID.from_string(self.other_user), requester=create_requester(self.other_user), field_name="m.status", - new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + new_value={"text": "On holiday", "emoji": "🏖"}, ) ) @@ -1219,7 +1216,7 @@ def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: target_user=UserID.from_string(self.other_user), requester=create_requester(self.other_user), field_name="m.status", - new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + new_value={"text": "On holiday", "emoji": "🏖"}, ) ) @@ -1248,7 +1245,7 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: target_user=UserID.from_string(self.other_user), requester=create_requester(self.other_user), field_name="m.status", - new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + new_value={"text": "On holiday", "emoji": "🏖"}, ) ) @@ -1274,7 +1271,7 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: assert initial_result.profile_updates["@other_user:test"] is not None self.assertEqual( initial_result.profile_updates["@other_user:test"]["m.status"], - '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', + {"text": "On holiday", "emoji": "🏖"}, ) self.assertCountEqual( initial_result.profile_updates.keys(), @@ -1302,7 +1299,7 @@ def test_initial_sync_does_not_include_untracked_users_profile_updates( target_user=UserID.from_string(third_user), requester=create_requester(third_user), field_name="m.status", - new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + new_value={"text": "On holiday", "emoji": "🏖"}, ) ) @@ -1361,7 +1358,7 @@ def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( target_user=UserID.from_string(self.other_user), requester=create_requester(self.other_user), field_name="m.status", - new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + new_value={"text": "On holiday", "emoji": "🏖"}, ) ) # Check that lazy-loading filters out profile updates as well on initial sync. @@ -1370,7 +1367,7 @@ def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( target_user=UserID.from_string(third_user), requester=create_requester(third_user), field_name="m.status", - new_value=json.dumps({"text": "On fire", "emoji": "🔥"}), + new_value={"text": "On fire", "emoji": "🔥"}, ) ) self.helper.send_messages( @@ -1439,7 +1436,7 @@ def test_incremental_sync_sends_down_profile_updates( target_user=UserID.from_string(self.other_user), requester=create_requester(self.other_user), field_name="m.status", - new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + new_value={"text": "On holiday", "emoji": "🏖"}, ) ) incremental_result = self.get_success( @@ -1463,7 +1460,7 @@ def test_incremental_sync_sends_down_profile_updates( assert incremental_result.profile_updates["@other_user:test"] is not None self.assertEqual( incremental_result.profile_updates["@other_user:test"]["m.status"], - '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', + {"text": "On holiday", "emoji": "🏖"}, ) # We only send diffs in incremental sync for profile field updates self.assertIsNone( @@ -1509,7 +1506,7 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( target_user=UserID.from_string(self.other_user), requester=create_requester(self.other_user), field_name="m.status", - new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + new_value={"text": "On holiday", "emoji": "🏖"}, ) ) self.get_success( @@ -1517,7 +1514,7 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( target_user=UserID.from_string(third_user), requester=create_requester(third_user), field_name="m.status", - new_value=json.dumps({"text": "On fire", "emoji": "🔥"}), + new_value={"text": "On fire", "emoji": "🔥"}, ) ) self.helper.send_messages( @@ -1562,7 +1559,7 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( # This is a field update, so should be here self.assertEqual( incremental_result.profile_updates["@other_user:test"]["m.status"], - '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', + {"text": "On holiday", "emoji": "🏖"}, ) # We don't have events for this user in this response, so their full profile # is not included @@ -1573,7 +1570,7 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( # This user has events in the timeline, thus their full profile is included self.assertEqual( incremental_result.profile_updates["@third_user:test"]["m.status"], - '{"text": "On fire", "emoji": "\\ud83d\\udd25"}', + {"text": "On fire", "emoji": "🔥"}, ) self.assertEqual( incremental_result.profile_updates["@third_user:test"]["displayname"], @@ -1847,7 +1844,7 @@ def test_incremental_sync_includes_own_profile_updates(self, is_lazy: bool) -> N target_user=UserID.from_string(self.user), requester=requester, field_name="m.status", - new_value=json.dumps({"text": "On holiday", "emoji": "🏖"}), + new_value={"text": "On holiday", "emoji": "🏖"}, ) ) incremental_result = self.get_success( @@ -1867,7 +1864,7 @@ def test_incremental_sync_includes_own_profile_updates(self, is_lazy: bool) -> N assert incremental_result.profile_updates["@user:test"] is not None self.assertEqual( incremental_result.profile_updates["@user:test"]["m.status"], - '{"text": "On holiday", "emoji": "\\ud83c\\udfd6"}', + {"text": "On holiday", "emoji": "🏖"}, ) From 868513720f0b0702eecb4cea3f839204753837f9 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 2 Jul 2026 22:25:43 +0300 Subject: [PATCH 100/106] Correctly type `ProfileUpdatesStreamRow.action` --- synapse/replication/tcp/streams/_base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 92d10cc7849..4f16b25b543 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -31,7 +31,7 @@ import attr -from synapse.api.constants import AccountDataTypes +from synapse.api.constants import AccountDataTypes, ProfileUpdateAction from synapse.replication.http.streams import ReplicationGetStreamUpdates from synapse.types import UserID @@ -772,9 +772,10 @@ class ProfileUpdatesStreamRow: user_id: UserID """The full user ID with the profile update.""" - action: str - """The action, either 'update' for a field update or 'left_room' if the user left a room, - see ProfileUpdateAction constant.""" + action: ProfileUpdateAction + """The action, either 'update' for a field update, 'left_room' if the user left + a room or `joined_room` if the user joined a room, see ProfileUpdateAction constant. + """ field_name: str """The profile field that was updated, see https://spec.matrix.org/unstable/client-server-api/#profiles """ From 56b7de210ec24aa98f90d24d9d6eacbcae9415f4 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 2 Jul 2026 22:27:18 +0300 Subject: [PATCH 101/106] Correctly type `ProfileUpdatesStreamRow.field_name` --- synapse/replication/tcp/streams/_base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 4f16b25b543..86242b23413 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -776,8 +776,10 @@ class ProfileUpdatesStreamRow: """The action, either 'update' for a field update, 'left_room' if the user left a room or `joined_room` if the user joined a room, see ProfileUpdateAction constant. """ - field_name: str - """The profile field that was updated, see https://spec.matrix.org/unstable/client-server-api/#profiles """ + field_name: str | None + """The profile field that was updated, see https://spec.matrix.org/unstable/client-server-api/#profiles. + This can be None if `action` is not 'update'. + """ class ProfileUpdatesStream(_StreamFromIdGen): From 182e4c2d107cbc69ceac45083226e604b5e3ca1e Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 2 Jul 2026 22:40:41 +0300 Subject: [PATCH 102/106] Clarify variable names and add comments to `ReplicationDataHandler` --- synapse/replication/tcp/client.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index dc68f554460..00a24db749b 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -270,11 +270,12 @@ async def on_rdata( updated_user_ids = {row.user_id for row in rows} if updated_user_ids: room_ids: set[str] = set() - user_ids_to_room_ids = await self.store.get_rooms_for_users( - updated_user_ids - ) - for batched_user_ids_to_room_ids in user_ids_to_room_ids.values(): - room_ids.update(batched_user_ids_to_room_ids) + # Get all the rooms of the updated users, dict of + # User ID -> [Room ID] + users_and_rooms = await self.store.get_rooms_for_users(updated_user_ids) + # Loop through each users room ID's and add to our set of rooms + for user_room_ids in users_and_rooms.values(): + room_ids.update(user_room_ids) if room_ids: self.notifier.on_new_event( From 2c7b418105c1597a6f25cd64b603fa05b37d6ec6 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 2 Jul 2026 23:13:05 +0300 Subject: [PATCH 103/106] Ensure we filter out federated users in lazy loading sync when collecting profile updates from timeline events --- synapse/handlers/sync.py | 16 ++++-- tests/handlers/test_sync.py | 110 ++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 17f0467deb3..b4b01c92574 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2255,22 +2255,28 @@ async def _generate_sync_entry_for_profile_updates( lazy_load_members = sync_config.filter_collection.lazy_load_members() include_users = None if lazy_load_members: - # Collect members from the existing `sync_result_builder` data + # Collect members from the existing `sync_result_builder` data. + # Ensure we filter out any remove users until we support profile + # updates for federated users. include_users = set() # invited for invited in sync_result_builder.invited: - include_users.add(invited.invite.sender) + if self._is_mine_id(invited.invite.sender): + include_users.add(invited.invite.sender) # joined for joined in sync_result_builder.joined: for timeline_event in joined.timeline.events: - include_users.add(timeline_event.event.sender) + if self._is_mine_id(timeline_event.event.sender): + include_users.add(timeline_event.event.sender) # knocked for knocked in sync_result_builder.knocked: - include_users.add(knocked.knock.sender) + if self._is_mine_id(knocked.knock.sender): + include_users.add(knocked.knock.sender) # archived for archived in sync_result_builder.archived: for timeline_event in archived.timeline.events: - include_users.add(timeline_event.event.sender) + if self._is_mine_id(timeline_event.event.sender): + include_users.add(timeline_event.event.sender) if since_token is None: await self._generate_initial_sync_entry_for_profile_updates( diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index cd59fc6147a..86f7b431666 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -55,6 +55,7 @@ import tests.unittest import tests.utils from tests.test_utils.event_builders import make_test_pdu_event +from tests.test_utils.event_injection import inject_member_event from tests.unittest import override_config _request_key = 0 @@ -1523,6 +1524,15 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( self.helper.send_messages( room_id=self.joined_room, num_events=10, tok=third_tok ) + # Join a federated user to the room + self.get_success( + inject_member_event( + self.hs, + self.joined_room, + "@federateduser:federatedhs", + "join", + ) + ) incremental_result = self.get_success( self.sync_handler.wait_for_sync_for_user( requester, @@ -1546,6 +1556,12 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( request_key=generate_request_key(), ) ) + + # Ensure our federated user is filtered out, even though they have an + # event in the joined room timeline + self.assertFalse( + "@federateduser:federatedhs" in incremental_result.profile_updates.keys() + ) # Lazy loading only filters initial sync profile updates. Incremental syncs # should include all tracked profile updates for the syncing user. self.assertCountEqual( @@ -1577,6 +1593,100 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( "third_user", ) + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sync_filters_out_profile_updates_from_federated_users( + self, + is_initial: bool, + ) -> None: + """Test that with MSC4429 enabled lazy loading sync response + doesn't contain federated users even if there are timeline events from them. + """ + # Join a federated user to the room, causing a membership event into + # the joined rooms sync response + self.get_success( + inject_member_event( + self.hs, + self.joined_room, + "@federateduser1:federatedhs", + "join", + ) + ) + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + # Ensure our federated user is filtered out, even though they have an + # event in the joined room timeline + self.assertFalse( + "@federateduser1:federatedhs" in initial_result.profile_updates.keys() + ) + if not is_initial: + # Join another federated user to the room, causing a membership event into + # the joined rooms sync response + self.get_success( + inject_member_event( + self.hs, + self.joined_room, + "@federateduser2:federatedhs", + "join", + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + + # Ensure our federated user is filtered out, even though they have an + # event in the joined room timeline + self.assertFalse( + "@federateduser2:federatedhs" + in incremental_result.profile_updates.keys() + ) + @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles( self, From 6f6e633b5c8b3f0c225d1442230ddf5bebb1796b Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 3 Jul 2026 14:58:33 +0300 Subject: [PATCH 104/106] Ensure we don't respond with profile fields the client didn't ask for --- synapse/handlers/sync.py | 35 ++++-- tests/handlers/test_sync.py | 210 +++++++++++++++++++++++++++--------- 2 files changed, 184 insertions(+), 61 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b4b01c92574..99d79a6ea4a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2329,12 +2329,21 @@ async def _generate_sync_entry_for_profile_updates( # Process field updates and users who have events in the sync response if users: - user_fields: dict[str, set[str]] = {} + updated_user_fields: dict[str, set[str]] = {} # Set fields from updates for update in updates: - if not update.field_name or update.user_id not in users: + # Skip the update if there is no field update (a joined or left room + # action), the client didn't ask for this field, or we're not + # interested in this user. + if ( + not update.field_name + or update.field_name not in profile_fields + or update.user_id not in users + ): continue - user_fields.setdefault(update.user_id, set()).add(update.field_name) + updated_user_fields.setdefault(update.user_id, set()).add( + update.field_name + ) # Note: there's a small race condition here where a profile update may # occur between fetching `now_token` above and reaching this step. In @@ -2357,10 +2366,11 @@ async def _generate_sync_entry_for_profile_updates( per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} if include_users and other_user_id in include_users: - # Include the full profile as this user has events in - # a lazy loaded sync response, except for fields we've recently - # sent in a previous lazy loaded sync response - for field_name in profile_data.keys(): + # Include all the fields the client asked for, as this user + # has events in a lazy loaded sync response, except for + # fields we've recently sent in a previous lazy loaded sync response + fields = set(profile_data.keys()).intersection(profile_fields) + for field_name in fields: cache_key = ( sync_config.user.to_string(), sync_config.device_id, @@ -2388,12 +2398,15 @@ async def _generate_sync_entry_for_profile_updates( cast(str, profile_data.get(field_name)), ) else: - # Include only the diff, unless the user recently joined - # We don't use a cache here as changes are always sent + # Include only the diff, unless the user recently joined, + # then send all the fields the client asked for. + # We don't use a cache here as for non-lazy sync we always + # send changes and/or fields the client asked for, if relevant + # as above joined condition. fields = ( - list(profile_data.keys()) + profile_fields if other_user_id in joined_room_user_ids - else user_fields.get(other_user_id, []) + else set(updated_user_fields.get(other_user_id, [])) ) for field_name in fields: per_user_updates[field_name] = profile_data.get(field_name) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 86f7b431666..bb230cdcf60 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1239,8 +1239,8 @@ def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: """Test that with MSC4429 enabled the initial sync response does - contain profile updates for users who share rooms, including our - syncing user.""" + contain profile updates for users who share rooms, for the fields the + client requests. This response should include our syncing user.""" self.get_success( self.profile_handler.set_field( target_user=UserID.from_string(self.other_user), @@ -1249,6 +1249,15 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: new_value={"text": "On holiday", "emoji": "🏖"}, ) ) + # Also set a field the client doesn't want + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="displayname", + new_value="New displayname", + ) + ) requester = create_requester(self.user) initial_result = self.get_success( @@ -1259,9 +1268,7 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: filter_collection=FilterCollection( hs=self.hs, filter_json={ - "org.matrix.msc4429.profile_fields": { - "ids": ["m.status", "displayname", "avatar_url"] - } + "org.matrix.msc4429.profile_fields": {"ids": ["m.status"]} }, ), ), @@ -1274,6 +1281,9 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: initial_result.profile_updates["@other_user:test"]["m.status"], {"text": "On holiday", "emoji": "🏖"}, ) + self.assertFalse( + "displayname" in initial_result.profile_updates["@other_user:test"].keys(), + ) self.assertCountEqual( initial_result.profile_updates.keys(), [ @@ -1409,11 +1419,11 @@ def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( ) @override_config({"include_profile_updates_in_sync": True}) - def test_incremental_sync_sends_down_profile_updates( + def test_incremental_sync_sends_down_profile_update_diffs( self, ) -> None: """Test that with MSC4429 enabled the incremental sync response does - contain profile updates.""" + contain profile update diffs.""" requester = create_requester(self.user) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( @@ -1440,6 +1450,15 @@ def test_incremental_sync_sends_down_profile_updates( new_value={"text": "On holiday", "emoji": "🏖"}, ) ) + # Set a field the client didn't ask for + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="uninterestingfield", + new_value="Content", + ) + ) incremental_result = self.get_success( self.sync_handler.wait_for_sync_for_user( requester, @@ -1464,8 +1483,14 @@ def test_incremental_sync_sends_down_profile_updates( {"text": "On holiday", "emoji": "🏖"}, ) # We only send diffs in incremental sync for profile field updates - self.assertIsNone( - incremental_result.profile_updates["@other_user:test"].get("displayname"), + self.assertFalse( + "displayname" + in incremental_result.profile_updates["@other_user:test"].keys(), + ) + # The client didn't ask for this field + self.assertFalse( + "uninterestingfield" + in incremental_result.profile_updates["@other_user:test"].keys(), ) @override_config({"include_profile_updates_in_sync": True}) @@ -1494,7 +1519,7 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( hs=self.hs, filter_json={ "org.matrix.msc4429.profile_fields": { - "ids": ["m.status", "displayname", "avatar_url"] + "ids": ["m.status", "displayname"] } }, ), @@ -1524,6 +1549,14 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( self.helper.send_messages( room_id=self.joined_room, num_events=10, tok=third_tok ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="uninterestingfield", + new_value="Content", + ) + ) # Join a federated user to the room self.get_success( inject_member_event( @@ -1543,7 +1576,7 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( hs=self.hs, filter_json={ "org.matrix.msc4429.profile_fields": { - "ids": ["m.status", "displayname", "avatar_url"] + "ids": ["m.status", "displayname"] }, "room": { "state": { @@ -1562,6 +1595,7 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( self.assertFalse( "@federateduser:federatedhs" in incremental_result.profile_updates.keys() ) + # Lazy loading only filters initial sync profile updates. Incremental syncs # should include all tracked profile updates for the syncing user. self.assertCountEqual( @@ -1572,22 +1606,31 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( ], ) assert incremental_result.profile_updates["@other_user:test"] is not None + # This is a field update, so should be here self.assertEqual( incremental_result.profile_updates["@other_user:test"]["m.status"], {"text": "On holiday", "emoji": "🏖"}, ) + # We don't have events for this user in this response, so their full profile # is not included - self.assertIsNone( - incremental_result.profile_updates["@other_user:test"].get("displayname"), + self.assertFalse( + "displayname" + in incremental_result.profile_updates["@other_user:test"].keys(), ) assert incremental_result.profile_updates["@third_user:test"] is not None - # This user has events in the timeline, thus their full profile is included + + # This user has events in the timeline, thus the fields the client asked for + # are included self.assertEqual( incremental_result.profile_updates["@third_user:test"]["m.status"], {"text": "On fire", "emoji": "🔥"}, ) + self.assertFalse( + "uninterestingfield" + in incremental_result.profile_updates["@third_user:test"].keys(), + ) self.assertEqual( incremental_result.profile_updates["@third_user:test"]["displayname"], "third_user", @@ -1595,16 +1638,19 @@ def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( @parameterized.expand( [ - True, - False, + [True, True], + [False, False], + [True, False], + [False, True], ] ) @override_config({"include_profile_updates_in_sync": True}) - def test_lazy_loading_sync_filters_out_profile_updates_from_federated_users( + def test_sync_filters_out_profile_updates_from_federated_users( self, is_initial: bool, + is_lazy: bool, ) -> None: - """Test that with MSC4429 enabled lazy loading sync response + """Test that with MSC4429 enabled any sync response doesn't contain federated users even if there are timeline events from them. """ # Join a federated user to the room, causing a membership event into @@ -1618,6 +1664,17 @@ def test_lazy_loading_sync_filters_out_profile_updates_from_federated_users( ) ) requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( requester, @@ -1625,16 +1682,7 @@ def test_lazy_loading_sync_filters_out_profile_updates_from_federated_users( user_id=self.user, filter_collection=FilterCollection( hs=self.hs, - filter_json={ - "org.matrix.msc4429.profile_fields": { - "ids": ["m.status", "displayname", "avatar_url"] - }, - "room": { - "state": { - "lazy_load_members": True, - }, - }, - }, + filter_json=filter_json, ), ), request_key=generate_request_key(), @@ -1664,16 +1712,7 @@ def test_lazy_loading_sync_filters_out_profile_updates_from_federated_users( user_id=self.user, filter_collection=FilterCollection( hs=self.hs, - filter_json={ - "org.matrix.msc4429.profile_fields": { - "ids": ["m.status", "displayname", "avatar_url"] - }, - "room": { - "state": { - "lazy_load_members": True, - }, - }, - }, + filter_json=filter_json, ), ), request_key=generate_request_key(), @@ -1688,13 +1727,21 @@ def test_lazy_loading_sync_filters_out_profile_updates_from_federated_users( ) @override_config({"include_profile_updates_in_sync": True}) - def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles( + def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles_and_fields( self, ) -> None: """Test that with MSC4429 enabled the incremental sync lazy loading response - filters out unchanged profiles we have recently sent to the client. + filters out unchanged profiles or fields we have recently sent to the client. """ requester = create_requester(self.user) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="sooninterestingfield", + new_value="Content", + ) + ) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( requester, @@ -1747,9 +1794,14 @@ def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles( "@other_user:test", ], ) + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"].keys(), + ["displayname", "avatar_url"], + ) # If we have more events from the other_user, and do another lazy sync, - # we don't expect the full profile to be sent again due to our cache + # we don't expect the full profile to be sent again due to our cache. self.helper.send_messages( room_id=self.joined_room, num_events=1, tok=self.other_tok ) @@ -1780,6 +1832,50 @@ def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles( incremental_result.profile_updates.keys(), [], ) + # However, if we again add an event, we do expect any fields the client didn't + # previously ask for to be there. + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": [ + "m.status", + "displayname", + "avatar_url", + "sooninterestingfield", + ] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [ + "@other_user:test", + ], + ) + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"].keys(), + ["sooninterestingfield"], + ) @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms( @@ -1832,11 +1928,12 @@ def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_room ) @override_config({"include_profile_updates_in_sync": True}) - def test_incremental_sync_sends_down_full_profile_for_users_who_have_joined( + def test_incremental_sync_sends_down_all_requested_fields_for_users_who_have_joined( self, ) -> None: """Test that with MSC4429 enabled the incremental sync response - includes the full profile of a user who has joined a room with the syncing user. + includes all the requested fields of a user who has joined a room with the + syncing user. """ requester = create_requester(self.user) initial_result = self.get_success( @@ -1848,7 +1945,7 @@ def test_incremental_sync_sends_down_full_profile_for_users_who_have_joined( hs=self.hs, filter_json={ "org.matrix.msc4429.profile_fields": { - "ids": ["m.status", "displayname", "avatar_url"] + "ids": ["displayname", "avatar_url"] }, }, ), @@ -1866,7 +1963,7 @@ def test_incremental_sync_sends_down_full_profile_for_users_who_have_joined( hs=self.hs, filter_json={ "org.matrix.msc4429.profile_fields": { - "ids": ["m.status", "displayname", "avatar_url"] + "ids": ["displayname", "avatar_url"] }, }, ), @@ -1882,7 +1979,15 @@ def test_incremental_sync_sends_down_full_profile_for_users_who_have_joined( user=third_user, tok=third_tok, ) - + # Set a status field we don't except to see in sync + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value={"text": "On fire", "emoji": "🔥"}, + ) + ) incremental_result = self.get_success( self.sync_handler.wait_for_sync_for_user( requester, @@ -1893,7 +1998,7 @@ def test_incremental_sync_sends_down_full_profile_for_users_who_have_joined( hs=self.hs, filter_json={ "org.matrix.msc4429.profile_fields": { - "ids": ["m.status", "displayname", "avatar_url"] + "ids": ["displayname", "avatar_url"] }, }, ), @@ -1913,6 +2018,9 @@ def test_incremental_sync_sends_down_full_profile_for_users_who_have_joined( self.assertIsNone( incremental_result.profile_updates["@third_user:test"]["avatar_url"], ) + self.assertFalse( + "m.status" in incremental_result.profile_updates["@third_user:test"].keys(), + ) @parameterized.expand( [ @@ -1926,9 +2034,7 @@ def test_incremental_sync_includes_own_profile_updates(self, is_lazy: bool) -> N ones own profile updates.""" requester = create_requester(self.user) filter_json: dict[str, dict] = { - "org.matrix.msc4429.profile_fields": { - "ids": ["m.status", "displayname", "avatar_url"] - } + "org.matrix.msc4429.profile_fields": {"ids": ["m.status", "avatar_url"]} } if is_lazy: filter_json["room"] = { @@ -1976,6 +2082,10 @@ def test_incremental_sync_includes_own_profile_updates(self, is_lazy: bool) -> N incremental_result.profile_updates["@user:test"]["m.status"], {"text": "On holiday", "emoji": "🏖"}, ) + # We didn't ask for displayname + self.assertFalse( + "displayname" in incremental_result.profile_updates["@user:test"].keys(), + ) class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): From d3dd12f027300b1469f25853a9527d450e5bf711 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 3 Jul 2026 15:39:02 +0300 Subject: [PATCH 105/106] Rewrite `lazy_loaded_profile_fields_cache` --- synapse/handlers/sync.py | 51 +++++++++++++++++-------------------- tests/handlers/test_sync.py | 8 +++--- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 99d79a6ea4a..6315a4c8335 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -26,7 +26,6 @@ Any, Mapping, Sequence, - cast, ) import attr @@ -346,9 +345,9 @@ def __init__(self, hs: "HomeServer"): max_len=0, expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE, ) - # ExpiringCache((User, Device, Other User, Profile Field)) -> LruCache(user_id => field_value) + # ExpiringCache((User, Device)) -> LruCache(Other User ID + Field Name -> bool) self.lazy_loaded_profile_fields_cache: ExpiringCache[ - tuple[str, str | None, str, str], LruCache[str, str] + tuple[str, str | None], LruCache[str, bool] ] = ExpiringCache( cache_name="lazy_loaded_profile_fields_cache", server_name=self.server_name, @@ -357,6 +356,16 @@ def __init__(self, hs: "HomeServer"): max_len=0, expiry_ms=LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE, ) + """This cache contains fields we have sent to clients as profile updates, + for a particular user + device combo. The cache entry is a combination of the + user + field name, with the value existing indicating the field has recently + been sent. The boolean value does not hold other significance. A missing + cache entry means "we have not sent this user + field name combo to the + syncing user". + + We don't manually remove entries from this cache, though it may be ignored + in cases where the sync must send the field down to the client. + """ self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync @@ -1065,9 +1074,9 @@ def get_lazy_loaded_members_cache( return cache def get_lazy_loaded_profile_fields_cache( - self, cache_key: tuple[str, str | None, str, str] - ) -> LruCache[str, str]: - cache: LruCache[str, str] | None = self.lazy_loaded_profile_fields_cache.get( + self, cache_key: tuple[str, str | None] + ) -> LruCache[str, bool]: + cache: LruCache[str, bool] | None = self.lazy_loaded_profile_fields_cache.get( cache_key ) if cache is None: @@ -2374,29 +2383,17 @@ async def _generate_sync_entry_for_profile_updates( cache_key = ( sync_config.user.to_string(), sync_config.device_id, - other_user_id, - field_name, ) cache = self.get_lazy_loaded_profile_fields_cache(cache_key) - # Only send the field if we haven't recently sent it - if cache.get(other_user_id) is None: - # If the field value is a None, don't send it down or - # set the cache unless we're sure it has become None due - # to a profile update, otherwise we'll just be sending the - # same field down in every single incremental lazy sync - # regardless of cache state - if ( - profile_data.get(field_name) is not None - or other_user_id in updated_users - ): - per_user_updates[field_name] = profile_data.get( - field_name - ) - # Update our cache - cache.set( - other_user_id, - cast(str, profile_data.get(field_name)), - ) + # Only send this users field if we haven't recently sent it + if cache.get(f"{other_user_id}-{field_name}") is None: + per_user_updates[field_name] = profile_data.get(field_name) + # Update our cache to indicate this user/field combo + # has been recently sent. + cache.set( + f"{other_user_id}-{field_name}", + True, + ) else: # Include only the diff, unless the user recently joined, # then send all the fields the client asked for. diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index bb230cdcf60..b1d8f6076b0 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1796,8 +1796,8 @@ def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles_and_ ) assert incremental_result.profile_updates["@other_user:test"] is not None self.assertEqual( - incremental_result.profile_updates["@other_user:test"].keys(), - ["displayname", "avatar_url"], + set(incremental_result.profile_updates["@other_user:test"].keys()), + {"avatar_url", "displayname"}, ) # If we have more events from the other_user, and do another lazy sync, @@ -1873,8 +1873,8 @@ def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles_and_ ) assert incremental_result.profile_updates["@other_user:test"] is not None self.assertEqual( - incremental_result.profile_updates["@other_user:test"].keys(), - ["sooninterestingfield"], + set(incremental_result.profile_updates["@other_user:test"].keys()), + {"sooninterestingfield"}, ) @override_config({"include_profile_updates_in_sync": True}) From 1955fbee22b253a7fe0a27a3019db42bb4eeb289 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 3 Jul 2026 18:28:49 +0300 Subject: [PATCH 106/106] Ensure falsey values survive the set_field / sync profile updates -cycle --- synapse/handlers/sync.py | 5 +- tests/handlers/test_sync.py | 94 ++++++++++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 6315a4c8335..2c1597417ee 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2230,7 +2230,7 @@ async def _generate_initial_sync_entry_for_profile_updates( per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} for field_name in profile_fields: - if profile_data.get(field_name): + if field_name in profile_data.keys(): per_user_updates[field_name] = profile_data[field_name] if per_user_updates: @@ -2405,8 +2405,9 @@ async def _generate_sync_entry_for_profile_updates( if other_user_id in joined_room_user_ids else set(updated_user_fields.get(other_user_id, [])) ) + fields = set(profile_data.keys()).intersection(fields) for field_name in fields: - per_user_updates[field_name] = profile_data.get(field_name) + per_user_updates[field_name] = profile_data[field_name] if per_user_updates: profile_updates[other_user_id] = per_user_updates diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index b1d8f6076b0..96c47191a93 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -18,7 +18,7 @@ # # from http import HTTPStatus -from typing import Collection, ContextManager +from typing import Collection, ContextManager, cast from unittest.mock import AsyncMock, Mock, patch from parameterized import parameterized, parameterized_class @@ -43,6 +43,7 @@ from synapse.server import HomeServer from synapse.types import ( JsonDict, + JsonValue, MultiWriterStreamToken, RoomStreamToken, StreamKeyType, @@ -1726,6 +1727,97 @@ def test_sync_filters_out_profile_updates_from_federated_users( in incremental_result.profile_updates.keys() ) + @parameterized.expand( + [ + [True, True], + [False, False], + [True, False], + [False, True], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_sync_profile_updates_works_correctly_with_falsey_values( + self, + is_initial: bool, + is_lazy: bool, + ) -> None: + """Test that with MSC4429 enabled a sync response correctly includes falsey + profile field values. + """ + requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": {"ids": ["falseyvaluefield"]}, + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + for value in [False, 0, "", [], {}, None]: + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="falseyvaluefield", + new_value=cast(JsonValue | dict[str, JsonValue], value), + ) + ) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + if is_initial: + assert initial_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + initial_result.profile_updates["@other_user:test"][ + "falseyvaluefield" + ], + value, + ) + else: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="falseyvaluefield", + new_value=cast(JsonValue | dict[str, JsonValue], value), + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + assert ( + incremental_result.profile_updates["@other_user:test"] is not None + ) + self.assertEqual( + incremental_result.profile_updates["@other_user:test"][ + "falseyvaluefield" + ], + value, + ) + @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles_and_fields( self,