PublicMirrors/conduwuit-mirror
0
0
Fork 0
mirror of https://github.com/girlbossceo/conduwuit.git synced 2025-04-07 02:42:49 +00:00
Code Issues Projects Releases Packages Wiki Activity

skip a few flakey complement tests

Browse source 
Signed-off-by: June Clementine Strawberry <june@3.dog>
This commit is contained in:
June Clementine Strawberry 2025-03-14 15:57:18 -04:00
parent 6c29792b3d
commit ee3c585555
No known key found for this signature in database
2 changed files with 8 additions and 78 deletions
bin
complement
tests/test_results/complement
test_results.jsonl

4
bin/complement
View file

@ -18,7 +18,7 @@ RESULTS_FILE="${3:-complement_test_results.jsonl}"
COMPLEMENT_BASE_IMAGE="${COMPLEMENT_BASE_IMAGE:-complement-conduwuit:main}"
# Complement tests that are skipped due to flakiness/reliability issues or we don't implement such features and won't for a long time
#SKIPPED_COMPLEMENT_TESTS='-skip=TestPartialStateJoin.*'
SKIPPED_COMPLEMENT_TESTS='TestPartialStateJoin.*|TestRoomDeleteAlias/Parallel/Regular_users_can_add_and_delete_aliases_when_m.*|TestRoomDeleteAlias/Parallel/Can_delete_canonical_alias|TestUnbanViaInvite.*|TestRoomState/Parallel/GET_/publicRooms_lists.*"|TestRoomDeleteAlias/Parallel/Users_with_sufficient_power-level_can_delete_other.*'
# $COMPLEMENT_SRC needs to be a directory to Complement source code
if [ -f "$COMPLEMENT_SRC" ]; then
@ -68,7 +68,7 @@ set +o pipefail
env \
-C "$COMPLEMENT_SRC" \
COMPLEMENT_BASE_IMAGE="$COMPLEMENT_BASE_IMAGE" \
go test -tags="conduwuit_blacklist" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE"
go test -tags="conduwuit_blacklist" -skip="$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE"
set -o pipefail
# Post-process the results into an easy-to-compare format, sorted by Test name for reproducible results

82
tests/test_results/complement/test_results.jsonl
View file

@ -174,10 +174,10 @@
{"Action":"pass","Test":"TestFilter"}
{"Action":"fail","Test":"TestFilterMessagesByRelType"}
{"Action":"pass","Test":"TestGappedSyncLeaveSection"}
{"Action":"fail","Test":"TestGetFilteredRoomMembers"}
{"Action":"fail","Test":"TestGetFilteredRoomMembers/membership/join"}
{"Action":"fail","Test":"TestGetFilteredRoomMembers/membership/leave"}
{"Action":"fail","Test":"TestGetFilteredRoomMembers/not_membership"}
{"Action":"pass","Test":"TestGetFilteredRoomMembers"}
{"Action":"pass","Test":"TestGetFilteredRoomMembers/membership/join"}
{"Action":"pass","Test":"TestGetFilteredRoomMembers/membership/leave"}
{"Action":"pass","Test":"TestGetFilteredRoomMembers/not_membership"}
{"Action":"fail","Test":"TestGetMissingEventsGapFilling"}
{"Action":"pass","Test":"TestGetRoomMembers"}
{"Action":"fail","Test":"TestGetRoomMembersAtPoint"}
@ -360,72 +360,6 @@
{"Action":"pass","Test":"TestOutboundFederationProfile"}
{"Action":"pass","Test":"TestOutboundFederationProfile/Outbound_federation_can_query_profile_data"}
{"Action":"pass","Test":"TestOutboundFederationSend"}
{"Action":"fail","Test":"TestPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanFastJoinDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanLazyLoadingSyncDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveDeviceListUpdateDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsWithHalfMissingGrandparentsDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsWithHalfMissingParentsDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsWithMissingParentsDuringPartialStateJoin"}
{"Action":"skip","Test":"TestPartialStateJoin/CanReceivePresenceDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveReceiptDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveSigningKeyUpdateDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveToDeviceDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveTypingDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/CanSendEventsDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/Can_change_display_name_during_partial_state_join"}
{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking"}
{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_for_user_incorrectly_believed_to_be_in_room"}
{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_when_failing_to_complete_partial_state_join"}
{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_when_leaving_partial_state_room"}
{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_when_new_member_leaves_partial_state_room"}
{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracked_for_new_members_in_partial_state_room"}
{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_pre-existing_members_in_partial_state_room"}
{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_user_incorrectly_believed_to_be_in_room_when_they_join_another_shared_room_before_partial_state_join_completes"}
{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_user_incorrectly_believed_to_be_in_room_when_they_rejoin_after_partial_state_join_completes"}
{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_user_incorrectly_believed_to_be_in_room_when_they_rejoin_before_partial_state_join_completes"}
{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_when_pre-existing_members_in_partial_state_room_join_another_shared_room"}
{"Action":"fail","Test":"TestPartialStateJoin/EagerIncrementalSyncDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/EagerInitialSyncDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/EagerLongPollingSyncWokenWhenResyncCompletes"}
{"Action":"fail","Test":"TestPartialStateJoin/GappySyncAfterPartialStateSynced"}
{"Action":"fail","Test":"TestPartialStateJoin/Lazy-loading_gappy_sync_includes_remote_memberships_during_partial_state_join"}
{"Action":"fail","Test":"TestPartialStateJoin/Lazy-loading_incremental_sync_includes_remote_memberships_during_partial_state_join"}
{"Action":"fail","Test":"TestPartialStateJoin/Lazy-loading_initial_sync_includes_remote_memberships_during_partial_state_join"}
{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync"}
{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/can_be_triggered_by_remote_ban"}
{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/can_be_triggered_by_remote_kick"}
{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/does_not_wait_for_resync"}
{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/is_seen_after_the_resync"}
{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/succeeds,_then_another_user_can_join_without_resync_completing"}
{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/succeeds,_then_rejoin_succeeds_without_resync_completing"}
{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/works_after_a_second_partial_join"}
{"Action":"fail","Test":"TestPartialStateJoin/MembersRequestBlocksDuringPartialStateJoin"}
{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates"}
{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_no_longer_reach_departed_servers_after_partial_state_join_completes"}
{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_all_servers_in_partial_state_rooms"}
{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_absent_servers_once_partial_state_join_completes"}
{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_absent_servers_once_partial_state_join_completes_even_though_remote_server_left_room"}
{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_kicked_servers_once_partial_state_join_completes"}
{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_kicked_servers_once_partial_state_join_completes_even_though_remote_server_left_room"}
{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_newly_joined_servers_in_partial_state_rooms"}
{"Action":"fail","Test":"TestPartialStateJoin/PartialStateJoinContinuesAfterRestart"}
{"Action":"fail","Test":"TestPartialStateJoin/PartialStateJoinSyncsUsingOtherHomeservers"}
{"Action":"skip","Test":"TestPartialStateJoin/Purge_during_resync"}
{"Action":"fail","Test":"TestPartialStateJoin/Rejected_events_remain_rejected_after_resync"}
{"Action":"fail","Test":"TestPartialStateJoin/Rejects_make_join_during_partial_join"}
{"Action":"fail","Test":"TestPartialStateJoin/Rejects_make_knock_during_partial_join"}
{"Action":"fail","Test":"TestPartialStateJoin/Rejects_send_join_during_partial_join"}
{"Action":"fail","Test":"TestPartialStateJoin/Rejects_send_knock_during_partial_join"}
{"Action":"fail","Test":"TestPartialStateJoin/Resync_completes_even_when_events_arrive_before_their_prev_events"}
{"Action":"fail","Test":"TestPartialStateJoin/Room_aliases_can_be_added_and_deleted_during_a_resync"}
{"Action":"fail","Test":"TestPartialStateJoin/Room_aliases_can_be_added_and_queried_during_a_resync"}
{"Action":"skip","Test":"TestPartialStateJoin/Room_stats_are_correctly_updated_once_state_re-sync_completes"}
{"Action":"fail","Test":"TestPartialStateJoin/State_accepted_incorrectly"}
{"Action":"fail","Test":"TestPartialStateJoin/State_rejected_incorrectly"}
{"Action":"fail","Test":"TestPartialStateJoin/User_directory_is_correctly_updated_once_state_re-sync_completes"}
{"Action":"fail","Test":"TestPartialStateJoin/joined_members_blocks_during_partial_state_join"}
{"Action":"fail","Test":"TestPollsLocalPushRules"}
{"Action":"fail","Test":"TestPollsLocalPushRules/Polls_push_rules_are_correctly_presented_to_the_client"}
{"Action":"pass","Test":"TestPowerLevels"}
@ -561,16 +495,13 @@
{"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Room_creation_reports_m.room.member_to_myself"}
{"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Setting_room_topic_reports_m.room.topic_to_myself"}
{"Action":"fail","Test":"TestRoomCreationReportsEventsToMyself/parallel/Setting_state_twice_is_idempotent"}
{"Action":"fail","Test":"TestRoomDeleteAlias"}
{"Action":"fail","Test":"TestRoomDeleteAlias/Parallel"}
{"Action":"pass","Test":"TestRoomDeleteAlias"}
{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel"}
{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Alias_creators_can_delete_alias_with_no_ops"}
{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Alias_creators_can_delete_canonical_alias_with_no_ops"}
{"Action":"fail","Test":"TestRoomDeleteAlias/Parallel/Can_delete_canonical_alias"}
{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Deleting_a_non-existent_alias_should_return_a_404"}
{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Regular_users_can_add_and_delete_aliases_in_the_default_room_configuration"}
{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Regular_users_can_add_and_delete_aliases_when_m.room.aliases_is_restricted"}
{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Users_can't_delete_other's_aliases"}
{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Users_with_sufficient_power-level_can_delete_other's_aliases"}
{"Action":"fail","Test":"TestRoomForget"}
{"Action":"fail","Test":"TestRoomForget/Parallel"}
{"Action":"pass","Test":"TestRoomForget/Parallel/Can't_forget_room_you're_still_in"}
@ -687,7 +618,6 @@
{"Action":"pass","Test":"TestTyping"}
{"Action":"pass","Test":"TestTyping/Typing_can_be_explicitly_stopped"}
{"Action":"pass","Test":"TestTyping/Typing_notification_sent_to_local_room_members"}
{"Action":"fail","Test":"TestUnbanViaInvite"}
{"Action":"fail","Test":"TestUnknownEndpoints"}
{"Action":"pass","Test":"TestUnknownEndpoints/Client-server_endpoints"}
{"Action":"fail","Test":"TestUnknownEndpoints/Key_endpoints"}