diff --git a/paper-server/src/main/java/org/bukkit/craftbukkit/entity/CraftPlayer.java b/paper-server/src/main/java/org/bukkit/craftbukkit/entity/CraftPlayer.java
index d20735dbd18a..2fdc19114f70 100644
--- a/paper-server/src/main/java/org/bukkit/craftbukkit/entity/CraftPlayer.java
+++ b/paper-server/src/main/java/org/bukkit/craftbukkit/entity/CraftPlayer.java
@@ -246,6 +246,7 @@ public class CraftPlayer extends CraftHumanEntity implements Player, PluginMessa
     private CraftWorldBorder clientWorldBorder = null;
     private BorderChangeListener clientWorldBorderListener = this.createWorldBorderListener();
     private long lastSaveTime; // Paper - getLastPlayed replacement API
+    private @Nullable CraftScoreboard scoreboardOverride;
 
     public CraftPlayer(CraftServer server, ServerPlayer entity) {
         super(server, entity);
@@ -2517,6 +2518,14 @@ public CraftScoreboard getScoreboard() {
         return this.server.getScoreboardManager().getPlayerBoard(this);
     }
 
+    public @Nullable CraftScoreboard getScoreboardOverride() {
+        return this.scoreboardOverride;
+    }
+
+    public void setScoreboardOverride(@Nullable CraftScoreboard scoreboardOverride) {
+        this.scoreboardOverride = scoreboardOverride;
+    }
+
     @Override
     public void setScoreboard(Scoreboard scoreboard) {
         Preconditions.checkArgument(scoreboard != null, "Scoreboard cannot be null");
diff --git a/paper-server/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboardManager.java b/paper-server/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboardManager.java
index 582cdc9e028c..9da0d9640874 100644
--- a/paper-server/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboardManager.java
+++ b/paper-server/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboardManager.java
@@ -23,7 +23,6 @@ public final class CraftScoreboardManager implements ScoreboardManager {
     private final CraftScoreboard mainScoreboard;
     private final MinecraftServer server;
     private final Collection<CraftScoreboard> scoreboards = new WeakCollection<>();
-    private final Map<CraftPlayer, CraftScoreboard> playerBoards = new HashMap<>();
 
     public CraftScoreboardManager(MinecraftServer server, net.minecraft.world.scores.Scoreboard scoreboard) {
         this.mainScoreboard = new CraftScoreboard(scoreboard);
@@ -54,7 +53,7 @@ public void registerScoreboardForVanilla(CraftScoreboard scoreboard) {
     }
 
     public CraftScoreboard getPlayerBoard(CraftPlayer player) {
-        CraftScoreboard board = this.playerBoards.get(player);
+        CraftScoreboard board = player.getScoreboardOverride();
         return board == null ? this.getMainScoreboard() : board;
     }
 
@@ -66,9 +65,9 @@ public void setPlayerBoard(CraftPlayer player, CraftScoreboard scoreboard) {
         }
 
         if (scoreboard == this.mainScoreboard) {
-            this.playerBoards.remove(player);
+            player.setScoreboardOverride(null);
         } else {
-            this.playerBoards.put(player, scoreboard);
+            player.setScoreboardOverride(scoreboard);
         }
 
         ServerPlayer serverPlayer = player.getHandle();
@@ -94,7 +93,7 @@ public void setPlayerBoard(CraftPlayer player, CraftScoreboard scoreboard) {
 
     // CraftBukkit method
     public void removePlayer(CraftPlayer player) {
-        this.playerBoards.remove(player);
+        player.setScoreboardOverride(null);
     }
 
     // CraftBukkit method