diff --git a/app/assets/javascripts/discourse/app/components/tag-groups-form.js b/app/assets/javascripts/discourse/app/components/tag-groups-form.js
index d9ba0fff3b..d701fb300c 100644
--- a/app/assets/javascripts/discourse/app/components/tag-groups-form.js
+++ b/app/assets/javascripts/discourse/app/components/tag-groups-form.js
@@ -107,6 +107,8 @@ export default Component.extend(bufferedProperty("model"), {
       this.allGroups.forEach((group) => {
         if (groupIds.includes(group.id)) {
           updatedPermissions[group.name] = PermissionType.FULL;
+        } else {
+          delete updatedPermissions[group.name];
         }
       });