diff --git a/src/components/views/rooms/RoomList.js b/src/components/views/rooms/RoomList.js
index 6157f65c9d..fbb563cf14 100644
--- a/src/components/views/rooms/RoomList.js
+++ b/src/components/views/rooms/RoomList.js
@@ -71,6 +71,8 @@ module.exports = React.createClass({
         this.mounted = false;
 
         const cli = MatrixClientPeg.get();
+        const dmRoomMap = new DMRoomMap(cli);
+
         cli.on("Room", this.onRoom);
         cli.on("deleteRoom", this.onDeleteRoom);
         cli.on("Room.timeline", this.onRoomTimeline);
@@ -84,8 +86,8 @@ module.exports = React.createClass({
         cli.on("Group.myMembership", this._onGroupMyMembership);
 
         this._groupStores = {};
+        this._selectedTagsRoomIdsForGroup = {};
         this._selectedTagsRoomIds = [];
-        this._selectedTagsUserIds = [];
         // When the selected tags are changed, initialise a group store if necessary
         this._filterStoreToken = FilterStore.addListener(() => {
             FilterStore.getSelectedTags().forEach((tag) => {
@@ -94,10 +96,12 @@ module.exports = React.createClass({
                 }
                 this._groupStores[tag] = GroupStoreCache.getGroupStore(tag);
                 this._groupStores[tag].registerListener(() => {
-                    this.updateSelectedTagsEntities();
+                    // This group's rooms or members may have updated, update rooms for its tag
+                    this.updateSelectedTagsRooms(dmRoomMap, [tag]);
                 });
             });
-            this.updateSelectedTagsEntities();
+            // Filters themselves have changed, refresh the selected tags
+            this.updateSelectedTagsRooms(dmRoomMap, FilterStore.getSelectedTags());
         });
 
         this.refreshRoomList();
@@ -260,20 +264,31 @@ module.exports = React.createClass({
     }, 500),
 
     // Update which rooms and users should appear in RoomList as dictated by selected tags
-    updateSelectedTagsEntities: function() {
+    updateSelectedTagsRooms: function(dmRoomMap, updatedTags) {
         if (!this.mounted) return;
-        this._selectedTagsRoomIds = [];
-        this._selectedTagsUserIds = [];
-        FilterStore.getSelectedTags().forEach((tag) => {
-            this._selectedTagsRoomIds = this._selectedTagsRoomIds.concat(
-               this._groupStores[tag].getGroupRooms().map((room) => room.roomId),
-            );
-            // TODO: Check if room has been tagged to the group by the user
+        updatedTags.forEach((tag) => {
+            // For now, only handle group tags
+            const store = this._groupStores[tag];
+            if (!store) return;
 
-            this._selectedTagsUserIds = this._selectedTagsUserIds.concat(
-               this._groupStores[tag].getGroupMembers().map((member) => member.userId),
+            this._selectedTagsRoomIdsForGroup[tag] = [];
+            store.getGroupRooms().forEach((room) => this._selectedTagsRoomIdsForGroup[tag].push(room.roomId));
+            store.getGroupMembers().forEach((member) => {
+                if (member.userId === MatrixClientPeg.get().credentials.userId) return;
+                dmRoomMap.getDMRoomsForUserId(member.userId).forEach(
+                    (roomId) => this._selectedTagsRoomIdsForGroup[tag].push(roomId),
+                );
+            });
+            // TODO: Check if room has been tagged to the group by the user
+        });
+
+        this._selectedTagsRoomIds = [];
+        FilterStore.getSelectedTags().forEach((tag) => {
+            (this._selectedTagsRoomIdsForGroup[tag] || []).forEach(
+                (roomId) => this._selectedTagsRoomIds.push(roomId),
             );
         });
+
         this.setState({
             selectedTags: FilterStore.getSelectedTags(),
         }, () => {
@@ -281,17 +296,9 @@ module.exports = React.createClass({
         });
     },
 
-    isRoomInSelectedTags: function(room, me, dmRoomMap) {
+    isRoomInSelectedTags: function(room) {
         // No selected tags = every room is visible in the list
-        if (this.state.selectedTags.length === 0) {
-            return true;
-        }
-        if (this._selectedTagsRoomIds.includes(room.roomId)) {
-            return true;
-        }
-        const dmUserId = dmRoomMap.getUserIdForRoomId(room.roomId);
-        return dmUserId && dmUserId !== me.userId &&
-            this._selectedTagsUserIds.includes(dmUserId);
+        return this.state.selectedTags.length === 0 || this._selectedTagsRoomIds.includes(room.roomId);
     },
 
     refreshRoomList: function() {
@@ -341,7 +348,7 @@ module.exports = React.createClass({
                 const tagNames = Object.keys(room.tags);
 
                 // Apply TagPanel filtering, derived from FilterStore
-                if (!this.isRoomInSelectedTags(room, me, dmRoomMap)) {
+                if (!this.isRoomInSelectedTags(room)) {
                     return;
                 }