diff --git a/src/Searching.js b/src/Searching.js index 9631afc36b..b1507e6a49 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -17,25 +17,71 @@ limitations under the License. import EventIndexPeg from "./indexing/EventIndexPeg"; import {MatrixClientPeg} from "./MatrixClientPeg"; -function serverSideSearch(term, roomId = undefined) { - let filter; - if (roomId !== undefined) { - // XXX: it's unintuitive that the filter for searching doesn't have - // the same shape as the v2 filter API :( - filter = { - rooms: [roomId], - }; - } +const SEARCH_LIMIT = 10; - const searchPromise = MatrixClientPeg.get().searchRoomEvents({ - filter, - term, - }); +async function serverSideSearch(term, roomId = undefined) { + const client = MatrixClientPeg.get(); - return searchPromise; + const filter = { + limit: SEARCH_LIMIT, + }; + + if (roomId !== undefined) filter.rooms = [roomId]; + + const body = { + search_categories: { + room_events: { + search_term: term, + filter: filter, + order_by: "recent", + event_context: { + before_limit: 1, + after_limit: 1, + include_profile: true, + }, + }, + }, + }; + + const response = await client.search({body: body}); + + const result = { + response: response, + query: body, + }; + + return result; +} + +async function serverSideSearchProcess(term, roomId = undefined) { + const client = MatrixClientPeg.get(); + const result = await serverSideSearch(term, roomId); + + // The js-sdk method backPaginateRoomEventsSearch() uses _query internally + // so we're reusing the concept here since we wan't to delegate the + // pagination back to backPaginateRoomEventsSearch() in some cases. + const searchResult = { + _query: result.query, + results: [], + highlights: [], + }; + + return client._processRoomEventsSearch(searchResult, result.response); +} + +function compareEvents(a, b) { + const aEvent = a.result; + const bEvent = b.result; + + if (aEvent.origin_server_ts > bEvent.origin_server_ts) return -1; + if (aEvent.origin_server_ts < bEvent.origin_server_ts) return 1; + + return 0; } async function combinedSearch(searchTerm) { + const client = MatrixClientPeg.get(); + // Create two promises, one for the local search, one for the // server-side search. const serverSidePromise = serverSideSearch(searchTerm); @@ -48,37 +94,59 @@ async function combinedSearch(searchTerm) { const localResult = await localPromise; const serverSideResult = await serverSidePromise; - // Combine the search results into one result. - const result = {}; + const serverQuery = serverSideResult.query; + const serverResponse = serverSideResult.response; - // Our localResult and serverSideResult are both ordered by - // recency separately, when we combine them the order might not - // be the right one so we need to sort them. - const compare = (a, b) => { - const aEvent = a.context.getEvent().event; - const bEvent = b.context.getEvent().event; + const localQuery = localResult.query; + const localResponse = localResult.response; - if (aEvent.origin_server_ts > - bEvent.origin_server_ts) return -1; - if (aEvent.origin_server_ts < - bEvent.origin_server_ts) return 1; - return 0; + // Store our queries for later on so we can support pagination. + // + // We're reusing _query here again to not introduce separate code paths and + // concepts for our different pagination methods. We're storing the + // server-side next batch separately since the query is the json body of + // the request and next_batch needs to be a query parameter. + // + // We can't put it in the final result that _processRoomEventsSearch() + // returns since that one can be either a server-side one, a local one or a + // fake one to fetch the remaining cached events. See the docs for + // combineEvents() for an explanation why we need to cache events. + const emptyResult = { + seshatQuery: localQuery, + _query: serverQuery, + serverSideNextBatch: serverResponse.next_batch, + cachedEvents: [], + oldestEventFrom: "server", + results: [], + highlights: [], }; - result.count = localResult.count + serverSideResult.count; - result.results = localResult.results.concat( - serverSideResult.results).sort(compare); - result.highlights = localResult.highlights.concat( - serverSideResult.highlights); + // Combine our results. + const combinedResult = combineResponses(emptyResult, localResponse, serverResponse.search_categories.room_events); + + // Let the client process the combined result. + const response = { + search_categories: { + room_events: combinedResult, + }, + }; + + const result = client._processRoomEventsSearch(emptyResult, response); + + // Restore our encryption info so we can properly re-verify the events. + restoreEncryptionInfo(result.results); return result; } -async function localSearch(searchTerm, roomId = undefined) { +async function localSearch(searchTerm, roomId = undefined, processResult = true) { + const eventIndex = EventIndexPeg.get(); + const searchArgs = { search_term: searchTerm, before_limit: 1, after_limit: 1, + limit: SEARCH_LIMIT, order_by_recency: true, room_id: undefined, }; @@ -87,6 +155,19 @@ async function localSearch(searchTerm, roomId = undefined) { searchArgs.room_id = roomId; } + const localResult = await eventIndex.search(searchArgs); + + searchArgs.next_batch = localResult.next_batch; + + const result = { + response: localResult, + query: searchArgs, + }; + + return result; +} + +async function localSearchProcess(searchTerm, roomId = undefined) { const emptyResult = { results: [], highlights: [], @@ -94,9 +175,34 @@ async function localSearch(searchTerm, roomId = undefined) { if (searchTerm === "") return emptyResult; + const result = await localSearch(searchTerm, roomId); + + emptyResult.seshatQuery = result.query; + + const response = { + search_categories: { + room_events: result.response, + }, + }; + + const processedResult = MatrixClientPeg.get()._processRoomEventsSearch(emptyResult, response); + // Restore our encryption info so we can properly re-verify the events. + restoreEncryptionInfo(processedResult.results); + + return processedResult; +} + +async function localPagination(searchResult) { const eventIndex = EventIndexPeg.get(); + const searchArgs = searchResult.seshatQuery; + const localResult = await eventIndex.search(searchArgs); + searchResult.seshatQuery.next_batch = localResult.next_batch; + + // We only need to restore the encryption state for the new results, so + // remember how many of them we got. + const newResultCount = localResult.results.length; const response = { search_categories: { @@ -104,15 +210,257 @@ async function localSearch(searchTerm, roomId = undefined) { }, }; - const result = MatrixClientPeg.get()._processRoomEventsSearch( - emptyResult, response); + const result = MatrixClientPeg.get()._processRoomEventsSearch(searchResult, response); // Restore our encryption info so we can properly re-verify the events. - for (let i = 0; i < result.results.length; i++) { - const timeline = result.results[i].context.getTimeline(); + const newSlice = result.results.slice(Math.max(result.results.length - newResultCount, 0)); + restoreEncryptionInfo(newSlice); + + searchResult.pendingRequest = null; + + return result; +} + +function compareOldestEvents(firstResults, secondResults) { + try { + const oldestFirstEvent = firstResults.results[firstResults.results.length - 1].result; + const oldestSecondEvent = secondResults.results[secondResults.results.length - 1].result; + + if (oldestFirstEvent.origin_server_ts <= oldestSecondEvent.origin_server_ts) { + return -1; + } else { + return 1; + } + } catch { + return 0; + } +} + +function combineEventSources(previousSearchResult, response, a, b) { + // Merge event sources and sort the events. + const combinedEvents = a.concat(b).sort(compareEvents); + // Put half of the events in the response, and cache the other half. + response.results = combinedEvents.slice(0, SEARCH_LIMIT); + previousSearchResult.cachedEvents = combinedEvents.slice(SEARCH_LIMIT); +} + +/** + * Combine the events from our event sources into a sorted result + * + * This method will first be called from the combinedSearch() method. In this + * case we will fetch SEARCH_LIMIT events from the server and the local index. + * + * The method will put the SEARCH_LIMIT newest events from the server and the + * local index in the results part of the response, the rest will be put in the + * cachedEvents field of the previousSearchResult (in this case an empty search + * result). + * + * Every subsequent call will be made from the combinedPagination() method, in + * this case we will combine the cachedEvents and the next SEARCH_LIMIT events + * from either the server or the local index. + * + * Since we have two event sources and we need to sort the results by date we + * need keep on looking for the oldest event. We are implementing a variation of + * a sliding window. + * + * The event sources are here represented as two sorted lists where the smallest + * number represents the newest event. The two lists need to be merged in a way + * that preserves the sorted property so they can be shown as one search result. + * We first fetch SEARCH_LIMIT events from both sources. + * + * If we set SEARCH_LIMIT to 3: + * + * Server events [01, 02, 04, 06, 07, 08, 11, 13] + * |01, 02, 04| + * Local events [03, 05, 09, 10, 12, 14, 15, 16] + * |03, 05, 09| + * + * We note that the oldest event is from the local index, and we combine the + * results: + * + * Server window [01, 02, 04] + * Local window [03, 05, 09] + * + * Combined events [01, 02, 03, 04, 05, 09] + * + * We split the combined result in the part that we want to present and a part + * that will be cached. + * + * Presented events [01, 02, 03] + * Cached events [04, 05, 09] + * + * We slide the window for the server since the oldest event is from the local + * index. + * + * Server events [01, 02, 04, 06, 07, 08, 11, 13] + * |06, 07, 08| + * Local events [03, 05, 09, 10, 12, 14, 15, 16] + * |XX, XX, XX| + * Cached events [04, 05, 09] + * + * We note that the oldest event is from the server and we combine the new + * server events with the cached ones. + * + * Cached events [04, 05, 09] + * Server events [06, 07, 08] + * + * Combined events [04, 05, 06, 07, 08, 09] + * + * We split again. + * + * Presented events [04, 05, 06] + * Cached events [07, 08, 09] + * + * We slide the local window, the oldest event is on the server. + * + * Server events [01, 02, 04, 06, 07, 08, 11, 13] + * |XX, XX, XX| + * Local events [03, 05, 09, 10, 12, 14, 15, 16] + * |10, 12, 14| + * + * Cached events [07, 08, 09] + * Local events [10, 12, 14] + * Combined events [07, 08, 09, 10, 12, 14] + * + * Presented events [07, 08, 09] + * Cached events [10, 12, 14] + * + * Next up we slide the server window again. + * + * Server events [01, 02, 04, 06, 07, 08, 11, 13] + * |11, 13| + * Local events [03, 05, 09, 10, 12, 14, 15, 16] + * |XX, XX, XX| + * + * Cached events [10, 12, 14] + * Server events [11, 13] + * Combined events [10, 11, 12, 13, 14] + * + * Presented events [10, 11, 12] + * Cached events [13, 14] + * + * We have one source exhausted, we fetch the rest of our events from the other + * source and combine it with our cached events. + * + * + * @param {object} previousSearchResult A search result from a previous search + * call. + * @param {object} localEvents An unprocessed search result from the event + * index. + * @param {object} serverEvents An unprocessed search result from the server. + * + * @return {object} A response object that combines the events from the + * different event sources. + * + */ +function combineEvents(previousSearchResult, localEvents = undefined, serverEvents = undefined) { + const response = {}; + + const cachedEvents = previousSearchResult.cachedEvents; + let oldestEventFrom = previousSearchResult.oldestEventFrom; + response.highlights = previousSearchResult.highlights; + + if (localEvents && serverEvents) { + // This is a first search call, combine the events from the server and + // the local index. Note where our oldest event came from, we shall + // fetch the next batch of events from the other source. + if (compareOldestEvents(localEvents, serverEvents) < 0) { + oldestEventFrom = "local"; + } + + combineEventSources(previousSearchResult, response, localEvents.results, serverEvents.results); + response.highlights = localEvents.highlights.concat(serverEvents.highlights); + } else if (localEvents) { + // This is a pagination call fetching more events from the local index, + // meaning that our oldest event was on the server. + // Change the source of the oldest event if our local event is older + // than the cached one. + if (compareOldestEvents(localEvents, cachedEvents) < 0) { + oldestEventFrom = "local"; + } + combineEventSources(previousSearchResult, response, localEvents.results, cachedEvents); + } else if (serverEvents) { + // This is a pagination call fetching more events from the server, + // meaning that our oldest event was in the local index. + // Change the source of the oldest event if our server event is older + // than the cached one. + if (compareOldestEvents(serverEvents, cachedEvents) < 0) { + oldestEventFrom = "server"; + } + combineEventSources(previousSearchResult, response, serverEvents.results, cachedEvents); + } else { + // This is a pagination call where we exhausted both of our event + // sources, let's push the remaining cached events. + response.results = cachedEvents; + previousSearchResult.cachedEvents = []; + } + + previousSearchResult.oldestEventFrom = oldestEventFrom; + + return response; +} + +/** + * Combine the local and server search responses + * + * @param {object} previousSearchResult A search result from a previous search + * call. + * @param {object} localEvents An unprocessed search result from the event + * index. + * @param {object} serverEvents An unprocessed search result from the server. + * + * @return {object} A response object that combines the events from the + * different event sources. + */ +function combineResponses(previousSearchResult, localEvents = undefined, serverEvents = undefined) { + // Combine our events first. + const response = combineEvents(previousSearchResult, localEvents, serverEvents); + + // Our first search will contain counts from both sources, subsequent + // pagination requests will fetch responses only from one of the sources, so + // reuse the first count when we're paginating. + if (previousSearchResult.count) { + response.count = previousSearchResult.count; + } else { + response.count = localEvents.count + serverEvents.count; + } + + // Update our next batch tokens for the given search sources. + if (localEvents) { + previousSearchResult.seshatQuery.next_batch = localEvents.next_batch; + } + if (serverEvents) { + previousSearchResult.serverSideNextBatch = serverEvents.next_batch; + } + + // Set the response next batch token to one of the tokens from the sources, + // this makes sure that if we exhaust one of the sources we continue with + // the other one. + if (previousSearchResult.seshatQuery.next_batch) { + response.next_batch = previousSearchResult.seshatQuery.next_batch; + } else if (previousSearchResult.serverSideNextBatch) { + response.next_batch = previousSearchResult.serverSideNextBatch; + } + + // We collected all search results from the server as well as from Seshat, + // we still have some events cached that we'll want to display on the next + // pagination request. + // + // Provide a fake next batch token for that case. + if (!response.next_batch && previousSearchResult.cachedEvents.length > 0) { + response.next_batch = "cached"; + } + + return response; +} + +function restoreEncryptionInfo(searchResultSlice) { + for (let i = 0; i < searchResultSlice.length; i++) { + const timeline = searchResultSlice[i].context.getTimeline(); for (let j = 0; j < timeline.length; j++) { const ev = timeline[j]; + if (ev.event.curve25519Key) { ev.makeEncrypted( "m.room.encrypted", @@ -129,6 +477,57 @@ async function localSearch(searchTerm, roomId = undefined) { } } } +} + +async function combinedPagination(searchResult) { + const eventIndex = EventIndexPeg.get(); + const client = MatrixClientPeg.get(); + + const searchArgs = searchResult.seshatQuery; + const oldestEventFrom = searchResult.oldestEventFrom; + + let localResult; + let serverSideResult; + + // Fetch events from the local index if we have a token for itand if it's + // the local indexes turn or the server has exhausted its results. + if (searchArgs.next_batch && (!searchResult.serverSideNextBatch || oldestEventFrom === "server")) { + localResult = await eventIndex.search(searchArgs); + } + + // Fetch events from the server if we have a token for it and if it's the + // local indexes turn or the local index has exhausted its results. + if (searchResult.serverSideNextBatch && (oldestEventFrom === "local" || !searchArgs.next_batch)) { + const body = {body: searchResult._query, next_batch: searchResult.serverSideNextBatch}; + serverSideResult = await client.search(body); + } + + let serverEvents; + + if (serverSideResult) { + serverEvents = serverSideResult.search_categories.room_events; + } + + // Combine our events. + const combinedResult = combineResponses(searchResult, localResult, serverEvents); + + const response = { + search_categories: { + room_events: combinedResult, + }, + }; + + const oldResultCount = searchResult.results.length; + + // Let the client process the combined result. + const result = client._processRoomEventsSearch(searchResult, response); + + // Restore our encryption info so we can properly re-verify the events. + const newResultCount = result.results.length - oldResultCount; + const newSlice = result.results.slice(Math.max(result.results.length - newResultCount, 0)); + restoreEncryptionInfo(newSlice); + + searchResult.pendingRequest = null; return result; } @@ -140,11 +539,11 @@ function eventIndexSearch(term, roomId = undefined) { if (MatrixClientPeg.get().isRoomEncrypted(roomId)) { // The search is for a single encrypted room, use our local // search method. - searchPromise = localSearch(term, roomId); + searchPromise = localSearchProcess(term, roomId); } else { // The search is for a single non-encrypted room, use the // server-side search. - searchPromise = serverSideSearch(term, roomId); + searchPromise = serverSideSearchProcess(term, roomId); } } else { // Search across all rooms, combine a server side search and a @@ -155,9 +554,45 @@ function eventIndexSearch(term, roomId = undefined) { return searchPromise; } +function eventIndexSearchPagination(searchResult) { + const client = MatrixClientPeg.get(); + + const seshatQuery = searchResult.seshatQuery; + const serverQuery = searchResult._query; + + if (!seshatQuery) { + // This is a search in a non-encrypted room. Do the normal server-side + // pagination. + return client.backPaginateRoomEventsSearch(searchResult); + } else if (!serverQuery) { + // This is a search in a encrypted room. Do a local pagination. + const promise = localPagination(searchResult); + searchResult.pendingRequest = promise; + + return promise; + } else { + // We have both queries around, this is a search across all rooms so a + // combined pagination needs to be done. + const promise = combinedPagination(searchResult); + searchResult.pendingRequest = promise; + + return promise; + } +} + +export function searchPagination(searchResult) { + const eventIndex = EventIndexPeg.get(); + const client = MatrixClientPeg.get(); + + if (searchResult.pendingRequest) return searchResult.pendingRequest; + + if (eventIndex === null) return client.backPaginateRoomEventsSearch(searchResult); + else return eventIndexSearchPagination(searchResult); +} + export default function eventSearch(term, roomId = undefined) { const eventIndex = EventIndexPeg.get(); - if (eventIndex === null) return serverSideSearch(term, roomId); + if (eventIndex === null) return serverSideSearchProcess(term, roomId); else return eventIndexSearch(term, roomId); } diff --git a/src/autocomplete/QueryMatcher.ts b/src/autocomplete/QueryMatcher.ts index 2c1899d813..7a0219e264 100644 --- a/src/autocomplete/QueryMatcher.ts +++ b/src/autocomplete/QueryMatcher.ts @@ -17,8 +17,6 @@ limitations under the License. */ import _at from 'lodash/at'; -import _flatMap from 'lodash/flatMap'; -import _sortBy from 'lodash/sortBy'; import _uniq from 'lodash/uniq'; function stripDiacritics(str: string): string { @@ -35,8 +33,9 @@ interface IOptions { /** * Simple search matcher that matches any results with the query string anywhere * in the search string. Returns matches in the order the query string appears - * in the search key, earliest first, then in the order the items appeared in - * the source array. + * in the search key, earliest first, then in the order the search key appears + * in the provided array of keys, then in the order the items appeared in the + * source array. * * @param {Object[]} objects Initial list of objects. Equivalent to calling * setObjects() after construction @@ -49,7 +48,7 @@ export default class QueryMatcher { private _options: IOptions; private _keys: IOptions["keys"]; private _funcs: Required["funcs"]>; - private _items: Map; + private _items: Map; constructor(objects: T[], options: IOptions = { keys: [] }) { this._options = options; @@ -85,13 +84,16 @@ export default class QueryMatcher { keyValues.push(f(object)); } - for (const keyValue of keyValues) { + for (const [index, keyValue] of Object.entries(keyValues)) { if (!keyValue) continue; // skip falsy keyValues const key = stripDiacritics(keyValue).toLowerCase(); if (!this._items.has(key)) { this._items.set(key, []); } - this._items.get(key).push(object); + this._items.get(key).push({ + keyWeight: Number(index), + object, + }); } } } @@ -104,32 +106,40 @@ export default class QueryMatcher { if (query.length === 0) { return []; } - const results = []; + const matches = []; // Iterate through the map & check each key. // ES6 Map iteration order is defined to be insertion order, so results // here will come out in the order they were put in. - for (const key of this._items.keys()) { + for (const [key, candidates] of this._items.entries()) { let resultKey = key; if (this._options.shouldMatchWordsOnly) { resultKey = resultKey.replace(/[^\w]/g, ''); } const index = resultKey.indexOf(query); if (index !== -1 && (!this._options.shouldMatchPrefix || index === 0)) { - results.push({key, index}); + matches.push( + ...candidates.map((candidate) => ({index, ...candidate})) + ); } } - // Sort them by where the query appeared in the search key - // lodash sortBy is a stable sort, so results where the query - // appeared in the same place will retain their order with - // respect to each other. - const sortedResults = _sortBy(results, (candidate) => { - return candidate.index; + // Sort matches by where the query appeared in the search key, then by + // where the matched key appeared in the provided array of keys. + matches.sort((a, b) => { + if (a.index < b.index) { + return -1; + } else if (a.index === b.index) { + if (a.keyWeight < b.keyWeight) { + return -1; + } else if (a.keyWeight === b.keyWeight) { + return 0; + } + } + + return 1; }); - // Now map the keys to the result objects. Each result object is a list, so - // flatMap will flatten those lists out into a single list. Also remove any - // duplicates. - return _uniq(_flatMap(sortedResults, (candidate) => this._items.get(candidate.key))); + // Now map the keys to the result objects. Also remove any duplicates. + return _uniq(matches.map((match) => match.object)); } } diff --git a/src/components/structures/RoomView.js b/src/components/structures/RoomView.js index 4a0cc470d5..b73f2d2cde 100644 --- a/src/components/structures/RoomView.js +++ b/src/components/structures/RoomView.js @@ -39,7 +39,7 @@ import Tinter from '../../Tinter'; import rate_limited_func from '../../ratelimitedfunc'; import * as ObjectUtils from '../../ObjectUtils'; import * as Rooms from '../../Rooms'; -import eventSearch from '../../Searching'; +import eventSearch, {searchPagination} from '../../Searching'; import {isOnlyCtrlOrCmdIgnoreShiftKeyEvent, isOnlyCtrlOrCmdKeyEvent, Key} from '../../Keyboard'; @@ -1036,8 +1036,7 @@ export default createReactClass({ if (this.state.searchResults.next_batch) { debuglog("requesting more search results"); - const searchPromise = this.context.backPaginateRoomEventsSearch( - this.state.searchResults); + const searchPromise = searchPagination(this.state.searchResults); return this._handleSearchResult(searchPromise); } else { debuglog("no more search results"); @@ -1314,6 +1313,14 @@ export default createReactClass({ const mxEv = result.context.getEvent(); const roomId = mxEv.getRoomId(); const room = this.context.getRoom(roomId); + if (!room) { + // if we do not have the room in js-sdk stores then hide it as we cannot easily show it + // As per the spec, an all rooms search can create this condition, + // it happens with Seshat but not Synapse. + // It will make the result count not match the displayed count. + console.log("Hiding search result from an unknown room", roomId); + continue; + } if (!haveTileForEvent(mxEv)) { // XXX: can this ever happen? It will make the result count @@ -1322,16 +1329,9 @@ export default createReactClass({ } if (this.state.searchScope === 'All') { - if (roomId != lastRoomId) { - - // XXX: if we've left the room, we might not know about - // it. We should tell the js sdk to go and find out about - // it. But that's not an issue currently, as synapse only - // returns results for rooms we're joined to. - const roomName = room ? room.name : _t("Unknown room %(roomId)s", { roomId: roomId }); - + if (roomId !== lastRoomId) { ret.push(
  • -

    { _t("Room") }: { roomName }

    +

    { _t("Room") }: { room.name }

  • ); lastRoomId = roomId; } diff --git a/src/components/views/rooms/NotificationBadge.tsx b/src/components/views/rooms/NotificationBadge.tsx index af5a84ed92..b742f8e8e7 100644 --- a/src/components/views/rooms/NotificationBadge.tsx +++ b/src/components/views/rooms/NotificationBadge.tsx @@ -240,6 +240,7 @@ export class ListNotificationState extends EventEmitter implements IDestroyable this.rooms = rooms; for (const oldRoom of diff.removed) { const state = this.states[oldRoom.roomId]; + if (!state) continue; // We likely just didn't have a badge (race condition) delete this.states[oldRoom.roomId]; state.off(NOTIFICATION_STATE_UPDATE, this.onRoomNotificationStateUpdate); state.destroy(); diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 4a77da0149..945aee6382 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -433,7 +433,7 @@ "Render simple counters in room header": "Render simple counters in room header", "Multiple integration managers": "Multiple integration managers", "Try out new ways to ignore people (experimental)": "Try out new ways to ignore people (experimental)", - "Use the improved room list (in development - will refresh to apply changes)": "Use the improved room list (in development - will refresh to apply changes)", + "Use the improved room list (will refresh to apply changes)": "Use the improved room list (will refresh to apply changes)", "Support adding custom themes": "Support adding custom themes", "Use IRC layout": "Use IRC layout", "Show info about bridges in room settings": "Show info about bridges in room settings", @@ -2038,7 +2038,6 @@ "Search failed": "Search failed", "Server may be unavailable, overloaded, or search timed out :(": "Server may be unavailable, overloaded, or search timed out :(", "No more results": "No more results", - "Unknown room %(roomId)s": "Unknown room %(roomId)s", "Room": "Room", "Failed to reject invite": "Failed to reject invite", "You have %(count)s unread notifications in a prior version of this room.|other": "You have %(count)s unread notifications in a prior version of this room.", diff --git a/src/indexing/BaseEventIndexManager.ts b/src/indexing/BaseEventIndexManager.ts index c40d1300ea..be7b89be37 100644 --- a/src/indexing/BaseEventIndexManager.ts +++ b/src/indexing/BaseEventIndexManager.ts @@ -134,6 +134,19 @@ export default abstract class BaseEventIndexManager { throw new Error("Unimplemented"); } + /** + * Check if the room with the given id is already indexed. + * + * @param {string} roomId The ID of the room which we want to check if it + * has been already indexed. + * + * @return {Promise} Returns true if the index contains events for + * the given room, false otherwise. + */ + isRoomIndexed(roomId: string): Promise { + throw new Error("Unimplemented"); + } + /** * Get statistical information of the index. * @@ -144,6 +157,29 @@ export default abstract class BaseEventIndexManager { throw new Error("Unimplemented"); } + + /** + * Get the user version of the database. + * @return {Promise} A promise that will resolve to the user stored + * version number. + */ + async getUserVersion(): Promise { + throw new Error("Unimplemented"); + } + + /** + * Set the user stored version to the given version number. + * + * @param {number} version The new version that should be stored in the + * database. + * + * @return {Promise} A promise that will resolve once the new version + * is stored. + */ + async setUserVersion(version: number): Promise { + throw new Error("Unimplemented"); + } + /** * Commit the previously queued up events to the index. * diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index d4e8ab0117..1dc31869c9 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -42,9 +42,6 @@ export default class EventIndex extends EventEmitter { async init() { const indexManager = PlatformPeg.get().getEventIndexingManager(); - await indexManager.initEventIndex(); - console.log("EventIndex: Successfully initialized the event index"); - this.crawlerCheckpoints = await indexManager.loadCheckpoints(); console.log("EventIndex: Loaded checkpoints", this.crawlerCheckpoints); @@ -62,6 +59,7 @@ export default class EventIndex extends EventEmitter { client.on('Event.decrypted', this.onEventDecrypted); client.on('Room.timelineReset', this.onTimelineReset); client.on('Room.redaction', this.onRedaction); + client.on('RoomState.events', this.onRoomStateEvent); } /** @@ -76,6 +74,7 @@ export default class EventIndex extends EventEmitter { client.removeListener('Event.decrypted', this.onEventDecrypted); client.removeListener('Room.timelineReset', this.onTimelineReset); client.removeListener('Room.redaction', this.onRedaction); + client.removeListener('RoomState.events', this.onRoomStateEvent); } /** @@ -194,6 +193,15 @@ export default class EventIndex extends EventEmitter { } } + onRoomStateEvent = async (ev, state) => { + if (!MatrixClientPeg.get().isRoomEncrypted(state.roomId)) return; + + if (ev.getType() === "m.room.encryption" && !await this.isRoomIndexed(state.roomId)) { + console.log("EventIndex: Adding a checkpoint for a newly encrypted room", state.roomId); + this.addRoomCheckpoint(state.roomId, true); + } + } + /* * The Event.decrypted listener. * @@ -234,26 +242,12 @@ export default class EventIndex extends EventEmitter { */ onTimelineReset = async (room, timelineSet, resetAllTimelines) => { if (room === null) return; - - const indexManager = PlatformPeg.get().getEventIndexingManager(); if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; - const timeline = room.getLiveTimeline(); - const token = timeline.getPaginationToken("b"); + console.log("EventIndex: Adding a checkpoint because of a limited timeline", + room.roomId); - const backwardsCheckpoint = { - roomId: room.roomId, - token: token, - fullCrawl: false, - direction: "b", - }; - - console.log("EventIndex: Added checkpoint because of a limited timeline", - backwardsCheckpoint); - - await indexManager.addCrawlerCheckpoint(backwardsCheckpoint); - - this.crawlerCheckpoints.push(backwardsCheckpoint); + this.addRoomCheckpoint(room.roomId, false); } /** @@ -334,7 +328,7 @@ export default class EventIndex extends EventEmitter { avatar_url: ev.sender.getMxcAvatarUrl(), }; - indexManager.addEventToIndex(e, profile); + await indexManager.addEventToIndex(e, profile); } /** @@ -345,6 +339,51 @@ export default class EventIndex extends EventEmitter { this.emit("changedCheckpoint", this.currentRoom()); } + async addEventsFromLiveTimeline(timeline) { + const events = timeline.getEvents(); + + for (let i = 0; i < events.length; i++) { + const ev = events[i]; + await this.addLiveEventToIndex(ev); + } + } + + async addRoomCheckpoint(roomId, fullCrawl = false) { + const indexManager = PlatformPeg.get().getEventIndexingManager(); + const client = MatrixClientPeg.get(); + const room = client.getRoom(roomId); + + if (!room) return; + + const timeline = room.getLiveTimeline(); + const token = timeline.getPaginationToken("b"); + + if (!token) { + // The room doesn't contain any tokens, meaning the live timeline + // contains all the events, add those to the index. + await this.addEventsFromLiveTimeline(timeline); + return; + } + + const checkpoint = { + roomId: room.roomId, + token: token, + fullCrawl: fullCrawl, + direction: "b", + }; + + console.log("EventIndex: Adding checkpoint", checkpoint); + + try { + await indexManager.addCrawlerCheckpoint(checkpoint); + } catch (e) { + console.log("EventIndex: Error adding new checkpoint for room", + room.roomId, checkpoint, e); + } + + this.crawlerCheckpoints.push(checkpoint); + } + /** * The main crawler loop. * @@ -833,6 +872,20 @@ export default class EventIndex extends EventEmitter { return indexManager.getStats(); } + /** + * Check if the room with the given id is already indexed. + * + * @param {string} roomId The ID of the room which we want to check if it + * has been already indexed. + * + * @return {Promise} Returns true if the index contains events for + * the given room, false otherwise. + */ + async isRoomIndexed(roomId) { + const indexManager = PlatformPeg.get().getEventIndexingManager(); + return indexManager.isRoomIndexed(roomId); + } + /** * Get the room that we are currently crawling. * diff --git a/src/indexing/EventIndexPeg.js b/src/indexing/EventIndexPeg.js index ae4c14dafd..20e05f985d 100644 --- a/src/indexing/EventIndexPeg.js +++ b/src/indexing/EventIndexPeg.js @@ -23,6 +23,8 @@ import PlatformPeg from "../PlatformPeg"; import EventIndex from "../indexing/EventIndex"; import SettingsStore, {SettingLevel} from '../settings/SettingsStore'; +const INDEX_VERSION = 1; + class EventIndexPeg { constructor() { this.index = null; @@ -66,8 +68,25 @@ class EventIndexPeg { */ async initEventIndex() { const index = new EventIndex(); + const indexManager = PlatformPeg.get().getEventIndexingManager(); try { + await indexManager.initEventIndex(); + + const userVersion = await indexManager.getUserVersion(); + const eventIndexIsEmpty = await indexManager.isEventIndexEmpty(); + + if (eventIndexIsEmpty) { + await indexManager.setUserVersion(INDEX_VERSION); + } else if (userVersion === 0 && !eventIndexIsEmpty) { + await indexManager.closeEventIndex(); + await this.deleteEventIndex(); + + await indexManager.initEventIndex(); + await indexManager.setUserVersion(INDEX_VERSION); + } + + console.log("EventIndex: Successfully initialized the event index"); await index.init(); } catch (e) { console.log("EventIndex: Error initializing the event index", e); diff --git a/src/settings/Settings.js b/src/settings/Settings.js index 5e439a1d71..5715909da3 100644 --- a/src/settings/Settings.js +++ b/src/settings/Settings.js @@ -140,7 +140,7 @@ export const SETTINGS = { }, "feature_new_room_list": { isFeature: true, - displayName: _td("Use the improved room list (in development - will refresh to apply changes)"), + displayName: _td("Use the improved room list (will refresh to apply changes)"), supportedLevels: LEVELS_FEATURE, default: false, controller: new ReloadOnChangeController(), diff --git a/test/autocomplete/QueryMatcher-test.js b/test/autocomplete/QueryMatcher-test.js index 03f28eb984..2d0e10563b 100644 --- a/test/autocomplete/QueryMatcher-test.js +++ b/test/autocomplete/QueryMatcher-test.js @@ -81,7 +81,34 @@ describe('QueryMatcher', function() { expect(reverseResults[1].name).toBe('Victoria'); }); - it('Returns results with search string in same place in insertion order', function() { + it('Returns results with search string in same place according to key index', function() { + const objects = [ + { name: "a", first: "hit", second: "miss", third: "miss" }, + { name: "b", first: "miss", second: "hit", third: "miss" }, + { name: "c", first: "miss", second: "miss", third: "hit" }, + ]; + const qm = new QueryMatcher(objects, {keys: ["second", "first", "third"]}); + const results = qm.match('hit'); + + expect(results.length).toBe(3); + expect(results[0].name).toBe('b'); + expect(results[1].name).toBe('a'); + expect(results[2].name).toBe('c'); + + + qm.setObjects(objects.slice().reverse()); + + const reverseResults = qm.match('hit'); + + // should still be in the same order: key index + // takes precedence over input order + expect(reverseResults.length).toBe(3); + expect(reverseResults[0].name).toBe('b'); + expect(reverseResults[1].name).toBe('a'); + expect(reverseResults[2].name).toBe('c'); + }); + + it('Returns results with search string in same place and key in same place in insertion order', function() { const qm = new QueryMatcher(OBJECTS, {keys: ["name"]}); const results = qm.match('Mel'); @@ -132,9 +159,9 @@ describe('QueryMatcher', function() { const results = qm.match('Emma'); expect(results.length).toBe(3); - expect(results[0].name).toBe('Mel B'); - expect(results[1].name).toBe('Mel C'); - expect(results[2].name).toBe('Emma'); + expect(results[0].name).toBe('Emma'); + expect(results[1].name).toBe('Mel B'); + expect(results[2].name).toBe('Mel C'); }); it('Matches words only by default', function() {