Merge branch 'poljar/seshat-ui-pr' into develop

res/css/_common.scss

@@ -428,6 +428,11 @@ input[type=text]:focus, input[type=password]:focus, textarea:focus {
     color: $accent-fg-color;
 }
 
+.mx_Dialog button.warning, .mx_Dialog input[type="submit"].warning {
+    border: solid 1px $warning-color;
+    color: $warning-color;
+}
+
 .mx_Dialog button:disabled, .mx_Dialog input[type="submit"]:disabled, .mx_Dialog_buttons button:disabled, .mx_Dialog_buttons input[type="submit"]:disabled {
     background-color: $light-fg-color;
     border: solid 1px $light-fg-color;

src/async-components/views/dialogs/eventindex/DisableEventIndexDialog.js

@@ -0,0 +1,73 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from 'react';
+import * as sdk from '../../../../index';
+import PropTypes from 'prop-types';
+import dis from "../../../../dispatcher";
+import { _t } from '../../../../languageHandler';
+
+import SettingsStore, {SettingLevel} from "../../../../settings/SettingsStore";
+import EventIndexPeg from "../../../../indexing/EventIndexPeg";
+
+/*
+ * Allows the user to disable the Event Index.
+ */
+export default class DisableEventIndexDialog extends React.Component {
+    static propTypes = {
+        onFinished: PropTypes.func.isRequired,
+    }
+
+    constructor(props) {
+        super(props);
+
+        this.state = {
+            disabling: false,
+        };
+    }
+
+    _onDisable = async () => {
+        this.setState({
+            disabling: true,
+        });
+
+        await SettingsStore.setValue('enableEventIndexing', null, SettingLevel.DEVICE, false);
+        await EventIndexPeg.deleteEventIndex();
+        this.props.onFinished();
+        dis.dispatch({ action: 'view_user_settings' });
+    }
+
+    render() {
+        const BaseDialog = sdk.getComponent('views.dialogs.BaseDialog');
+        const Spinner = sdk.getComponent('elements.Spinner');
+        const DialogButtons = sdk.getComponent('views.elements.DialogButtons');
+
+        return (
+            <BaseDialog onFinished={this.props.onFinished} title={_t("Are you sure?")}>
+                {_t("If disabled, messages from encrypted rooms won't appear in search results.")}
+                {this.state.disabling ? <Spinner /> : <div />}
+                <DialogButtons
+                    primaryButton={_t('Disable')}
+                    onPrimaryButtonClick={this._onDisable}
+                    primaryButtonClass="danger"
+                    cancelButtonClass="warning"
+                    onCancel={this.props.onFinished}
+                    disabled={this.state.disabling}
+                />
+            </BaseDialog>
+        );
+    }
+}

src/async-components/views/dialogs/eventindex/ManageEventIndexDialog.js

@@ -0,0 +1,154 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from 'react';
+import * as sdk from '../../../../index';
+import PropTypes from 'prop-types';
+import { _t } from '../../../../languageHandler';
+
+import Modal from '../../../../Modal';
+import {formatBytes, formatCountLong} from "../../../../utils/FormattingUtils";
+import EventIndexPeg from "../../../../indexing/EventIndexPeg";
+
+/*
+ * Allows the user to introspect the event index state and disable it.
+ */
+export default class ManageEventIndexDialog extends React.Component {
+    static propTypes = {
+        onFinished: PropTypes.func.isRequired,
+    }
+
+    constructor(props) {
+        super(props);
+
+        this.state = {
+            eventIndexSize: 0,
+            eventCount: 0,
+            roomCount: 0,
+            currentRoom: null,
+        };
+    }
+
+    async updateCurrentRoom(room) {
+        const eventIndex = EventIndexPeg.get();
+        const stats = await eventIndex.getStats();
+        let currentRoom = null;
+
+        if (room) currentRoom = room.name;
+
+        this.setState({
+            eventIndexSize: stats.size,
+            roomCount: stats.roomCount,
+            eventCount: stats.eventCount,
+            currentRoom: currentRoom,
+        });
+    }
+
+    componentWillUnmount(): void {
+        const eventIndex = EventIndexPeg.get();
+
+        if (eventIndex !== null) {
+            eventIndex.removeListener("changedCheckpoint", this.updateCurrentRoom.bind(this));
+        }
+    }
+
+    async componentWillMount(): void {
+        let eventIndexSize = 0;
+        let roomCount = 0;
+        let eventCount = 0;
+        let currentRoom = null;
+
+        const eventIndex = EventIndexPeg.get();
+
+        if (eventIndex !== null) {
+            eventIndex.on("changedCheckpoint", this.updateCurrentRoom.bind(this));
+
+            const stats = await eventIndex.getStats();
+            eventIndexSize = stats.size;
+            roomCount = stats.roomCount;
+            eventCount = stats.eventCount;
+
+            const room = eventIndex.currentRoom();
+            if (room) currentRoom = room.name;
+        }
+
+        this.setState({
+            eventIndexSize,
+            eventCount,
+            roomCount,
+            currentRoom,
+        });
+    }
+
+    _onDisable = async () => {
+        Modal.createTrackedDialogAsync("Disable message search", "Disable message search",
+            import("./DisableEventIndexDialog"),
+            null, null, /* priority = */ false, /* static = */ true,
+        );
+    }
+
+    _onDone = () => {
+        this.props.onFinished(true);
+    }
+
+    render() {
+        let crawlerState;
+
+        if (this.state.currentRoom === null) {
+            crawlerState = _t("Not currently downloading messages for any room.");
+        } else {
+            crawlerState = (
+                    _t("Downloading mesages for %(currentRoom)s.", { currentRoom: this.state.currentRoom })
+            );
+        }
+
+        const eventIndexingSettings = (
+            <div>
+                {
+                    _t( "Riot is securely caching encrypted messages locally for them " +
+                        "to appear in search results:",
+                    )
+                }
+                <div className='mx_SettingsTab_subsectionText'>
+                    {_t("Space used:")} {formatBytes(this.state.eventIndexSize, 0)}<br />
+                    {_t("Indexed messages:")} {formatCountLong(this.state.eventCount)}<br />
+                    {_t("Number of rooms:")} {formatCountLong(this.state.roomCount)}<br />
+                    {crawlerState}<br />
+                </div>
+            </div>
+        );
+
+        const BaseDialog = sdk.getComponent('views.dialogs.BaseDialog');
+        const DialogButtons = sdk.getComponent('views.elements.DialogButtons');
+
+        return (
+            <BaseDialog className='mx_ManageEventIndexDialog'
+                onFinished={this.props.onFinished}
+                title={_t("Message search")}
+            >
+                {eventIndexingSettings}
+                <DialogButtons
+                    primaryButton={_t("Done")}
+                    onPrimaryButtonClick={this.props.onFinished}
+                    primaryButtonClass="primary"
+                    cancelButton={_t("Disable")}
+                    onCancel={this._onDisable}
+                    cancelButtonClass="danger"
+                />
+            </BaseDialog>
+        );
+    }
+}

src/components/views/elements/DialogButtons.js

@@ -43,6 +43,10 @@ export default createReactClass({
         // should there be a cancel button? default: true
         hasCancel: PropTypes.bool,
 
+        // The class of the cancel button, only used if a cancel button is
+        // enabled
+        cancelButtonClass: PropTypes.node,
+
         // onClick handler for the cancel button.
         onCancel: PropTypes.func,
 
@@ -72,12 +76,14 @@ export default createReactClass({
             primaryButtonClassName += " " + this.props.primaryButtonClass;
         }
         let cancelButton;
+
         if (this.props.cancelButton || this.props.hasCancel) {
             cancelButton = <button
                 // important: the default type is 'submit' and this button comes before the
                 // primary in the DOM so will get form submissions unless we make it not a submit.
                 type="button"
                 onClick={this._onCancelClick}
+		className={this.props.cancelButtonClass}
                 disabled={this.props.disabled}
             >
                 { this.props.cancelButton || _t("Cancel") }

src/components/views/settings/EventIndexPanel.js

@@ -0,0 +1,187 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from 'react';
+
+import { _t } from '../../../languageHandler';
+import * as sdk from '../../../index';
+import Modal from '../../../Modal';
+import SettingsStore, {SettingLevel} from "../../../settings/SettingsStore";
+import AccessibleButton from "../elements/AccessibleButton";
+import {formatBytes, formatCountLong} from "../../../utils/FormattingUtils";
+import EventIndexPeg from "../../../indexing/EventIndexPeg";
+
+export default class EventIndexPanel extends React.Component {
+    constructor() {
+        super();
+
+        this.state = {
+            enabling: false,
+            eventIndexSize: 0,
+            roomCount: 0,
+            eventIndexingEnabled:
+                SettingsStore.getValueAt(SettingLevel.DEVICE, 'enableEventIndexing'),
+        };
+    }
+
+    async updateCurrentRoom(room) {
+        const eventIndex = EventIndexPeg.get();
+        const stats = await eventIndex.getStats();
+
+        this.setState({
+            eventIndexSize: stats.size,
+            roomCount: stats.roomCount,
+        });
+    }
+
+    componentWillUnmount(): void {
+        const eventIndex = EventIndexPeg.get();
+
+        if (eventIndex !== null) {
+            eventIndex.removeListener("changedCheckpoint", this.updateCurrentRoom.bind(this));
+        }
+    }
+
+    async componentWillMount(): void {
+        this.updateState();
+    }
+
+    async updateState() {
+        const eventIndex = EventIndexPeg.get();
+        const eventIndexingEnabled = SettingsStore.getValueAt(SettingLevel.DEVICE, 'enableEventIndexing');
+        const enabling = false;
+
+        let eventIndexSize = 0;
+        let roomCount = 0;
+
+        if (eventIndex !== null) {
+            eventIndex.on("changedCheckpoint", this.updateCurrentRoom.bind(this));
+
+            const stats = await eventIndex.getStats();
+            eventIndexSize = stats.size;
+            roomCount = stats.roomCount;
+        }
+
+        this.setState({
+            enabling,
+            eventIndexSize,
+            roomCount,
+            eventIndexingEnabled,
+        });
+    }
+
+    _onManage = async () => {
+        Modal.createTrackedDialogAsync('Message search', 'Message search',
+            import('../../../async-components/views/dialogs/eventindex/ManageEventIndexDialog'),
+            {
+                onFinished: () => {},
+            }, null, /* priority = */ false, /* static = */ true,
+        );
+    }
+
+    _onEnable = async () => {
+        this.setState({
+            enabling: true,
+        });
+
+        await EventIndexPeg.initEventIndex();
+        await EventIndexPeg.get().addInitialCheckpoints();
+        await EventIndexPeg.get().startCrawler();
+        await SettingsStore.setValue('enableEventIndexing', null, SettingLevel.DEVICE, true);
+        await this.updateState();
+    }
+
+    render() {
+        let eventIndexingSettings = null;
+        const InlineSpinner = sdk.getComponent('elements.InlineSpinner');
+
+        if (EventIndexPeg.get() !== null) {
+            eventIndexingSettings = (
+                <div>
+                    <div className='mx_SettingsTab_subsectionText'>
+                        {_t( "Securely cache encrypted messages locally for them " +
+                             "to appear in search results, using ")
+                        } {formatBytes(this.state.eventIndexSize, 0)}
+                        {_t( " to store messages from ")}
+                        {formatCountLong(this.state.roomCount)} {_t("rooms.")}
+                    </div>
+                    <div>
+                        <AccessibleButton kind="primary" onClick={this._onManage}>
+                            {_t("Manage")}
+                        </AccessibleButton>
+                    </div>
+                </div>
+            );
+        } else if (!this.state.eventIndexingEnabled && EventIndexPeg.supportIsInstalled()) {
+            eventIndexingSettings = (
+                <div>
+                    <div className='mx_SettingsTab_subsectionText'>
+                        {_t( "Securely cache encrypted messages locally for them to " +
+                             "appear in search results.")}
+                    </div>
+                    <div>
+                        <AccessibleButton kind="primary" disabled={this.state.enabling}
+                            onClick={this._onEnable}>
+                            {_t("Enable")}
+                        </AccessibleButton>
+                        {this.state.enabling ? <InlineSpinner /> : <div />}
+                    </div>
+                </div>
+            );
+        } else if (EventIndexPeg.platformHasSupport() && !EventIndexPeg.supportIsInstalled()) {
+            const nativeLink = (
+                "https://github.com/vector-im/riot-web/blob/develop/" +
+                "docs/native-node-modules.md#" +
+                "adding-seshat-for-search-in-e2e-encrypted-rooms"
+            );
+
+            eventIndexingSettings = (
+                <div>
+                    {
+                        _t( "Riot is missing some components required for securely " +
+                            "caching encrypted messages locally. If you'd like to " +
+                            "experiment with this feature, build a custom Riot Desktop " +
+                            "with <nativeLink>search components added</nativeLink>.",
+                            {},
+                            {
+                                'nativeLink': (sub) => <a href={nativeLink} target="_blank"
+                                    rel="noopener">{sub}</a>,
+                            },
+                        )
+                    }
+                </div>
+            );
+        } else {
+            eventIndexingSettings = (
+                <div>
+                    {
+                        _t( "Riot can't securely cache encrypted messages locally " +
+                            "while running in a web browser. Use <riotLink>Riot Desktop</riotLink> " +
+                            "for encrypted messages to appear in search results.",
+                            {},
+                            {
+                                'riotLink': (sub) => <a href="https://riot.im/download/desktop"
+                                    target="_blank" rel="noopener">{sub}</a>,
+                            },
+                        )
+                    }
+                </div>
+            );
+        }
+
+        return eventIndexingSettings;
+    }
+}

src/components/views/settings/tabs/user/PreferencesUserSettingsTab.js

@@ -170,6 +170,7 @@ export default class PreferencesUserSettingsTab extends React.Component {
         return (
             <div className="mx_SettingsTab mx_PreferencesUserSettingsTab">
                 <div className="mx_SettingsTab_heading">{_t("Preferences")}</div>
+
                 <div className="mx_SettingsTab_section">
                     <span className="mx_SettingsTab_subheading">{_t("Composer")}</span>
                     {this._renderGroup(PreferencesUserSettingsTab.COMPOSER_SETTINGS)}

src/components/views/settings/tabs/user/SecurityUserSettingsTab.js

@@ -242,6 +242,7 @@ export default class SecurityUserSettingsTab extends React.Component {
     render() {
         const DevicesPanel = sdk.getComponent('views.settings.DevicesPanel');
         const SettingsFlag = sdk.getComponent('views.elements.SettingsFlag');
+        const EventIndexPanel = sdk.getComponent('views.settings.EventIndexPanel');
 
         const KeyBackupPanel = sdk.getComponent('views.settings.KeyBackupPanel');
         const keyBackup = (
@@ -253,6 +254,16 @@ export default class SecurityUserSettingsTab extends React.Component {
             </div>
         );
 
+        let eventIndex;
+        if (SettingsStore.isFeatureEnabled("feature_event_indexing")) {
+            eventIndex = (
+                <div className="mx_SettingsTab_section">
+                    <span className="mx_SettingsTab_subheading">{_t("Message search")}</span>
+                    <EventIndexPanel />
+                </div>
+            );
+        }
+
         // XXX: There's no such panel in the current cross-signing designs, but
         // it's useful to have for testing the feature. If there's no interest
         // in having advanced details here once all flows are implemented, we
@@ -281,6 +292,7 @@ export default class SecurityUserSettingsTab extends React.Component {
                     </div>
                 </div>
                 {keyBackup}
+                {eventIndex}
                 {crossSigning}
                 {this._renderCurrentDeviceInfo()}
                 <div className='mx_SettingsTab_section'>

src/i18n/strings/en_EN.json

@@ -414,6 +414,7 @@
     "Allow fallback call assist server turn.matrix.org when your homeserver does not offer one (your IP address would be shared during a call)": "Allow fallback call assist server turn.matrix.org when your homeserver does not offer one (your IP address would be shared during a call)",
     "Send read receipts for messages (requires compatible homeserver to disable)": "Send read receipts for messages (requires compatible homeserver to disable)",
     "Show previews/thumbnails for images": "Show previews/thumbnails for images",
+    "Enable message search in encrypted rooms": "Enable message search in encrypted rooms",
     "Collecting app version information": "Collecting app version information",
     "Collecting logs": "Collecting logs",
     "Uploading report": "Uploading report",
@@ -559,6 +560,14 @@
     "Failed to set display name": "Failed to set display name",
     "Disable Notifications": "Disable Notifications",
     "Enable Notifications": "Enable Notifications",
+    "Securely cache encrypted messages locally for them to appear in search results, using ": "Securely cache encrypted messages locally for them to appear in search results, using ",
+    " to store messages from ": " to store messages from ",
+    "rooms.": "rooms.",
+    "Manage": "Manage",
+    "Securely cache encrypted messages locally for them to appear in search results.": "Securely cache encrypted messages locally for them to appear in search results.",
+    "Enable": "Enable",
+    "Riot is missing some components required for securely caching encrypted messages locally. If you'd like to experiment with this feature, build a custom Riot Desktop with <nativeLink>search components added</nativeLink>.": "Riot is missing some components required for securely caching encrypted messages locally. If you'd like to experiment with this feature, build a custom Riot Desktop with <nativeLink>search components added</nativeLink>.",
+    "Riot can't securely cache encrypted messages locally while running in a web browser. Use <riotLink>Riot Desktop</riotLink> for encrypted messages to appear in search results.": "Riot can't securely cache encrypted messages locally while running in a web browser. Use <riotLink>Riot Desktop</riotLink> for encrypted messages to appear in search results.",
     "Connecting to integration manager...": "Connecting to integration manager...",
     "Cannot connect to integration manager": "Cannot connect to integration manager",
     "The integration manager is offline or it cannot reach your homeserver.": "The integration manager is offline or it cannot reach your homeserver.",
@@ -752,6 +761,7 @@
     "Accept all %(invitedRooms)s invites": "Accept all %(invitedRooms)s invites",
     "Reject all %(invitedRooms)s invites": "Reject all %(invitedRooms)s invites",
     "Key backup": "Key backup",
+    "Message search": "Message search",
     "Cross-signing": "Cross-signing",
     "Security & Privacy": "Security & Privacy",
     "Devices": "Devices",
@@ -2051,6 +2061,14 @@
     "This device has detected that your recovery passphrase and key for Secure Messages have been removed.": "This device has detected that your recovery passphrase and key for Secure Messages have been removed.",
     "If you did this accidentally, you can setup Secure Messages on this device which will re-encrypt this device's message history with a new recovery method.": "If you did this accidentally, you can setup Secure Messages on this device which will re-encrypt this device's message history with a new recovery method.",
     "If you didn't remove the recovery method, an attacker may be trying to access your account. Change your account password and set a new recovery method immediately in Settings.": "If you didn't remove the recovery method, an attacker may be trying to access your account. Change your account password and set a new recovery method immediately in Settings.",
+    "If disabled, messages from encrypted rooms won't appear in search results.": "If disabled, messages from encrypted rooms won't appear in search results.",
+    "Disable": "Disable",
+    "Not currently downloading messages for any room.": "Not currently downloading messages for any room.",
+    "Downloading mesages for %(currentRoom)s.": "Downloading mesages for %(currentRoom)s.",
+    "Riot is securely caching encrypted messages locally for them to appear in search results:": "Riot is securely caching encrypted messages locally for them to appear in search results:",
+    "Space used:": "Space used:",
+    "Indexed messages:": "Indexed messages:",
+    "Number of rooms:": "Number of rooms:",
     "Failed to set direct chat tag": "Failed to set direct chat tag",
     "Failed to remove tag %(tagName)s from room": "Failed to remove tag %(tagName)s from room",
     "Failed to add tag %(tagName)s to room": "Failed to add tag %(tagName)s to room"

src/indexing/BaseEventIndexManager.js

@@ -74,6 +74,12 @@ export interface LoadArgs {
     direction: string;
 }
 
+export interface IndexStats {
+    size: number;
+    event_count: number;
+    room_count: number;
+}
+
 /**
  * Base class for classes that provide platform-specific event indexing.
  *
@@ -124,6 +130,16 @@ export default class BaseEventIndexManager {
         throw new Error("Unimplemented");
     }
 
+    /**
+     * Get statistical information of the index.
+     *
+     * @return {Promise<IndexStats>} A promise that will resolve to the index
+     * statistics.
+     */
+    async getStats(): Promise<IndexStats> {
+        throw new Error("Unimplemented");
+    }
+
     /**
      * Commit the previously queued up events to the index.
      *

src/indexing/EventIndex.js

@@ -17,20 +17,25 @@ limitations under the License.
 import PlatformPeg from "../PlatformPeg";
 import {MatrixClientPeg} from "../MatrixClientPeg";
 import {EventTimeline, RoomMember} from 'matrix-js-sdk';
+import {sleep} from "../utils/promise";
+import {EventEmitter} from "events";
 
 /*
  * Event indexing class that wraps the platform specific event indexing.
  */
-export default class EventIndex {
+export default class EventIndex extends EventEmitter {
     constructor() {
+        super();
         this.crawlerCheckpoints = [];
-        // The time that the crawler will wait between /rooms/{room_id}/messages
+        // The time in ms that the crawler will wait loop iterations if there
-        // requests
+        // have not been any checkpoints to consume in the last iteration.
-        this._crawlerTimeout = 3000;
+        this._crawlerIdleTime = 5000;
+        this._crawlerSleepTime = 3000;
         // The maximum number of events our crawler should fetch in a single
         // crawl.
         this._eventsPerCrawl = 100;
         this._crawler = null;
+        this._currentCheckpoint = null;
         this.liveEventsForIndex = new Set();
     }
 
@@ -65,59 +70,62 @@ export default class EventIndex {
         client.removeListener('Room.timelineReset', this.onTimelineReset);
     }
 
+    /**
+     * Get crawler checkpoints for the encrypted rooms and store them in the index.
+     */
+    async addInitialCheckpoints() {
+        const indexManager = PlatformPeg.get().getEventIndexingManager();
+        const client = MatrixClientPeg.get();
+        const rooms = client.getRooms();
+
+        const isRoomEncrypted = (room) => {
+            return client.isRoomEncrypted(room.roomId);
+        };
+
+        // We only care to crawl the encrypted rooms, non-encrypted
+        // rooms can use the search provided by the homeserver.
+        const encryptedRooms = rooms.filter(isRoomEncrypted);
+
+        console.log("EventIndex: Adding initial crawler checkpoints");
+
+        // Gather the prev_batch tokens and create checkpoints for
+        // our message crawler.
+        await Promise.all(encryptedRooms.map(async (room) => {
+            const timeline = room.getLiveTimeline();
+            const token = timeline.getPaginationToken("b");
+
+            console.log("EventIndex: Got token for indexer",
+                        room.roomId, token);
+
+            const backCheckpoint = {
+                roomId: room.roomId,
+                token: token,
+                direction: "b",
+            };
+
+            const forwardCheckpoint = {
+                roomId: room.roomId,
+                token: token,
+                direction: "f",
+            };
+
+            await indexManager.addCrawlerCheckpoint(backCheckpoint);
+            await indexManager.addCrawlerCheckpoint(forwardCheckpoint);
+            this.crawlerCheckpoints.push(backCheckpoint);
+            this.crawlerCheckpoints.push(forwardCheckpoint);
+        }));
+    }
+
     onSync = async (state, prevState, data) => {
         const indexManager = PlatformPeg.get().getEventIndexingManager();
 
         if (prevState === "PREPARED" && state === "SYNCING") {
-            const addInitialCheckpoints = async () => {
-                const client = MatrixClientPeg.get();
-                const rooms = client.getRooms();
-
-                const isRoomEncrypted = (room) => {
-                    return client.isRoomEncrypted(room.roomId);
-                };
-
-                // We only care to crawl the encrypted rooms, non-encrypted.
-                // rooms can use the search provided by the homeserver.
-                const encryptedRooms = rooms.filter(isRoomEncrypted);
-
-                console.log("EventIndex: Adding initial crawler checkpoints");
-
-                // Gather the prev_batch tokens and create checkpoints for
-                // our message crawler.
-                await Promise.all(encryptedRooms.map(async (room) => {
-                    const timeline = room.getLiveTimeline();
-                    const token = timeline.getPaginationToken("b");
-
-                    console.log("EventIndex: Got token for indexer",
-                                room.roomId, token);
-
-                    const backCheckpoint = {
-                        roomId: room.roomId,
-                        token: token,
-                        direction: "b",
-                    };
-
-                    const forwardCheckpoint = {
-                        roomId: room.roomId,
-                        token: token,
-                        direction: "f",
-                    };
-
-                    await indexManager.addCrawlerCheckpoint(backCheckpoint);
-                    await indexManager.addCrawlerCheckpoint(forwardCheckpoint);
-                    this.crawlerCheckpoints.push(backCheckpoint);
-                    this.crawlerCheckpoints.push(forwardCheckpoint);
-                }));
-            };
-
             // If our indexer is empty we're most likely running Riot the
             // first time with indexing support or running it with an
             // initial sync. Add checkpoints to crawl our encrypted rooms.
             const eventIndexWasEmpty = await indexManager.isEventIndexEmpty();
-            if (eventIndexWasEmpty) await addInitialCheckpoints();
+            if (eventIndexWasEmpty) await this.addInitialCheckpoints();
 
-            // Start our crawler.
             this.startCrawler();
             return;
         }
@@ -182,13 +190,11 @@ export default class EventIndex {
         indexManager.addEventToIndex(e, profile);
     }
 
-    async crawlerFunc() {
+    emitNewCheckpoint() {
-        // TODO either put this in a better place or find a library provided
+        this.emit("changedCheckpoint", this.currentRoom());
-        // method that does this.
+    }
-        const sleep = async (ms) => {
-            return new Promise(resolve => setTimeout(resolve, ms));
-        };
 
+    async crawlerFunc() {
         let cancelled = false;
 
         console.log("EventIndex: Started crawler function");
@@ -202,11 +208,27 @@ export default class EventIndex {
             cancelled = true;
         };
 
+        let idle = false;
+
         while (!cancelled) {
             // This is a low priority task and we don't want to spam our
             // homeserver with /messages requests so we set a hefty timeout
             // here.
-            await sleep(this._crawlerTimeout);
+            let sleepTime = this._crawlerSleepTime;
+
+            // Don't let the user configure a lower sleep time than 100 ms.
+            sleepTime = Math.max(sleepTime, 100);
+
+            if (idle) {
+                sleepTime = this._crawlerIdleTime;
+            }
+
+            if (this._currentCheckpoint !== null) {
+                this._currentCheckpoint = null;
+                this.emitNewCheckpoint();
+            }
+
+            await sleep(sleepTime);
 
             console.log("EventIndex: Running the crawler loop.");
 
@@ -219,9 +241,15 @@ export default class EventIndex {
             /// There is no checkpoint available currently, one may appear if
             // a sync with limited room timelines happens, so go back to sleep.
             if (checkpoint === undefined) {
+                idle = true;
                 continue;
             }
 
+            this._currentCheckpoint = checkpoint;
+            this.emitNewCheckpoint();
+
+            idle = false;
+
             console.log("EventIndex: crawling using checkpoint", checkpoint);
 
             // We have a checkpoint, let us fetch some messages, again, very
@@ -241,6 +269,11 @@ export default class EventIndex {
                 continue;
             }
 
+            if (cancelled) {
+                this.crawlerCheckpoints.push(checkpoint);
+                break;
+            }
+
             if (res.chunk.length === 0) {
                 console.log("EventIndex: Done with the checkpoint", checkpoint);
                 // We got to the start/end of our timeline, lets just
@@ -600,4 +633,29 @@ export default class EventIndex {
 
         return paginationPromise;
     }
+
+    async getStats() {
+        const indexManager = PlatformPeg.get().getEventIndexingManager();
+        return indexManager.getStats();
+    }
+
+    /**
+     * Get the room that we are currently crawling.
+     *
+     * @returns {Room} A MatrixRoom that is being currently crawled, null
+     * if no room is currently being crawled.
+     */
+    currentRoom() {
+        if (this._currentCheckpoint === null && this.crawlerCheckpoints.length === 0) {
+            return null;
+        }
+
+        const client = MatrixClientPeg.get();
+
+        if (this._currentCheckpoint !== null) {
+            return client.getRoom(this._currentCheckpoint.roomId);
+        } else {
+            return client.getRoom(this.crawlerCheckpoints[0].roomId);
+        }
+    }
 }

src/indexing/EventIndexPeg.js

@@ -21,17 +21,19 @@ limitations under the License.
 
 import PlatformPeg from "../PlatformPeg";
 import EventIndex from "../indexing/EventIndex";
-import SettingsStore from '../settings/SettingsStore';
+import SettingsStore, {SettingLevel} from '../settings/SettingsStore';
 
 class EventIndexPeg {
     constructor() {
         this.index = null;
+        this._supportIsInstalled = false;
     }
 
     /**
-     * Create a new EventIndex and initialize it if the platform supports it.
+     * Initialize the EventIndexPeg and if event indexing is enabled initialize
+     * the event index.
      *
-     * @return {Promise<bool>} A promise that will resolve to true if an
+     * @return {Promise<boolean>} A promise that will resolve to true if an
      * EventIndex was successfully initialized, false otherwise.
      */
     async init() {
@@ -40,12 +42,33 @@ class EventIndexPeg {
         }
 
         const indexManager = PlatformPeg.get().getEventIndexingManager();
-        if (!indexManager || await indexManager.supportsEventIndexing() !== true) {
+        if (!indexManager) {
-            console.log("EventIndex: Platform doesn't support event indexing,",
+            console.log("EventIndex: Platform doesn't support event indexing, not initializing.");
-                        "not initializing.");
             return false;
         }
 
+        this._supportIsInstalled = await indexManager.supportsEventIndexing();
+
+        if (!this.supportIsInstalled()) {
+            console.log("EventIndex: Event indexing isn't installed for the platform, not initializing.");
+            return false;
+        }
+
+        if (!SettingsStore.getValueAt(SettingLevel.DEVICE, 'enableEventIndexing')) {
+            console.log("EventIndex: Event indexing is disabled, not initializing");
+            return false;
+        }
+
+        return this.initEventIndex();
+    }
+
+    /**
+     * Initialize the event index.
+     *
+     * @returns {boolean} True if the event index was succesfully initialized,
+     * false otherwise.
+     */
+    async initEventIndex() {
         const index = new EventIndex();
 
         try {
@@ -60,6 +83,29 @@ class EventIndexPeg {
         return true;
     }
 
+    /**
+     * Check if the current platform has support for event indexing.
+     *
+     * @return {boolean} True if it has support, false otherwise. Note that this
+     * does not mean that support is installed.
+     */
+    platformHasSupport(): boolean {
+        return PlatformPeg.get().getEventIndexingManager() !== null;
+    }
+
+    /**
+     * Check if event indexing support is installed for the platfrom.
+     *
+     * Event indexing might require additional optional modules to be installed,
+     * this tells us if those are installed. Note that this should only be
+     * called after the init() method was called.
+     *
+     * @return {boolean} True if support is installed, false otherwise.
+     */
+    supportIsInstalled(): boolean {
+        return this._supportIsInstalled;
+    }
+
     /**
      * Get the current event index.
      *
@@ -69,6 +115,11 @@ class EventIndexPeg {
         return this.index;
     }
 
+    start() {
+        if (this.index === null) return;
+        this.index.startCrawler();
+    }
+
     stop() {
         if (this.index === null) return;
         this.index.stopCrawler();

src/settings/Settings.js

@@ -480,4 +480,9 @@ export const SETTINGS = {
         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
         default: RIGHT_PANEL_PHASES.GroupMemberList,
     },
+    "enableEventIndexing": {
+        supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
+        displayName: _td("Enable message search in encrypted rooms"),
+        default: true,
+    },
 };

src/utils/FormattingUtils.js

@@ -30,6 +30,31 @@ export function formatCount(count) {
    return (count / 1000000000).toFixed(1) + "B"; // 10B is enough for anyone, right? :S
 }
 
+/**
+ * Format a count showing the whole number but making it a bit more readable.
+ * e.g: 1000 => 1,000
+ */
+export function formatCountLong(count) {
+    const formatter = new Intl.NumberFormat();
+    return formatter.format(count)
+}
+
+/**
+ * format a size in bytes into a human readable form
+ * e.g: 1024 -> 1.00 KB
+ */
+export function formatBytes(bytes, decimals = 2) {
+    if (bytes === 0) return '0 Bytes';
+
+    const k = 1024;
+    const dm = decimals < 0 ? 0 : decimals;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
+}
+
 /**
  * format a key into groups of 4 characters, for easier visual inspection
  *