Poll history: fetch last 30 days of polls (#10157)

* use timeline pagination

* fetch last 30 days of poll history

* add comments, tidy

* more comments

* finish comment

* wait for responses to resolve before displaying in list

* dont use state for list

* return unsubscribe

* strict fixes

* unnecessary event type in filter

* add catch

src/components/views/dialogs/polls/PollHistoryDialog.tsx

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { useEffect, useState } from "react";
+import React, { useState } from "react";
 import { MatrixClient } from "matrix-js-sdk/src/client";
 import { MatrixEvent, Poll } from "matrix-js-sdk/src/matrix";
 
@@ -23,7 +23,8 @@ import BaseDialog from "../BaseDialog";
 import { IDialogProps } from "../IDialogProps";
 import { PollHistoryList } from "./PollHistoryList";
 import { PollHistoryFilter } from "./types";
-import { usePolls } from "./usePollHistory";
+import { usePollsWithRelations } from "./usePollHistory";
+import { useFetchPastPolls } from "./fetchPastPolls";
 
 type PollHistoryDialogProps = Pick<IDialogProps, "onFinished"> & {
     roomId: string;
@@ -34,7 +35,10 @@ const sortEventsByLatest = (left: MatrixEvent, right: MatrixEvent): number => ri
 const filterPolls =
     (filter: PollHistoryFilter) =>
     (poll: Poll): boolean =>
-        (filter === "ACTIVE") !== poll.isEnded;
+        // exclude polls while they are still loading
+        // to avoid jitter in list
+        !poll.isFetchingResponses && (filter === "ACTIVE") !== poll.isEnded;
+
 const filterAndSortPolls = (polls: Map<string, Poll>, filter: PollHistoryFilter): MatrixEvent[] => {
     return [...polls.values()]
         .filter(filterPolls(filter))
@@ -43,19 +47,20 @@ const filterAndSortPolls = (polls: Map<string, Poll>, filter: PollHistoryFilter)
 };
 
 export const PollHistoryDialog: React.FC<PollHistoryDialogProps> = ({ roomId, matrixClient, onFinished }) => {
-    const { polls } = usePolls(roomId, matrixClient);
+    const room = matrixClient.getRoom(roomId)!;
+    const { isLoading } = useFetchPastPolls(room, matrixClient);
+    const { polls } = usePollsWithRelations(roomId, matrixClient);
     const [filter, setFilter] = useState<PollHistoryFilter>("ACTIVE");
-    const [pollStartEvents, setPollStartEvents] = useState(filterAndSortPolls(polls, filter));
 
-    useEffect(() => {
-        setPollStartEvents(filterAndSortPolls(polls, filter));
-    }, [filter, polls]);
+    const pollStartEvents = filterAndSortPolls(polls, filter);
+    const isLoadingPollResponses = [...polls.values()].some((poll) => poll.isFetchingResponses);
 
     return (
         <BaseDialog title={_t("Polls history")} onFinished={onFinished}>
             <div className="mx_PollHistoryDialog_content">
                 <PollHistoryList
                     pollStartEvents={pollStartEvents}
+                    isLoading={isLoading || isLoadingPollResponses}
                     polls={polls}
                     filter={filter}
                     onFilterChange={setFilter}

src/components/views/dialogs/polls/PollHistoryList.tsx

@@ -19,18 +19,37 @@ import classNames from "classnames";
 import { MatrixEvent, Poll } from "matrix-js-sdk/src/matrix";
 
 import { _t } from "../../../../languageHandler";
+import { FilterTabGroup } from "../../elements/FilterTabGroup";
+import InlineSpinner from "../../elements/InlineSpinner";
 import { PollHistoryFilter } from "./types";
 import { PollListItem } from "./PollListItem";
 import { PollListItemEnded } from "./PollListItemEnded";
-import { FilterTabGroup } from "../../elements/FilterTabGroup";
+
+const LoadingPolls: React.FC<{ noResultsYet?: boolean }> = ({ noResultsYet }) => (
+    <div
+        className={classNames("mx_PollHistoryList_loading", {
+            mx_PollHistoryList_noResultsYet: noResultsYet,
+        })}
+    >
+        <InlineSpinner />
+        {_t("Loading polls")}
+    </div>
+);
 
 type PollHistoryListProps = {
     pollStartEvents: MatrixEvent[];
     polls: Map<string, Poll>;
     filter: PollHistoryFilter;
     onFilterChange: (filter: PollHistoryFilter) => void;
+    isLoading?: boolean;
 };
-export const PollHistoryList: React.FC<PollHistoryListProps> = ({ pollStartEvents, polls, filter, onFilterChange }) => {
+export const PollHistoryList: React.FC<PollHistoryListProps> = ({
+    pollStartEvents,
+    polls,
+    filter,
+    isLoading,
+    onFilterChange,
+}) => {
     return (
         <div className="mx_PollHistoryList">
             <FilterTabGroup<PollHistoryFilter>
@@ -42,7 +61,7 @@ export const PollHistoryList: React.FC<PollHistoryListProps> = ({ pollStartEvent
                     { id: "ENDED", label: "Past polls" },
                 ]}
             />
-            {!!pollStartEvents.length ? (
+            {!!pollStartEvents.length && (
                 <ol className={classNames("mx_PollHistoryList_list", `mx_PollHistoryList_list_${filter}`)}>
                     {pollStartEvents.map((pollStartEvent) =>
                         filter === "ACTIVE" ? (
@@ -55,14 +74,17 @@ export const PollHistoryList: React.FC<PollHistoryListProps> = ({ pollStartEvent
                             />
                         ),
                     )}
+                    {isLoading && <LoadingPolls />}
                 </ol>
-            ) : (
+            )}
+            {!pollStartEvents.length && !isLoading && (
                 <span className="mx_PollHistoryList_noResults">
                     {filter === "ACTIVE"
                         ? _t("There are no active polls in this room")
                         : _t("There are no past polls in this room")}
                 </span>
             )}
+            {!pollStartEvents.length && isLoading && <LoadingPolls noResultsYet />}
         </div>
     );
 };

src/components/views/dialogs/polls/fetchPastPolls.ts

@@ -0,0 +1,129 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { useEffect, useState } from "react";
+import { M_POLL_START } from "matrix-js-sdk/src/@types/polls";
+import { MatrixClient } from "matrix-js-sdk/src/client";
+import { EventTimeline, EventTimelineSet, Room } from "matrix-js-sdk/src/matrix";
+import { Filter, IFilterDefinition } from "matrix-js-sdk/src/filter";
+import { logger } from "matrix-js-sdk/src/logger";
+
+/**
+ * Page timeline backwards until either:
+ * - event older than endOfHistoryPeriodTimestamp is encountered
+ * - end of timeline is reached
+ * @param timelineSet - timelineset to page
+ * @param matrixClient - client
+ * @param endOfHistoryPeriodTimestamp - epoch timestamp to fetch until
+ * @returns void
+ */
+const pagePolls = async (
+    timelineSet: EventTimelineSet,
+    matrixClient: MatrixClient,
+    endOfHistoryPeriodTimestamp: number,
+): Promise<void> => {
+    const liveTimeline = timelineSet.getLiveTimeline();
+    const events = liveTimeline.getEvents();
+    const oldestEventTimestamp = events[0]?.getTs() || Date.now();
+    const hasMorePages = !!liveTimeline.getPaginationToken(EventTimeline.BACKWARDS);
+
+    if (!hasMorePages || oldestEventTimestamp <= endOfHistoryPeriodTimestamp) {
+        return;
+    }
+
+    await matrixClient.paginateEventTimeline(liveTimeline, {
+        backwards: true,
+    });
+
+    return pagePolls(timelineSet, matrixClient, endOfHistoryPeriodTimestamp);
+};
+
+const ONE_DAY_MS = 60000 * 60 * 24;
+/**
+ * Fetches timeline history for given number of days in past
+ * @param timelineSet - timelineset to page
+ * @param matrixClient - client
+ * @param historyPeriodDays - number of days of history to fetch, from current day
+ * @returns isLoading - true while fetching history
+ */
+const useTimelineHistory = (
+    timelineSet: EventTimelineSet | null,
+    matrixClient: MatrixClient,
+    historyPeriodDays: number,
+): { isLoading: boolean } => {
+    const [isLoading, setIsLoading] = useState(true);
+
+    useEffect(() => {
+        if (!timelineSet) {
+            return;
+        }
+        const endOfHistoryPeriodTimestamp = Date.now() - ONE_DAY_MS * historyPeriodDays;
+
+        const doFetchHistory = async (): Promise<void> => {
+            setIsLoading(true);
+            try {
+                await pagePolls(timelineSet, matrixClient, endOfHistoryPeriodTimestamp);
+            } catch (error) {
+                logger.error("Failed to fetch room polls history", error);
+            } finally {
+                setIsLoading(false);
+            }
+        };
+        doFetchHistory();
+    }, [timelineSet, historyPeriodDays, matrixClient]);
+
+    return { isLoading };
+};
+
+const filterDefinition: IFilterDefinition = {
+    room: {
+        timeline: {
+            types: [M_POLL_START.name, M_POLL_START.altName],
+        },
+    },
+};
+
+/**
+ * Fetch poll start events in the last N days of room history
+ * @param room - room to fetch history for
+ * @param matrixClient - client
+ * @param historyPeriodDays - number of days of history to fetch, from current day
+ * @returns isLoading - true while fetching history
+ */
+export const useFetchPastPolls = (
+    room: Room,
+    matrixClient: MatrixClient,
+    historyPeriodDays = 30,
+): { isLoading: boolean } => {
+    const [timelineSet, setTimelineSet] = useState<EventTimelineSet | null>(null);
+
+    useEffect(() => {
+        const filter = new Filter(matrixClient.getSafeUserId());
+        filter.setDefinition(filterDefinition);
+        const getFilteredTimelineSet = async (): Promise<void> => {
+            const filterId = await matrixClient.getOrCreateFilter(`POLL_HISTORY_FILTER_${room.roomId}}`, filter);
+            filter.filterId = filterId;
+            const timelineSet = room.getOrCreateFilteredTimelineSet(filter);
+            setTimelineSet(timelineSet);
+        };
+
+        getFilteredTimelineSet();
+    }, [room, matrixClient]);
+
+    const { isLoading } = useTimelineHistory(timelineSet, matrixClient, historyPeriodDays);
+
+    return { isLoading };
+};

src/components/views/dialogs/polls/usePollHistory.ts

@@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { useEffect, useState } from "react";
 import { Poll, PollEvent } from "matrix-js-sdk/src/matrix";
 import { MatrixClient } from "matrix-js-sdk/src/client";
 
@@ -21,6 +22,7 @@ import { useEventEmitterState } from "../../../../hooks/useEventEmitter";
 
 /**
  * Get poll instances from a room
+ * Updates to include new polls
  * @param roomId - id of room to retrieve polls for
  * @param matrixClient - client
  * @returns {Map<string, Poll>} - Map of Poll instances
@@ -37,9 +39,58 @@ export const usePolls = (
         throw new Error("Cannot find room");
     }
 
-    const polls = useEventEmitterState(room, PollEvent.New, () => room.polls);
-
-    // @TODO(kerrya) watch polls for end events, trigger refiltering
+    // copy room.polls map so changes can be detected
+    const polls = useEventEmitterState(room, PollEvent.New, () => new Map<string, Poll>(room.polls));
 
     return { polls };
 };
+
+/**
+ * Get all poll instances from a room
+ * Fetch their responses (using cached poll responses)
+ * Updates on:
+ * - new polls added to room
+ * - new responses added to polls
+ * - changes to poll ended state
+ * @param roomId - id of room to retrieve polls for
+ * @param matrixClient - client
+ * @returns {Map<string, Poll>} - Map of Poll instances
+ */
+export const usePollsWithRelations = (
+    roomId: string,
+    matrixClient: MatrixClient,
+): {
+    polls: Map<string, Poll>;
+} => {
+    const { polls } = usePolls(roomId, matrixClient);
+    const [pollsWithRelations, setPollsWithRelations] = useState<Map<string, Poll>>(polls);
+
+    useEffect(() => {
+        const onPollUpdate = async (): Promise<void> => {
+            // trigger rerender by creating a new poll map
+            setPollsWithRelations(new Map(polls));
+        };
+        if (polls) {
+            for (const poll of polls.values()) {
+                // listen to changes in responses and end state
+                poll.on(PollEvent.End, onPollUpdate);
+                poll.on(PollEvent.Responses, onPollUpdate);
+                // trigger request to get all responses
+                // if they are not already in cache
+                poll.getResponses();
+            }
+            setPollsWithRelations(polls);
+        }
+        // unsubscribe
+        return () => {
+            if (polls) {
+                for (const poll of polls.values()) {
+                    poll.off(PollEvent.End, onPollUpdate);
+                    poll.off(PollEvent.Responses, onPollUpdate);
+                }
+            }
+        };
+    }, [polls, setPollsWithRelations]);
+
+    return { polls: pollsWithRelations };
+};