feat(ui): gallery image selection ux

The selection logic is a bit complicated. We have image selection and pagination, both of which can be triggered using the mouse or hotkeys. We have viewer image selection and comparison image selection, which is determined by the alt key. This change ties the room together with these behaviours: - Changing the page using pagination buttons never changes the selection. - Changing the selected image using arrows may change the page, if the arrow key pressed would select an image off the current page. - `right` on the last image of the current page goes to the next page - `down` on the last row of images goes to the next page - `left` on the first image of the current page goes to the previous page - `up` on the first row of images goes to the previous page - If `alt` is held when using arrow keys, we change the page, but we only change the comparison image selection. - When using arrow keys, if the page has changed since the last image was selected, the selection is reset to the first image on the page. - The next/previous buttons on the image viewer do the same thing as `left` and `right` without `alt`. - When clicking an image in the gallery: - If no modifier keys are held, the image is exclusively selected. - If `ctrl` or `meta` are held, the image's selection status is toggled. - If `shift` is held, all images from the last-selected image to the image are selected. If there are no images on the current page, the selection is unchanged. - If `alt` is held, the image is set as the compare image. - `ctrl+a` and `meta+a` add the current page to the selection. The logic for gallery navigation and selection is now pretty hairy. It's spread across 3 hooks, a listener, redux slice, components. When we next make changes to this part of the app, we should consider consolidating some of the related logic. Probably most of it can go into a single listener and make it much simpler to grok.
2024-08-30 20:32:17 +00:00 · 2024-07-02 12:42:17 +10:00 · 2024-07-02 12:42:17 +10:00 · 4075a81676
commit 4075a81676
parent 4d39976909
10 changed files with 243 additions and 94 deletions
--- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/index.ts
+++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/index.ts
@ -23,6 +23,7 @@ import { addEnqueueRequestedCanvasListener } from 'app/store/middleware/listener
 import { addEnqueueRequestedLinear } from 'app/store/middleware/listenerMiddleware/listeners/enqueueRequestedLinear';
 import { addEnqueueRequestedNodes } from 'app/store/middleware/listenerMiddleware/listeners/enqueueRequestedNodes';
 import { addGalleryImageClickedListener } from 'app/store/middleware/listenerMiddleware/listeners/galleryImageClicked';
+import { addGalleryOffsetChangedListener } from 'app/store/middleware/listenerMiddleware/listeners/galleryOffsetChanged';
 import { addGetOpenAPISchemaListener } from 'app/store/middleware/listenerMiddleware/listeners/getOpenAPISchema';
 import { addImageAddedToBoardFulfilledListener } from 'app/store/middleware/listenerMiddleware/listeners/imageAddedToBoard';
 import { addRequestedSingleImageDeletionListener } from 'app/store/middleware/listenerMiddleware/listeners/imageDeleted';
@ -79,6 +80,7 @@ addImagesUnstarredListener(startAppListening);

 // Gallery
 addGalleryImageClickedListener(startAppListening);
+addGalleryOffsetChangedListener(startAppListening);

 // User Invoked
 addEnqueueRequestedCanvasListener(startAppListening);
--- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/galleryOffsetChanged.ts
+++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/galleryOffsetChanged.ts
@ -0,0 +1,119 @@
+import type { AppStartListening } from 'app/store/middleware/listenerMiddleware';
+import { selectListImagesQueryArgs } from 'features/gallery/store/gallerySelectors';
+import { imageToCompareChanged, offsetChanged, selectionChanged } from 'features/gallery/store/gallerySlice';
+import { imagesApi } from 'services/api/endpoints/images';
+
+export const addGalleryOffsetChangedListener = (startAppListening: AppStartListening) => {
+  /**
+   * When the user changes pages in the gallery, we need to wait until the next page of images is loaded, then maybe
+   * update the selection.
+   *
+   * There are a three scenarios:
+   *
+   * 1. The page is changed by clicking the pagination buttons. No changes to selection are needed.
+   *
+   * 2. The page is changed by using the arrow keys (without alt).
+   * - When going backwards, select the last image.
+   * - When going forwards, select the first image.
+   *
+   * 3. The page is changed by using the arrows keys with alt. This means the user is changing the comparison image.
+   * - When going backwards, select the last image _as the comparison image_.
+   * - When going forwards, select the first image _as the comparison image_.
+   */
+  startAppListening({
+    actionCreator: offsetChanged,
+    effect: async (action, { dispatch, getState, getOriginalState, take, cancelActiveListeners }) => {
+      // Cancel any active listeners to prevent the selection from changing without user input
+      cancelActiveListeners();
+
+      const { withHotkey } = action.payload;
+
+      if (!withHotkey) {
+        // User changed pages by clicking the pagination buttons - no changes to selection
+        return;
+      }
+
+      const originalState = getOriginalState();
+      const prevOffset = originalState.gallery.offset;
+      const offset = getState().gallery.offset;
+
+      if (offset === prevOffset) {
+        // The page didn't change - bail
+        return;
+      }
+
+      /**
+       * We need to wait until the next page of images is loaded before updating the selection, so we use the correct
+       * page of images.
+       *
+       * The simplest way to do it would be to use `take` to wait for the next fulfilled action, but RTK-Q doesn't
+       * dispatch an action on cache hits. This means the `take` will only return if the cache is empty. If the user
+       * changes to a cached page - a common situation - the `take` will never resolve.
+       *
+       * So we need to take a two-step approach. First, check if we have data in the cache for the page of images. If
+       * we have data cached, use it to update the selection. If we don't have data cached, wait for the next fulfilled
+       * action, which updates the cache, then use the cache to update the selection.
+       */
+
+      // Check if we have data in the cache for the page of images
+      const queryArgs = selectListImagesQueryArgs(getState());
+      let { data } = imagesApi.endpoints.listImages.select(queryArgs)(getState());
+
+      // No data yet - wait for the network request to complete
+      if (!data) {
+        const takeResult = await take(imagesApi.endpoints.listImages.matchFulfilled, 5000);
+        if (!takeResult) {
+          // The request didn't complete in time - bail
+          return;
+        }
+        data = takeResult[0].payload;
+      }
+
+      // We awaited a network request - state could have changed, get fresh state
+      const state = getState();
+      const { selection, imageToCompare } = state.gallery;
+      const imageDTOs = data?.items;
+
+      if (!imageDTOs) {
+        // The page didn't load - bail
+        return;
+      }
+
+      if (withHotkey === 'arrow') {
+        // User changed pages by using the arrow keys - selection changes to first or last image depending
+        if (offset < prevOffset) {
+          // We've gone backwards
+          const lastImage = imageDTOs[imageDTOs.length - 1];
+          if (!selection.some((selectedImage) => selectedImage.image_name === lastImage?.image_name)) {
+            dispatch(selectionChanged(lastImage ? [lastImage] : []));
+          }
+        } else {
+          // We've gone forwards
+          const firstImage = imageDTOs[0];
+          if (!selection.some((selectedImage) => selectedImage.image_name === firstImage?.image_name)) {
+            dispatch(selectionChanged(firstImage ? [firstImage] : []));
+          }
+        }
+        return;
+      }
+
+      if (withHotkey === 'alt+arrow') {
+        // User changed pages by using the arrow keys with alt - comparison image changes to first or last depending
+        if (offset < prevOffset) {
+          // We've gone backwards
+          const lastImage = imageDTOs[imageDTOs.length - 1];
+          if (lastImage && imageToCompare?.image_name !== lastImage.image_name) {
+            dispatch(imageToCompareChanged(lastImage));
+          }
+        } else {
+          // We've gone forwards
+          const firstImage = imageDTOs[0];
+          if (firstImage && imageToCompare?.image_name !== firstImage.image_name) {
+            dispatch(imageToCompareChanged(firstImage));
+          }
+        }
+        return;
+      }
+    },
+  });
+};
--- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/socketio/socketInvocationComplete.ts
+++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/socketio/socketInvocationComplete.ts
@ -91,7 +91,7 @@ export const addInvocationCompleteEventListener = (startAppListening: AppStartLi
              );
            }

-            dispatch(offsetChanged(0));
+            dispatch(offsetChanged({ offset: 0 }));

            if (!imageDTO.board_id && gallery.selectedBoardId !== 'none') {
              dispatch(
--- a/invokeai/frontend/web/src/features/gallery/components/ImageGrid/GalleryPagination.tsx
+++ b/invokeai/frontend/web/src/features/gallery/components/ImageGrid/GalleryPagination.tsx
@ -1,11 +1,20 @@
 import { Button, Flex, IconButton, Spacer } from '@invoke-ai/ui-library';
 import { ELLIPSIS, useGalleryPagination } from 'features/gallery/hooks/useGalleryPagination';
+import { useCallback } from 'react';
 import { PiCaretLeftBold, PiCaretRightBold } from 'react-icons/pi';

 export const GalleryPagination = () => {
  const { goPrev, goNext, isPrevEnabled, isNextEnabled, pageButtons, goToPage, currentPage, total } =
    useGalleryPagination();

+  const onClickPrev = useCallback(() => {
+    goPrev();
+  }, [goPrev]);
+
+  const onClickNext = useCallback(() => {
+    goNext();
+  }, [goNext]);
+
  if (!total) {
    return null;
  }
@ -16,7 +25,7 @@ export const GalleryPagination = () => {
        size="sm"
        aria-label="prev"
        icon={<PiCaretLeftBold />}
-        onClick={goPrev}
+        onClick={onClickPrev}
        isDisabled={!isPrevEnabled}
        variant="ghost"
      />
@ -45,7 +54,7 @@ export const GalleryPagination = () => {
        size="sm"
        aria-label="next"
        icon={<PiCaretRightBold />}
-        onClick={goNext}
+        onClick={onClickNext}
        isDisabled={!isNextEnabled}
        variant="ghost"
      />
--- a/invokeai/frontend/web/src/features/gallery/components/ImageGrid/GallerySelectionCountTag.tsx
+++ b/invokeai/frontend/web/src/features/gallery/components/ImageGrid/GallerySelectionCountTag.tsx
@ -17,8 +17,8 @@ export const GallerySelectionCountTag = () => {
  }, [dispatch]);

  const onSelectPage = useCallback(() => {
-    dispatch(selectionChanged(imageDTOs));
-  }, [dispatch, imageDTOs]);
+    dispatch(selectionChanged([...selection, ...imageDTOs]));
+  }, [dispatch, selection, imageDTOs]);

  useHotkeys(['ctrl+a', 'meta+a'], onSelectPage, { preventDefault: true }, [onSelectPage]);

--- a/invokeai/frontend/web/src/features/gallery/components/NextPrevImageButtons.tsx
+++ b/invokeai/frontend/web/src/features/gallery/components/NextPrevImageButtons.tsx
@ -1,11 +1,11 @@
 import type { ChakraProps } from '@invoke-ai/ui-library';
-import { Box, Flex, IconButton, Spinner } from '@invoke-ai/ui-library';
+import { Box, IconButton } from '@invoke-ai/ui-library';
 import { useGalleryImages } from 'features/gallery/hooks/useGalleryImages';
 import { useGalleryNavigation } from 'features/gallery/hooks/useGalleryNavigation';
 import { useGalleryPagination } from 'features/gallery/hooks/useGalleryPagination';
-import { memo } from 'react';
+import { memo, useCallback, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
-import { PiCaretDoubleRightBold, PiCaretLeftBold, PiCaretRightBold } from 'react-icons/pi';
+import { PiCaretLeftBold, PiCaretRightBold } from 'react-icons/pi';

 const nextPrevButtonStyles: ChakraProps['sx'] = {
  color: 'base.100',
@ -14,52 +14,78 @@ const nextPrevButtonStyles: ChakraProps['sx'] = {

 const NextPrevImageButtons = () => {
  const { t } = useTranslation();
-
-  const { prevImage, nextImage, isOnFirstImage, isOnLastImage } = useGalleryNavigation();
+  const { prevImage, nextImage, isOnFirstImageOfView, isOnLastImageOfView } = useGalleryNavigation();

  const { isFetching } = useGalleryImages().queryResult;
-  const { isNextEnabled, goNext } = useGalleryPagination();
+  const { isNextEnabled, goNext, isPrevEnabled, goPrev } = useGalleryPagination();
+
+  const shouldShowLeftArrow = useMemo(() => {
+    if (!isOnFirstImageOfView) {
+      return true;
+    }
+    if (isPrevEnabled) {
+      return true;
+    }
+    return false;
+  }, [isOnFirstImageOfView, isPrevEnabled]);
+
+  const onClickLeftArrow = useCallback(() => {
+    if (isOnFirstImageOfView) {
+      if (isPrevEnabled && !isFetching) {
+        goPrev('arrow');
+      }
+    } else {
+      prevImage();
+    }
+  }, [goPrev, isFetching, isOnFirstImageOfView, isPrevEnabled, prevImage]);
+
+  const shouldShowRightArrow = useMemo(() => {
+    if (!isOnLastImageOfView) {
+      return true;
+    }
+    if (isNextEnabled) {
+      return true;
+    }
+    return false;
+  }, [isNextEnabled, isOnLastImageOfView]);
+
+  const onClickRightArrow = useCallback(() => {
+    if (isOnLastImageOfView) {
+      if (isNextEnabled && !isFetching) {
+        goNext('arrow');
+      }
+    } else {
+      nextImage();
+    }
+  }, [goNext, isFetching, isNextEnabled, isOnLastImageOfView, nextImage]);

  return (
    <Box pos="relative" h="full" w="full">
      <Box pos="absolute" top="50%" transform="translate(0, -50%)" insetInlineStart={1}>
-        {!isOnFirstImage && (
+        {shouldShowLeftArrow && (
          <IconButton
            aria-label={t('accessibility.previousImage')}
            icon={<PiCaretLeftBold size={64} />}
            variant="unstyled"
-            onClick={prevImage}
+            onClick={onClickLeftArrow}
            boxSize={16}
            sx={nextPrevButtonStyles}
+            isDisabled={isFetching}
          />
        )}
      </Box>
      <Box pos="absolute" top="50%" transform="translate(0, -50%)" insetInlineEnd={6}>
-        {!isOnLastImage && (
+        {shouldShowRightArrow && (
          <IconButton
            aria-label={t('accessibility.nextImage')}
            icon={<PiCaretRightBold size={64} />}
            variant="unstyled"
-            onClick={nextImage}
+            onClick={onClickRightArrow}
            boxSize={16}
            sx={nextPrevButtonStyles}
+            isDisabled={isFetching}
          />
        )}
-        {isOnLastImage && isNextEnabled && !isFetching && (
-          <IconButton
-            aria-label={t('accessibility.loadMore')}
-            icon={<PiCaretDoubleRightBold size={64} />}
-            variant="unstyled"
-            onClick={goNext}
-            boxSize={16}
-            sx={nextPrevButtonStyles}
-          />
-        )}
-        {isOnLastImage && isNextEnabled && isFetching && (
-          <Flex w={16} h={16} alignItems="center" justifyContent="center">
-            <Spinner opacity={0.5} size="xl" />
-          </Flex>
-        )}
      </Box>
    </Box>
  );
--- a/invokeai/frontend/web/src/features/gallery/hooks/useGalleryHotkeys.ts
+++ b/invokeai/frontend/web/src/features/gallery/hooks/useGalleryHotkeys.ts
@ -28,7 +28,8 @@ export const useGalleryHotkeys = () => {
    handleRightImage,
    handleUpImage,
    handleDownImage,
-    areImagesBelowCurrent,
+    isOnFirstRow,
+    isOnLastRow,
    isOnFirstImageOfView,
    isOnLastImageOfView,
  } = useGalleryNavigation();
@ -37,7 +38,7 @@ export const useGalleryHotkeys = () => {
    ['left', 'alt+left'],
    (e) => {
      if (isOnFirstImageOfView && isPrevEnabled && !queryResult.isFetching) {
-        goPrev();
+        goPrev(e.altKey ? 'alt+arrow' : 'arrow');
        return;
      }
      canNavigateGallery && handleLeftImage(e.altKey);
@ -52,7 +53,7 @@ export const useGalleryHotkeys = () => {
        return;
      }
      if (isOnLastImageOfView && isNextEnabled && !queryResult.isFetching) {
-        goNext();
+        goNext(e.altKey ? 'alt+arrow' : 'arrow');
        return;
      }
      if (!isOnLastImageOfView) {
@ -65,22 +66,26 @@ export const useGalleryHotkeys = () => {
  useHotkeys(
    ['up', 'alt+up'],
    (e) => {
+      if (isOnFirstRow && isPrevEnabled && !queryResult.isFetching) {
+        goPrev(e.altKey ? 'alt+arrow' : 'arrow');
+        return;
+      }
      handleUpImage(e.altKey);
    },
    { preventDefault: true },
-    [handleUpImage]
+    [handleUpImage, canNavigateGallery, isOnFirstRow, goPrev, isPrevEnabled, queryResult.isFetching]
  );

  useHotkeys(
    ['down', 'alt+down'],
    (e) => {
-      if (!areImagesBelowCurrent && isNextEnabled && !queryResult.isFetching) {
-        goNext();
+      if (isOnLastRow && isNextEnabled && !queryResult.isFetching) {
+        goNext(e.altKey ? 'alt+arrow' : 'arrow');
        return;
      }
      handleDownImage(e.altKey);
    },
    { preventDefault: true },
-    [areImagesBelowCurrent, goNext, isNextEnabled, queryResult.isFetching, handleDownImage]
+    [isOnLastRow, goNext, isNextEnabled, queryResult.isFetching, handleDownImage]
  );
 };
--- a/invokeai/frontend/web/src/features/gallery/hooks/useGalleryNavigation.ts
+++ b/invokeai/frontend/web/src/features/gallery/hooks/useGalleryNavigation.ts
@ -112,7 +112,8 @@ type UseGalleryNavigationReturn = {
  nextImage: () => void;
  isOnFirstImage: boolean;
  isOnLastImage: boolean;
-  areImagesBelowCurrent: boolean;
+  isOnFirstRow: boolean;
+  isOnLastRow: boolean;
  isOnFirstImageOfView: boolean;
  isOnLastImageOfView: boolean;
 };
@ -167,10 +168,11 @@ export const useGalleryNavigation = (): UseGalleryNavigationReturn => {
    [lastSelectedImageIndex, loadedImagesCount]
  );

-  const areImagesBelowCurrent = useMemo(() => {
-    const imagesPerRow = getImagesPerRow();
-    return lastSelectedImageIndex + imagesPerRow < loadedImagesCount;
-  }, [lastSelectedImageIndex, loadedImagesCount]);
+  const isOnFirstRow = useMemo(() => lastSelectedImageIndex < getImagesPerRow(), [lastSelectedImageIndex]);
+  const isOnLastRow = useMemo(
+    () => lastSelectedImageIndex >= loadedImagesCount - getImagesPerRow(),
+    [lastSelectedImageIndex, loadedImagesCount]
+  );

  const isOnFirstImageOfView = useMemo(() => {
    return lastSelectedImageIndex === 0;
@ -223,7 +225,8 @@ export const useGalleryNavigation = (): UseGalleryNavigationReturn => {
    handleDownImage,
    isOnFirstImage,
    isOnLastImage,
-    areImagesBelowCurrent,
+    isOnFirstRow,
+    isOnLastRow,
    nextImage,
    prevImage,
    isOnFirstImageOfView,
--- a/invokeai/frontend/web/src/features/gallery/hooks/useGalleryPagination.ts
+++ b/invokeai/frontend/web/src/features/gallery/hooks/useGalleryPagination.ts
@ -80,25 +80,31 @@ export const useGalleryPagination = () => {
    return offset > 0;
  }, [count, offset]);

-  const goNext = useCallback(() => {
-    dispatch(offsetChanged(offset + (limit || 0)));
-  }, [dispatch, offset, limit]);
+  const goNext = useCallback(
+    (withHotkey?: 'arrow' | 'alt+arrow') => {
+      dispatch(offsetChanged({ offset: offset + (limit || 0), withHotkey }));
+    },
+    [dispatch, offset, limit]
+  );

-  const goPrev = useCallback(() => {
-    dispatch(offsetChanged(Math.max(offset - (limit || 0), 0)));
-  }, [dispatch, offset, limit]);
+  const goPrev = useCallback(
+    (withHotkey?: 'arrow' | 'alt+arrow') => {
+      dispatch(offsetChanged({ offset: Math.max(offset - (limit || 0), 0), withHotkey }));
+    },
+    [dispatch, offset, limit]
+  );

  const goToPage = useCallback(
    (page: number) => {
-      dispatch(offsetChanged(page * (limit || 0)));
+      dispatch(offsetChanged({ offset: page * (limit || 0) }));
    },
    [dispatch, limit]
  );
  const goToFirst = useCallback(() => {
-    dispatch(offsetChanged(0));
+    dispatch(offsetChanged({ offset: 0 }));
  }, [dispatch]);
  const goToLast = useCallback(() => {
-    dispatch(offsetChanged((pages - 1) * (limit || 0)));
+    dispatch(offsetChanged({ offset: (pages - 1) * (limit || 0) }));
  }, [dispatch, pages, limit]);

  // handle when total/pages decrease and user is on high page number (ie bulk removing or deleting)
@ -127,44 +133,22 @@ export const useGalleryPagination = () => {
    return Math.min((currentPage + 1) * (limit || 0), total);
  }, [currentPage, limit, total]);

-  const api = useMemo(
-    () => ({
-      count,
-      total,
-      currentPage,
-      pages,
-      isNextEnabled,
-      isPrevEnabled,
-      goNext,
-      goPrev,
-      goToPage,
-      goToFirst,
-      goToLast,
-      pageButtons,
-      isFirstEnabled,
-      isLastEnabled,
-      rangeDisplay,
-      numberOnPage,
-    }),
-    [
-      count,
-      total,
-      currentPage,
-      pages,
-      isNextEnabled,
-      isPrevEnabled,
-      goNext,
-      goPrev,
-      goToPage,
-      goToFirst,
-      goToLast,
-      pageButtons,
-      isFirstEnabled,
-      isLastEnabled,
-      rangeDisplay,
-      numberOnPage,
-    ]
-  );
-
-  return api;
+  return {
+    count,
+    total,
+    currentPage,
+    pages,
+    isNextEnabled,
+    isPrevEnabled,
+    goNext,
+    goPrev,
+    goToPage,
+    goToFirst,
+    goToLast,
+    pageButtons,
+    isFirstEnabled,
+    isLastEnabled,
+    rangeDisplay,
+    numberOnPage,
+  };
 };
--- a/invokeai/frontend/web/src/features/gallery/store/gallerySlice.ts
+++ b/invokeai/frontend/web/src/features/gallery/store/gallerySlice.ts
@ -104,8 +104,9 @@ export const gallerySlice = createSlice({
    comparisonFitChanged: (state, action: PayloadAction<'contain' | 'fill'>) => {
      state.comparisonFit = action.payload;
    },
-    offsetChanged: (state, action: PayloadAction<number>) => {
-      state.offset = action.payload;
+    offsetChanged: (state, action: PayloadAction<{ offset: number; withHotkey?: 'arrow' | 'alt+arrow' }>) => {
+      const { offset } = action.payload;
+      state.offset = offset;
    },
    limitChanged: (state, action: PayloadAction<number>) => {
      state.limit = action.payload;