From 19c5435332d219f0fa6b89fdb334e37cb585b1f8 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:11:46 +1100 Subject: [PATCH] fix(ui): copy image via img onload to blob There's a bug in chrome that screws with headers on fetch requests and 307 responses. This causes images to fail to copy in the commercial environment. This change attempts to get around this by copying images in a different way (similar to how the canvas works). When the user requests a copy we: - create an `` element - set `crossOrigin` if needed - add an onload handler: - create a canvas element - draw image onto it - export canvas to blob This is wrapped in a promise which resolves to the blob, which can then be copied to clipboard. --- A customized version of Konva's `useImage` hook is also included, which returns the image blob in addition to the `` element. Unfortunately, this hook is not suitable for use across the app, because it does all the image fetching up front, regardless of whether we actually want to copy the image. In other words, we'd have to fetch the whole image file even if the user is just skipping through image metadata, in order to have the blob to copy. The callback approach means we only fetch the image when the user clicks copy. The hook is thus currently unused. --- .../frontend/web/src/common/hooks/useImage.ts | 102 ++++++++++++++++++ .../web/src/common/hooks/useImageUrlToBlob.ts | 40 +++++++ .../system/util/copyBlobToClipboard.ts | 2 +- .../ui/hooks/useCopyImageToClipboard.ts | 20 ++-- 4 files changed, 153 insertions(+), 11 deletions(-) create mode 100644 invokeai/frontend/web/src/common/hooks/useImage.ts create mode 100644 invokeai/frontend/web/src/common/hooks/useImageUrlToBlob.ts diff --git a/invokeai/frontend/web/src/common/hooks/useImage.ts b/invokeai/frontend/web/src/common/hooks/useImage.ts new file mode 100644 index 0000000000..60c973ce59 --- /dev/null +++ b/invokeai/frontend/web/src/common/hooks/useImage.ts @@ -0,0 +1,102 @@ +import { useLayoutEffect, useRef, useState } from 'react'; + +// Adapted from https://github.com/konvajs/use-image + +type CrossOrigin = 'anonymous' | 'use-credentials'; +type ReferrerPolicy = + | 'no-referrer' + | 'no-referrer-when-downgrade' + | 'origin' + | 'origin-when-cross-origin' + | 'same-origin' + | 'strict-origin' + | 'strict-origin-when-cross-origin' + | 'unsafe-url'; +type ImageStatus = 'loaded' | 'loading' | 'failed'; + +export const useImage = ( + url: string, + crossOrigin?: CrossOrigin, + referrerpolicy?: ReferrerPolicy +): [undefined | HTMLImageElement, ImageStatus, Blob | null] => { + // lets use refs for image and status + // so we can update them during render + // to have instant update in status/image when new data comes in + const statusRef = useRef('loading'); + const imageRef = useRef(); + const blobRef = useRef(null); + + // we are not going to use token + // but we need to just to trigger state update + const [_, setStateToken] = useState(0); + + // keep track of old props to trigger changes + const oldUrl = useRef(); + const oldCrossOrigin = useRef(); + const oldReferrerPolicy = useRef(); + + if ( + oldUrl.current !== url || + oldCrossOrigin.current !== crossOrigin || + oldReferrerPolicy.current !== referrerpolicy + ) { + statusRef.current = 'loading'; + imageRef.current = undefined; + oldUrl.current = url; + oldCrossOrigin.current = crossOrigin; + oldReferrerPolicy.current = referrerpolicy; + } + + useLayoutEffect( + function () { + if (!url) { + return; + } + const img = document.createElement('img'); + + function onload() { + statusRef.current = 'loaded'; + imageRef.current = img; + const canvas = document.createElement('canvas'); + canvas.width = img.clientWidth; + canvas.height = img.clientHeight; + + const context = canvas.getContext('2d'); + if (context) { + context.drawImage(img, 0, 0); + canvas.toBlob(function (blob) { + blobRef.current = blob; + }, 'image/png'); + } + setStateToken(Math.random()); + } + + function onerror() { + statusRef.current = 'failed'; + imageRef.current = undefined; + setStateToken(Math.random()); + } + + img.addEventListener('load', onload); + img.addEventListener('error', onerror); + if (crossOrigin) { + img.crossOrigin = crossOrigin; + } + if (referrerpolicy) { + img.referrerPolicy = referrerpolicy; + } + img.src = url; + + return function cleanup() { + img.removeEventListener('load', onload); + img.removeEventListener('error', onerror); + }; + }, + [url, crossOrigin, referrerpolicy] + ); + + // return array because it is better to use in case of several useImage hooks + // const [background, backgroundStatus] = useImage(url1); + // const [patter] = useImage(url2); + return [imageRef.current, statusRef.current, blobRef.current]; +}; diff --git a/invokeai/frontend/web/src/common/hooks/useImageUrlToBlob.ts b/invokeai/frontend/web/src/common/hooks/useImageUrlToBlob.ts new file mode 100644 index 0000000000..77538a929d --- /dev/null +++ b/invokeai/frontend/web/src/common/hooks/useImageUrlToBlob.ts @@ -0,0 +1,40 @@ +import { useCallback } from 'react'; +import { $authToken } from 'services/api/client'; + +/** + * Converts an image URL to a Blob by creating an element, drawing it to canvas + * and then converting the canvas to a Blob. + * + * @returns A function that takes a URL and returns a Promise that resolves with a Blob + */ +export const useImageUrlToBlob = () => { + const imageUrlToBlob = useCallback( + async (url: string) => + new Promise((resolve) => { + const img = new Image(); + img.onload = () => { + const canvas = document.createElement('canvas'); + canvas.width = img.width; + canvas.height = img.height; + + const context = canvas.getContext('2d'); + if (!context) { + return; + } + context.drawImage(img, 0, 0); + resolve( + new Promise((resolve) => { + canvas.toBlob(function (blob) { + resolve(blob); + }, 'image/png'); + }) + ); + }; + img.crossOrigin = $authToken.get() ? 'use-credentials' : 'anonymous'; + img.src = url; + }), + [] + ); + + return imageUrlToBlob; +}; diff --git a/invokeai/frontend/web/src/features/system/util/copyBlobToClipboard.ts b/invokeai/frontend/web/src/features/system/util/copyBlobToClipboard.ts index cf59f2a687..b5e896f3bf 100644 --- a/invokeai/frontend/web/src/features/system/util/copyBlobToClipboard.ts +++ b/invokeai/frontend/web/src/features/system/util/copyBlobToClipboard.ts @@ -2,7 +2,7 @@ * Copies a blob to the clipboard by calling navigator.clipboard.write(). */ export const copyBlobToClipboard = ( - blob: Promise, + blob: Promise | Blob, type = 'image/png' ) => { navigator.clipboard.write([ diff --git a/invokeai/frontend/web/src/features/ui/hooks/useCopyImageToClipboard.ts b/invokeai/frontend/web/src/features/ui/hooks/useCopyImageToClipboard.ts index 4b42a45e93..ef9db44a9d 100644 --- a/invokeai/frontend/web/src/features/ui/hooks/useCopyImageToClipboard.ts +++ b/invokeai/frontend/web/src/features/ui/hooks/useCopyImageToClipboard.ts @@ -1,11 +1,13 @@ import { useAppToaster } from 'app/components/Toaster'; +import { useImageUrlToBlob } from 'common/hooks/useImageUrlToBlob'; +import { copyBlobToClipboard } from 'features/system/util/copyBlobToClipboard'; import { useCallback, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; -import { copyBlobToClipboard } from 'features/system/util/copyBlobToClipboard'; export const useCopyImageToClipboard = () => { const toaster = useAppToaster(); const { t } = useTranslation(); + const imageUrlToBlob = useImageUrlToBlob(); const isClipboardAPIAvailable = useMemo(() => { return Boolean(navigator.clipboard) && Boolean(window.ClipboardItem); @@ -23,15 +25,13 @@ export const useCopyImageToClipboard = () => { }); } try { - const getImageBlob = async () => { - const response = await fetch(image_url); - if (!response.ok) { - throw new Error(`Problem retrieving image data`); - } - return await response.blob(); - }; + const blob = await imageUrlToBlob(image_url); - copyBlobToClipboard(getImageBlob()); + if (!blob) { + throw new Error('Unable to create Blob'); + } + + copyBlobToClipboard(blob); toaster({ title: t('toast.imageCopied'), @@ -49,7 +49,7 @@ export const useCopyImageToClipboard = () => { }); } }, - [isClipboardAPIAvailable, t, toaster] + [imageUrlToBlob, isClipboardAPIAvailable, t, toaster] ); return { isClipboardAPIAvailable, copyImageToClipboard };