feat: Replace Seam Painting with Refine Pass for Outpainting

This commit is contained in:
blessedcoolant 2023-08-24 04:40:59 +12:00 committed by Kent Keirsey
parent 9bafe4a94f
commit 0af8a0e84b
13 changed files with 87 additions and 435 deletions

View File

@ -509,12 +509,9 @@
"maskAdjustmentsHeader": "Mask Adjustments",
"maskBlur": "Mask Blur",
"maskBlurMethod": "Mask Blur Method",
"seamPaintingHeader": "Seam Painting",
"seamSize": "Seam Size",
"seamBlur": "Seam Blur",
"seamSteps": "Seam Steps",
"seamStrength": "Seam Strength",
"seamThreshold": "Seam Threshold",
"refinePassHeader": "Refine Pass",
"refineSteps": "Refine Steps",
"refineStrength": "Refine Strength",
"seamLowThreshold": "Low",
"seamHighThreshold": "High",
"scaleBeforeProcessing": "Scale Before Processing",

View File

@ -20,7 +20,6 @@ import {
CANVAS_OUTPAINT_GRAPH,
CANVAS_OUTPUT,
CLIP_SKIP,
COLOR_CORRECT,
DENOISE_LATENTS,
INPAINT_IMAGE,
INPAINT_IMAGE_RESIZE_DOWN,
@ -32,16 +31,15 @@ import {
MAIN_MODEL_LOADER,
MASK_BLUR,
MASK_COMBINE,
MASK_EDGE,
MASK_FROM_ALPHA,
MASK_RESIZE_DOWN,
MASK_RESIZE_UP,
NEGATIVE_CONDITIONING,
NOISE,
OUTPAINT_REFINE_DENOISE_LATENTS,
POSITIVE_CONDITIONING,
RANDOM_INT,
RANGE_OF_SIZE,
SEAM_FIX_DENOISE_LATENTS,
SEAM_MASK_RESIZE_UP,
} from './constants';
@ -70,12 +68,8 @@ export const buildCanvasOutpaintGraph = (
shouldUseCpuNoise,
maskBlur,
maskBlurMethod,
seamSize,
seamBlur,
seamSteps,
seamStrength,
seamLowThreshold,
seamHighThreshold,
refineSteps,
refineStrength,
tileSize,
infillMethod,
clipSkip,
@ -174,23 +168,23 @@ export const buildCanvasOutpaintGraph = (
denoising_start: 1 - strength,
denoising_end: 1,
},
[MASK_EDGE]: {
type: 'mask_edge',
id: MASK_EDGE,
is_intermediate: true,
edge_size: seamSize,
edge_blur: seamBlur,
low_threshold: seamLowThreshold,
high_threshold: seamHighThreshold,
},
[SEAM_FIX_DENOISE_LATENTS]: {
// [MASK_EDGE]: {
// type: 'mask_edge',
// id: MASK_EDGE,
// is_intermediate: true,
// edge_size: seamSize,
// edge_blur: seamBlur,
// low_threshold: seamLowThreshold,
// high_threshold: seamHighThreshold,
// },
[OUTPAINT_REFINE_DENOISE_LATENTS]: {
type: 'denoise_latents',
id: SEAM_FIX_DENOISE_LATENTS,
id: OUTPAINT_REFINE_DENOISE_LATENTS,
is_intermediate: true,
steps: seamSteps,
steps: refineSteps,
cfg_scale: cfg_scale,
scheduler: scheduler,
denoising_start: 1 - seamStrength,
denoising_start: 1 - refineStrength,
denoising_end: 1,
},
[LATENTS_TO_IMAGE]: {
@ -199,13 +193,8 @@ export const buildCanvasOutpaintGraph = (
is_intermediate: true,
fp32: vaePrecision === 'fp32' ? true : false,
},
[COLOR_CORRECT]: {
type: 'color_correct',
id: COLOR_CORRECT,
is_intermediate: true,
},
[CANVAS_OUTPUT]: {
type: 'img_paste',
type: 'color_correct',
id: CANVAS_OUTPUT,
is_intermediate: !shouldAutoSave,
},
@ -361,24 +350,25 @@ export const buildCanvasOutpaintGraph = (
field: 'seed',
},
},
// Seam Paint
// Image To Image Clean Up
{
source: {
node_id: MAIN_MODEL_LOADER,
field: 'unet',
},
destination: {
node_id: SEAM_FIX_DENOISE_LATENTS,
node_id: OUTPAINT_REFINE_DENOISE_LATENTS,
field: 'unet',
},
},
{
source: {
node_id: POSITIVE_CONDITIONING,
field: 'conditioning',
},
destination: {
node_id: SEAM_FIX_DENOISE_LATENTS,
node_id: OUTPAINT_REFINE_DENOISE_LATENTS,
field: 'positive_conditioning',
},
},
@ -388,7 +378,7 @@ export const buildCanvasOutpaintGraph = (
field: 'conditioning',
},
destination: {
node_id: SEAM_FIX_DENOISE_LATENTS,
node_id: OUTPAINT_REFINE_DENOISE_LATENTS,
field: 'negative_conditioning',
},
},
@ -398,7 +388,7 @@ export const buildCanvasOutpaintGraph = (
field: 'noise',
},
destination: {
node_id: SEAM_FIX_DENOISE_LATENTS,
node_id: OUTPAINT_REFINE_DENOISE_LATENTS,
field: 'noise',
},
},
@ -408,14 +398,14 @@ export const buildCanvasOutpaintGraph = (
field: 'latents',
},
destination: {
node_id: SEAM_FIX_DENOISE_LATENTS,
node_id: OUTPAINT_REFINE_DENOISE_LATENTS,
field: 'latents',
},
},
// Decode the result from Inpaint
{
source: {
node_id: SEAM_FIX_DENOISE_LATENTS,
node_id: OUTPAINT_REFINE_DENOISE_LATENTS,
field: 'latents',
},
destination: {
@ -525,37 +515,6 @@ export const buildCanvasOutpaintGraph = (
field: 'image',
},
},
// Seam Paint Mask
{
source: {
node_id: MASK_FROM_ALPHA,
field: 'image',
},
destination: {
node_id: MASK_EDGE,
field: 'image',
},
},
{
source: {
node_id: MASK_EDGE,
field: 'image',
},
destination: {
node_id: SEAM_MASK_RESIZE_UP,
field: 'image',
},
},
{
source: {
node_id: SEAM_MASK_RESIZE_UP,
field: 'image',
},
destination: {
node_id: SEAM_FIX_DENOISE_LATENTS,
field: 'mask',
},
},
// Resize Results Down
{
source: {
@ -594,7 +553,7 @@ export const buildCanvasOutpaintGraph = (
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
node_id: CANVAS_OUTPUT,
field: 'reference',
},
},
@ -603,37 +562,6 @@ export const buildCanvasOutpaintGraph = (
node_id: INPAINT_IMAGE_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'image',
},
},
{
source: {
node_id: MASK_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'mask',
},
},
// Paste Everything Back
{
source: {
node_id: INPAINT_INFILL_RESIZE_DOWN,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'base_image',
},
},
{
source: {
node_id: COLOR_CORRECT,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',
@ -683,27 +611,6 @@ export const buildCanvasOutpaintGraph = (
field: 'image',
},
},
// Seam Paint Mask
{
source: {
node_id: MASK_FROM_ALPHA,
field: 'image',
},
destination: {
node_id: MASK_EDGE,
field: 'image',
},
},
{
source: {
node_id: MASK_EDGE,
field: 'image',
},
destination: {
node_id: SEAM_FIX_DENOISE_LATENTS,
field: 'mask',
},
},
// Color Correct The Inpainted Result
{
source: {
@ -711,7 +618,7 @@ export const buildCanvasOutpaintGraph = (
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
node_id: CANVAS_OUTPUT,
field: 'reference',
},
},
@ -720,37 +627,6 @@ export const buildCanvasOutpaintGraph = (
node_id: LATENTS_TO_IMAGE,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'image',
},
},
{
source: {
node_id: MASK_BLUR,
field: 'image',
},
destination: {
node_id: COLOR_CORRECT,
field: 'mask',
},
},
// Paste Everything Back
{
source: {
node_id: INPAINT_INFILL,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'base_image',
},
},
{
source: {
node_id: COLOR_CORRECT,
field: 'image',
},
destination: {
node_id: CANVAS_OUTPUT,
field: 'image',

View File

@ -26,6 +26,8 @@ export const INPAINT_INFILL = 'inpaint_infill';
export const INPAINT_INFILL_RESIZE_DOWN = 'inpaint_infill_resize_down';
export const INPAINT_FINAL_IMAGE = 'inpaint_final_image';
export const SEAM_FIX_DENOISE_LATENTS = 'seam_fix_denoise_latents';
export const OUTPAINT_REFINE_DENOISE_LATENTS =
'outpaint_refine_denoise_latents';
export const MASK_FROM_ALPHA = 'tomask';
export const MASK_EDGE = 'mask_edge';
export const MASK_BLUR = 'mask_blur';

View File

@ -0,0 +1,21 @@
import { Flex } from '@chakra-ui/react';
import IAICollapse from 'common/components/IAICollapse';
import { memo } from 'react';
import { useTranslation } from 'react-i18next';
import ParamRefineSteps from './ParamRefineSteps';
import ParamRefineStrength from './ParamRefineStrength';
const ParamRefinePassCollapse = () => {
const { t } = useTranslation();
return (
<IAICollapse label={t('parameters.refinePassHeader')}>
<Flex sx={{ flexDirection: 'column', gap: 2, paddingBottom: 2 }}>
<ParamRefineSteps />
<ParamRefineStrength />
</Flex>
</IAICollapse>
);
};
export default memo(ParamRefinePassCollapse);

View File

@ -1,36 +1,36 @@
import type { RootState } from 'app/store/store';
import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
import IAISlider from 'common/components/IAISlider';
import { setSeamSteps } from 'features/parameters/store/generationSlice';
import { setRefineSteps } from 'features/parameters/store/generationSlice';
import { memo } from 'react';
import { useTranslation } from 'react-i18next';
const ParamSeamSteps = () => {
const ParamRefineSteps = () => {
const dispatch = useAppDispatch();
const seamSteps = useAppSelector(
(state: RootState) => state.generation.seamSteps
const refineSteps = useAppSelector(
(state: RootState) => state.generation.refineSteps
);
const { t } = useTranslation();
return (
<IAISlider
label={t('parameters.seamSteps')}
label={t('parameters.refineSteps')}
min={0}
max={100}
step={1}
sliderNumberInputProps={{ max: 999 }}
value={seamSteps}
value={refineSteps}
onChange={(v) => {
dispatch(setSeamSteps(v));
dispatch(setRefineSteps(v));
}}
withInput
withSliderMarks
withReset
handleReset={() => {
dispatch(setSeamSteps(20));
dispatch(setRefineSteps(20));
}}
/>
);
};
export default memo(ParamSeamSteps);
export default memo(ParamRefineSteps);

View File

@ -1,36 +1,36 @@
import type { RootState } from 'app/store/store';
import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
import IAISlider from 'common/components/IAISlider';
import { setSeamStrength } from 'features/parameters/store/generationSlice';
import { setRefineStrength } from 'features/parameters/store/generationSlice';
import { memo } from 'react';
import { useTranslation } from 'react-i18next';
const ParamSeamStrength = () => {
const ParamRefineStrength = () => {
const dispatch = useAppDispatch();
const seamStrength = useAppSelector(
(state: RootState) => state.generation.seamStrength
const refineStrength = useAppSelector(
(state: RootState) => state.generation.refineStrength
);
const { t } = useTranslation();
return (
<IAISlider
label={t('parameters.seamStrength')}
label={t('parameters.refineStrength')}
min={0}
max={1}
step={0.01}
sliderNumberInputProps={{ max: 999 }}
value={seamStrength}
value={refineStrength}
onChange={(v) => {
dispatch(setSeamStrength(v));
dispatch(setRefineStrength(v));
}}
withInput
withSliderMarks
withReset
handleReset={() => {
dispatch(setSeamStrength(0.7));
dispatch(setRefineStrength(0.3));
}}
/>
);
};
export default memo(ParamSeamStrength);
export default memo(ParamRefineStrength);

View File

@ -1,36 +0,0 @@
import type { RootState } from 'app/store/store';
import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
import IAISlider from 'common/components/IAISlider';
import { setSeamBlur } from 'features/parameters/store/generationSlice';
import { memo } from 'react';
import { useTranslation } from 'react-i18next';
const ParamSeamBlur = () => {
const dispatch = useAppDispatch();
const seamBlur = useAppSelector(
(state: RootState) => state.generation.seamBlur
);
const { t } = useTranslation();
return (
<IAISlider
label={t('parameters.seamBlur')}
min={0}
max={64}
step={8}
sliderNumberInputProps={{ max: 512 }}
value={seamBlur}
onChange={(v) => {
dispatch(setSeamBlur(v));
}}
withInput
withSliderMarks
withReset
handleReset={() => {
dispatch(setSeamBlur(8));
}}
/>
);
};
export default memo(ParamSeamBlur);

View File

@ -1,27 +0,0 @@
import { Flex } from '@chakra-ui/react';
import IAICollapse from 'common/components/IAICollapse';
import { memo } from 'react';
import { useTranslation } from 'react-i18next';
import ParamSeamBlur from './ParamSeamBlur';
import ParamSeamSize from './ParamSeamSize';
import ParamSeamSteps from './ParamSeamSteps';
import ParamSeamStrength from './ParamSeamStrength';
import ParamSeamThreshold from './ParamSeamThreshold';
const ParamSeamPaintingCollapse = () => {
const { t } = useTranslation();
return (
<IAICollapse label={t('parameters.seamPaintingHeader')}>
<Flex sx={{ flexDirection: 'column', gap: 2, paddingBottom: 2 }}>
<ParamSeamSize />
<ParamSeamBlur />
<ParamSeamSteps />
<ParamSeamStrength />
<ParamSeamThreshold />
</Flex>
</IAICollapse>
);
};
export default memo(ParamSeamPaintingCollapse);

View File

@ -1,36 +0,0 @@
import type { RootState } from 'app/store/store';
import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
import IAISlider from 'common/components/IAISlider';
import { setSeamSize } from 'features/parameters/store/generationSlice';
import { memo } from 'react';
import { useTranslation } from 'react-i18next';
const ParamSeamSize = () => {
const dispatch = useAppDispatch();
const seamSize = useAppSelector(
(state: RootState) => state.generation.seamSize
);
const { t } = useTranslation();
return (
<IAISlider
label={t('parameters.seamSize')}
min={0}
max={128}
step={8}
sliderNumberInputProps={{ max: 512 }}
value={seamSize}
onChange={(v) => {
dispatch(setSeamSize(v));
}}
withInput
withSliderMarks
withReset
handleReset={() => {
dispatch(setSeamSize(16));
}}
/>
);
};
export default memo(ParamSeamSize);

View File

@ -1,121 +0,0 @@
import {
FormControl,
FormLabel,
HStack,
RangeSlider,
RangeSliderFilledTrack,
RangeSliderMark,
RangeSliderThumb,
RangeSliderTrack,
Tooltip,
} from '@chakra-ui/react';
import type { RootState } from 'app/store/store';
import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
import IAIIconButton from 'common/components/IAIIconButton';
import {
setSeamHighThreshold,
setSeamLowThreshold,
} from 'features/parameters/store/generationSlice';
import { memo, useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import { BiReset } from 'react-icons/bi';
const ParamSeamThreshold = () => {
const dispatch = useAppDispatch();
const seamLowThreshold = useAppSelector(
(state: RootState) => state.generation.seamLowThreshold
);
const seamHighThreshold = useAppSelector(
(state: RootState) => state.generation.seamHighThreshold
);
const { t } = useTranslation();
const handleSeamThresholdChange = useCallback(
(v: number[]) => {
dispatch(setSeamLowThreshold(v[0] as number));
dispatch(setSeamHighThreshold(v[1] as number));
},
[dispatch]
);
const handleSeamThresholdReset = () => {
dispatch(setSeamLowThreshold(100));
dispatch(setSeamHighThreshold(200));
};
return (
<FormControl>
<FormLabel>{t('parameters.seamThreshold')}</FormLabel>
<HStack w="100%" gap={4} mt={-2}>
<RangeSlider
aria-label={[
t('parameters.seamLowThreshold'),
t('parameters.seamHighThreshold'),
]}
value={[seamLowThreshold, seamHighThreshold]}
min={0}
max={255}
step={1}
minStepsBetweenThumbs={1}
onChange={handleSeamThresholdChange}
>
<RangeSliderTrack>
<RangeSliderFilledTrack />
</RangeSliderTrack>
<Tooltip label={seamLowThreshold} placement="top" hasArrow>
<RangeSliderThumb index={0} />
</Tooltip>
<Tooltip label={seamHighThreshold} placement="top" hasArrow>
<RangeSliderThumb index={1} />
</Tooltip>
<RangeSliderMark
value={0}
sx={{
insetInlineStart: '0 !important',
insetInlineEnd: 'unset !important',
}}
>
0
</RangeSliderMark>
<RangeSliderMark
value={0.392}
sx={{
insetInlineStart: '38.4% !important',
transform: 'translateX(-38.4%)',
}}
>
100
</RangeSliderMark>
<RangeSliderMark
value={0.784}
sx={{
insetInlineStart: '79.8% !important',
transform: 'translateX(-79.8%)',
}}
>
200
</RangeSliderMark>
<RangeSliderMark
value={1}
sx={{
insetInlineStart: 'unset !important',
insetInlineEnd: '0 !important',
}}
>
255
</RangeSliderMark>
</RangeSlider>
<IAIIconButton
size="sm"
aria-label={t('accessibility.reset')}
tooltip={t('accessibility.reset')}
icon={<BiReset />}
onClick={handleSeamThresholdReset}
/>
</HStack>
</FormControl>
);
};
export default memo(ParamSeamThreshold);

View File

@ -37,12 +37,8 @@ export interface GenerationState {
scheduler: SchedulerParam;
maskBlur: number;
maskBlurMethod: MaskBlurMethodParam;
seamSize: number;
seamBlur: number;
seamSteps: number;
seamStrength: StrengthParam;
seamLowThreshold: number;
seamHighThreshold: number;
refineSteps: number;
refineStrength: StrengthParam;
seed: SeedParam;
seedWeights: string;
shouldFitToWidthHeight: boolean;
@ -80,12 +76,8 @@ export const initialGenerationState: GenerationState = {
scheduler: 'euler',
maskBlur: 16,
maskBlurMethod: 'box',
seamSize: 16,
seamBlur: 8,
seamSteps: 20,
seamStrength: 0.7,
seamLowThreshold: 100,
seamHighThreshold: 200,
refineSteps: 20,
refineStrength: 0.3,
seed: 0,
seedWeights: '',
shouldFitToWidthHeight: true,
@ -212,23 +204,11 @@ export const generationSlice = createSlice({
setMaskBlurMethod: (state, action: PayloadAction<MaskBlurMethodParam>) => {
state.maskBlurMethod = action.payload;
},
setSeamSize: (state, action: PayloadAction<number>) => {
state.seamSize = action.payload;
setRefineSteps: (state, action: PayloadAction<number>) => {
state.refineSteps = action.payload;
},
setSeamBlur: (state, action: PayloadAction<number>) => {
state.seamBlur = action.payload;
},
setSeamSteps: (state, action: PayloadAction<number>) => {
state.seamSteps = action.payload;
},
setSeamStrength: (state, action: PayloadAction<number>) => {
state.seamStrength = action.payload;
},
setSeamLowThreshold: (state, action: PayloadAction<number>) => {
state.seamLowThreshold = action.payload;
},
setSeamHighThreshold: (state, action: PayloadAction<number>) => {
state.seamHighThreshold = action.payload;
setRefineStrength: (state, action: PayloadAction<number>) => {
state.refineStrength = action.payload;
},
setTileSize: (state, action: PayloadAction<number>) => {
state.tileSize = action.payload;
@ -338,12 +318,8 @@ export const {
setScheduler,
setMaskBlur,
setMaskBlurMethod,
setSeamSize,
setSeamBlur,
setSeamSteps,
setSeamStrength,
setSeamLowThreshold,
setSeamHighThreshold,
setRefineSteps,
setRefineStrength,
setSeed,
setSeedWeights,
setShouldFitToWidthHeight,

View File

@ -2,7 +2,7 @@ import ParamDynamicPromptsCollapse from 'features/dynamicPrompts/components/Para
import ParamLoraCollapse from 'features/lora/components/ParamLoraCollapse';
import ParamInfillAndScalingCollapse from 'features/parameters/components/Parameters/Canvas/InfillAndScaling/ParamInfillAndScalingCollapse';
import ParamMaskAdjustmentCollapse from 'features/parameters/components/Parameters/Canvas/MaskAdjustment/ParamMaskAdjustmentCollapse';
import ParamSeamPaintingCollapse from 'features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamPaintingCollapse';
import ParamRefinePassCollapse from 'features/parameters/components/Parameters/Canvas/SeamPainting/ParamRefinePassCollapse';
import ParamControlNetCollapse from 'features/parameters/components/Parameters/ControlNet/ParamControlNetCollapse';
import ParamNoiseCollapse from 'features/parameters/components/Parameters/Noise/ParamNoiseCollapse';
import ParamSDXLPromptArea from './ParamSDXLPromptArea';
@ -21,7 +21,7 @@ export default function SDXLUnifiedCanvasTabParameters() {
<ParamNoiseCollapse />
<ParamMaskAdjustmentCollapse />
<ParamInfillAndScalingCollapse />
<ParamSeamPaintingCollapse />
<ParamRefinePassCollapse />
</>
);
}

View File

@ -2,11 +2,11 @@ import ParamDynamicPromptsCollapse from 'features/dynamicPrompts/components/Para
import ParamLoraCollapse from 'features/lora/components/ParamLoraCollapse';
import ParamAdvancedCollapse from 'features/parameters/components/Parameters/Advanced/ParamAdvancedCollapse';
import ParamInfillAndScalingCollapse from 'features/parameters/components/Parameters/Canvas/InfillAndScaling/ParamInfillAndScalingCollapse';
import ParamControlNetCollapse from 'features/parameters/components/Parameters/ControlNet/ParamControlNetCollapse';
import ParamSymmetryCollapse from 'features/parameters/components/Parameters/Symmetry/ParamSymmetryCollapse';
import ParamMaskAdjustmentCollapse from 'features/parameters/components/Parameters/Canvas/MaskAdjustment/ParamMaskAdjustmentCollapse';
import ParamSeamPaintingCollapse from 'features/parameters/components/Parameters/Canvas/SeamPainting/ParamSeamPaintingCollapse';
import ParamRefinePassCollapse from 'features/parameters/components/Parameters/Canvas/SeamPainting/ParamRefinePassCollapse';
import ParamControlNetCollapse from 'features/parameters/components/Parameters/ControlNet/ParamControlNetCollapse';
import ParamPromptArea from 'features/parameters/components/Parameters/Prompt/ParamPromptArea';
import ParamSymmetryCollapse from 'features/parameters/components/Parameters/Symmetry/ParamSymmetryCollapse';
import { memo } from 'react';
import UnifiedCanvasCoreParameters from './UnifiedCanvasCoreParameters';
@ -21,7 +21,7 @@ const UnifiedCanvasParameters = () => {
<ParamSymmetryCollapse />
<ParamMaskAdjustmentCollapse />
<ParamInfillAndScalingCollapse />
<ParamSeamPaintingCollapse />
<ParamRefinePassCollapse />
<ParamAdvancedCollapse />
</>
);