diff --git a/index.html b/index.html index 421ac8a..fdb4e07 100644 --- a/index.html +++ b/index.html @@ -1185,6 +1185,7 @@
+ diff --git a/index.js b/index.js index 9198809..569677d 100644 --- a/index.js +++ b/index.js @@ -94,6 +94,8 @@ const { lexica, api_ts, comfyui, + comfyui_util, + diffusion_chain, } = require('./typescripts/dist/bundle') const io = require('./utility/io') @@ -1814,3 +1816,6 @@ async function openFileFromUrlExe(url, format = 'gif') { await openFileFromUrl(url, format) }) } + +let comfy_server = new diffusion_chain.ComfyServer('http://127.0.0.1:8188') +let comfy_object_info = diffusion_chain.ComfyApi.objectInfo(comfy_server) diff --git a/package-lock.json b/package-lock.json index a04a78f..47dab9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@types/react": "^18.2.6", "@types/react-dom": "^18.2.4", "changedpi": "^1.0.4", + "diffusion-chain": "file:../diffusion-chain", "fastify": "^4.10.2", "jimp": "^0.16.2", "madge": "^6.0.0", @@ -49,6 +50,16 @@ "yazl": "^2.5.1" } }, + "../diffusion-chain": { + "version": "1.0.7", + "license": "MIT", + "dependencies": { + "@types/node": "^20.4.0", + "mkdirp": "^3.0.1", + "ts-node": "^10.9.1", + "typescript": "^5.1.6" + } + }, "node_modules/@ampproject/remapping": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", @@ -4731,6 +4742,10 @@ "node": ">=4.2.0" } }, + "node_modules/diffusion-chain": { + "resolved": "../diffusion-chain", + "link": true + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -9272,15 +9287,15 @@ } }, "node_modules/typescript": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", - "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", + "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=12.20" + "node": ">=14.17" } }, "node_modules/undefsafe": { @@ -13368,6 +13383,15 @@ } } }, + "diffusion-chain": { + "version": "file:../diffusion-chain", + "requires": { + "@types/node": "^20.4.0", + "mkdirp": "^3.0.1", + "ts-node": "^10.9.1", + "typescript": "^5.1.6" + } + }, "dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -16652,9 +16676,9 @@ } }, "typescript": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", - "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==" + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", + "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==" }, "undefsafe": { "version": "2.0.5", diff --git a/package.json b/package.json index 356303a..bd4b005 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "@types/react": "^18.2.6", "@types/react-dom": "^18.2.4", "changedpi": "^1.0.4", + "diffusion-chain": "file:../diffusion-chain", "fastify": "^4.10.2", "jimp": "^0.16.2", "madge": "^6.0.0", @@ -30,6 +31,7 @@ "@babel/plugin-transform-react-jsx": "^7.21.5", "@svgr/webpack": "^8.0.1", "babel-loader": "^9.1.2", + "chalk": "^5.3.0", "clean-webpack-plugin": "^4.0.0", "commander": "^11.0.0", "copy-webpack-plugin": "^11.0.0", @@ -45,7 +47,6 @@ "url-loader": "^4.1.1", "webpack": "^5.82.1", "webpack-cli": "^5.1.1", - "chalk": "^5.3.0", "yazl": "^2.5.1" }, "scripts": { @@ -64,4 +65,4 @@ "url": "https://github.com/AbdullahAlfaraj/Auto-Photoshop-StableDiffusion-Plugin/issues" }, "homepage": "https://github.com/AbdullahAlfaraj/Auto-Photoshop-StableDiffusion-Plugin#readme" -} \ No newline at end of file +} diff --git a/typescripts/comfyui/animatediff_workflow.json b/typescripts/comfyui/animatediff_workflow.json new file mode 100644 index 0000000..c1353e1 --- /dev/null +++ b/typescripts/comfyui/animatediff_workflow.json @@ -0,0 +1,92 @@ +{ + "1": { + "inputs": { + "ckpt_name": "cardosAnime_v20.safetensors", + "beta_schedule": "sqrt_linear (AnimateDiff)" + }, + "class_type": "CheckpointLoaderSimpleWithNoiseSelect" + }, + "2": { + "inputs": { + "vae_name": "MoistMix.vae.pt" + }, + "class_type": "VAELoader" + }, + "3": { + "inputs": { + "text": "ship in storm, waves, dark, night, Artstation ", + "clip": ["4", 0] + }, + "class_type": "CLIPTextEncode" + }, + "4": { + "inputs": { + "stop_at_clip_layer": -2, + "clip": ["1", 1] + }, + "class_type": "CLIPSetLastLayer" + }, + "6": { + "inputs": { + "text": "(ugly:1.2), (worst quality, low quality: 1.4)", + "clip": ["4", 0] + }, + "class_type": "CLIPTextEncode" + }, + "7": { + "inputs": { + "seed": 711493021904285, + "steps": 20, + "cfg": 8, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 1, + "model": ["8", 0], + "positive": ["3", 0], + "negative": ["6", 0], + "latent_image": ["8", 1] + }, + "class_type": "KSampler" + }, + "8": { + "inputs": { + "model_name": "mm_sd_v14.ckpt", + "unlimited_area_hack": true, + "model": ["1", 0], + "latents": ["9", 0] + }, + "class_type": "AnimateDiffLoaderV1" + }, + "9": { + "inputs": { + "width": 512, + "height": 512, + "batch_size": 16 + }, + "class_type": "EmptyLatentImage" + }, + "10": { + "inputs": { + "samples": ["7", 0], + "vae": ["2", 0] + }, + "class_type": "VAEDecode" + }, + "12": { + "inputs": { + "filename_prefix": "AA_readme", + "images": ["10", 0] + }, + "class_type": "SaveImage" + }, + "26": { + "inputs": { + "frame_rate": 8, + "loop_count": 0, + "save_image": "Enabled", + "filename_prefix": "AA_readme_gif", + "images": ["10", 0] + }, + "class_type": "ADE_AnimateDiffCombine" + } +} diff --git a/typescripts/comfyui/comfyui.tsx b/typescripts/comfyui/comfyui.tsx index aad1d33..b3a2686 100644 --- a/typescripts/comfyui/comfyui.tsx +++ b/typescripts/comfyui/comfyui.tsx @@ -2,10 +2,13 @@ import React from 'react' import ReactDOM from 'react-dom/client' import { requestGet, requestPost } from '../util/ts/api' import { observer } from 'mobx-react' +import { runInAction } from 'mobx' import { MoveToCanvasSvg, + SliderType, SpMenu, SpSlider, + SpSliderWithLabel, SpTextfield, } from '../util/elements' import { ErrorBoundary } from '../util/errorBoundary' @@ -13,15 +16,15 @@ import { Collapsible } from '../util/collapsible' import Locale from '../locale/locale' import { AStore } from '../main/astore' -import hi_res_prompt from './prompt.json' import { Grid } from '../util/grid' import { io } from '../util/oldSystem' import { app } from 'photoshop' import { reaction, toJS } from 'mobx' import { storage } from 'uxp' -export let hi_res_prompt_temp = hi_res_prompt -console.log('hi_res_prompt: ', hi_res_prompt) +import util from './util' +import * as diffusion_chain from 'diffusion-chain' +import { urlToCanvas } from '../util/ts/general' interface Error { type: string message: string @@ -40,49 +43,6 @@ interface Result { node_errors?: { [key: string]: NodeError } } -const result: Result = { - error: { - type: 'prompt_outputs_failed_validation', - message: 'Prompt outputs failed validation', - details: '', - extra_info: {}, - }, - node_errors: { - '16': { - errors: [ - { - type: 'value_not_in_list', - message: 'Value not in list', - details: - "ckpt_name: 'v2-1_768-ema-pruned.ckpt' not in ['anythingV5Anything_anythingV5PrtRE.safetensors', 'deliberate_v2.safetensors', 'dreamshaper_631BakedVae.safetensors', 'dreamshaper_631Inpainting.safetensors', 'edgeOfRealism_eorV20Fp16BakedVAE.safetensors', 'juggernaut_final-inpainting.safetensors', 'juggernaut_final.safetensors', 'loraChGirl.safetensors', 'sd-v1-5-inpainting.ckpt', 'sd_xl_base_1.0.safetensors', 'sd_xl_refiner_1.0.safetensors', 'v1-5-pruned-emaonly.ckpt']", - extra_info: { - input_name: 'ckpt_name', - input_config: [ - [ - 'anythingV5Anything_anythingV5PrtRE.safetensors', - 'deliberate_v2.safetensors', - 'dreamshaper_631BakedVae.safetensors', - 'dreamshaper_631Inpainting.safetensors', - 'edgeOfRealism_eorV20Fp16BakedVAE.safetensors', - 'juggernaut_final-inpainting.safetensors', - 'juggernaut_final.safetensors', - 'loraChGirl.safetensors', - 'sd-v1-5-inpainting.ckpt', - 'sd_xl_base_1.0.safetensors', - 'sd_xl_refiner_1.0.safetensors', - 'v1-5-pruned-emaonly.ckpt', - ], - ], - received_value: 'v2-1_768-ema-pruned.ckpt', - }, - }, - ], - dependent_outputs: ['9', '12'], - class_type: 'CheckpointLoaderSimple', - }, - }, -} - function logError(result: Result) { // Top-level error let has_error = false @@ -179,6 +139,7 @@ export async function generateRequest(prompt: any) { export async function generateImage(prompt: any) { try { let { history_result, prompt_id }: any = await generateRequest(prompt) + const outputs: any[] = Object.values(history_result[prompt_id].outputs) const images: any[] = [] for (const output of outputs) { @@ -188,6 +149,8 @@ export async function generateImage(prompt: any) { } const base64_imgs = [] + const formats: string[] = [] + for (const image of images) { const img = await loadImage( image.filename, @@ -195,14 +158,20 @@ export async function generateImage(prompt: any) { image.type ) base64_imgs.push(img) + formats.push(util.getFileFormat(image.filename)) } store.data.comfyui_output_images = base64_imgs const thumbnails = [] - for (const image of base64_imgs) { - thumbnails.push(await io.createThumbnail(image, 300)) + for (let i = 0; i < base64_imgs.length; ++i) { + if (['png', 'webp', 'jpg'].includes(formats[i])) { + thumbnails.push(await io.createThumbnail(base64_imgs[i], 300)) + } else if (['gif'].includes(formats[i])) { + thumbnails.push('data:image/gif;base64,' + base64_imgs[i]) + } } + store.data.comfyui_output_thumbnail_images = thumbnails return base64_imgs @@ -242,6 +211,7 @@ export async function loadImage( } export async function getConfig() { + //TODO: replace this method with get_object_info from comfyapi try { const prompt = { '1': { @@ -376,11 +346,31 @@ export const store = new AStore({ // workflows_paths: [] as string[], // workflows_names: [] as string[], workflows: {} as any, - selected_workflow: '', // the selected workflow from the workflow menu + selected_workflow_name: '', // the selected workflow from the workflow menu current_prompt: {} as any, // current prompt extracted from the workflow thumbnail_image_size: 100, load_image_nodes: {} as any, //our custom loadImageBase64 nodes, we need to substitute comfyui LoadImage nodes with before generating a prompt // load_image_base64_strings: {} as any, //images the user added to the plugin comfy ui + object_info: undefined as any, + current_prompt2: {} as any, + current_prompt2_output: {} as any, + output_thumbnail_image_size: {} as Record, + comfy_server: new diffusion_chain.ComfyServer( + 'http://127.0.0.1:8188' + ) as diffusion_chain.ComfyServer, + loaded_images_base64_url: [] as string[], + current_loaded_image: {} as Record, + loaded_images_list: [] as string[], // store an array of all images in the comfy's input directory + nodes_order: [] as string[], // nodes with smaller index will be rendered first, + can_edit_nodes: false as boolean, + nodes_label: {} as Record, + workflows2: { + hi_res_workflow: util.hi_res_workflow, + lora_less_workflow: util.lora_less_workflow, + img2img_workflow: util.img2img_workflow, + animatediff_workflow: util.animatediff_workflow, + } as Record, + progress_value: 0, }) export function storeToPrompt(store: any, basePrompt: any) { @@ -615,9 +605,9 @@ class ComfyNodeComponent extends React.Component<{}> { label_item="Select a workflow" selected_index={Object.values( store.data.workflows - ).indexOf(store.data.selected_workflow)} + ).indexOf(store.data.selected_workflow_name)} onChange={async (id: any, value: any) => { - store.data.selected_workflow = value.item + store.data.selected_workflow_name = value.item await loadWorkflow( store.data.workflows[value.item] ) @@ -745,17 +735,803 @@ class ComfyNodeComponent extends React.Component<{}> { } } +function setSliderValue(store: any, node_id: string, name: string, value: any) { + runInAction(() => { + store.data.current_prompt2[node_id].inputs[name] = value + }) +} +async function onChangeLoadImage(node_id: string, filename: string) { + try { + store.data.current_loaded_image[node_id] = + await util.base64UrlFromComfy(store.data.comfy_server, { + filename: encodeURIComponent(filename), + type: 'input', + subfolder: '', + }) + } catch (e) { + console.warn(e) + } +} +function renderNode(node_id: string, node: any) { + const comfy_node_info = toJS(store.data.object_info[node.class_type]) + const is_output = comfy_node_info.output_node + console.log('comfy_node_info: ', comfy_node_info) + const node_type = util.getNodeType(node.class_type) + let node_html + if (node_type === util.ComfyNodeType.LoadImage) { + const loaded_images = store.data.loaded_images_list + const inputs = store.data.current_prompt2[node_id].inputs + const node_name = node.class_type + node_html = ( +
+ New load image component + + {node_name} + +
+ + ) => { + console.log('onChange value.item: ', item) + inputs.image = item + //load image store for each LoadImage Node + //use node_id to store these + + onChangeLoadImage(node_id, item) + }} + > +
+
+ + + +
+ { + console.error( + 'error loading image: ', + store.data.current_loaded_image[node_id] + ) + // try { + // const filename = inputs.image + // store.data.current_loaded_image[node_id] = + // await util.base64UrlFromComfy( + // store.data.comfy_server, + // { + // filename: encodeURIComponent(filename), + // type: 'input', + // subfolder: '', + // } + // ) + // console.log( + // 'store.data.current_loaded_image[node_id]: ', + // toJS(store.data.current_loaded_image[node_id]) + // ) + // } catch (e) { + // console.warn(e) + // } + onChangeLoadImage(node_id, inputs.image) + }} + /> + {/* */} +
+ ) + } else if (node_type === util.ComfyNodeType.Normal) { + node_html = Object.entries(node.inputs).map(([name, value], index) => { + // store.data.current_prompt2[node_id].inputs[name] = value + try { + const input = comfy_node_info.input.required[name] + let { type, config } = util.parseComfyInput(input) + const html_element = renderInput( + node_id, + name, + type, + config, + `${node_id}_${name}_${type}_${index}` + ) + return html_element + } catch (e) { + console.error(e) + } + }) + } + if (is_output) { + const output_node_element = ( +
+ { + store.data.output_thumbnail_image_size[node_id] = + evt.target.value + }} + > + + Thumbnail Size: + + + {parseInt( + store.data.output_thumbnail_image_size[ + node_id + ] as any + )} + + + { + // io.IO.base64ToLayer( + // store.data.current_prompt2_output[node_id][ + // index + // ] + // ) + urlToCanvas( + store.data.current_prompt2_output[node_id][ + index + ], + 'comfy_output.png' + ) + }, + title: 'Copy Image to Canvas', + }, + ]} + > +
+ ) + + return output_node_element + } + return node_html +} +function renderInput( + node_id: string, + name: string, + type: any, + config: any, + key?: string +) { + let html_element = ( +
+ {name},{type}, {JSON.stringify(config)} +
+ ) + const inputs = store.data.current_prompt2[node_id].inputs + if (type === util.ComfyInputType.BigNumber) { + html_element = ( + <> + {name}: + { + // store.data.search_query = event.target.value + inputs[name] = event.target.value + console.log(`${name}: ${event.target.value}`) + }} + > + + ) + } else if (type === util.ComfyInputType.TextFieldNumber) { + html_element = ( + <> + {name}: + { + const v = e.target.value + let new_value = + v !== '' + ? Math.max(config.min, Math.min(config.max, v)) + : v + inputs[name] = new_value + + console.log(`${name}: ${e.target.value}`) + }} + > + + ) + } else if (type === util.ComfyInputType.Slider) { + html_element = ( + { + // inputs[name] = new_value + // setSliderValue(store, node_id, name, new_value) + store.data.current_prompt2[node_id].inputs[name] = new_value + console.log('slider_change: ', new_value) + }} + /> + ) + } else if (type === util.ComfyInputType.Menu) { + html_element = ( + <> + + {name} + + + ) => { + console.log('onChange value.item: ', item) + inputs[name] = item + }} + > + + ) + } else if (type === util.ComfyInputType.TextArea) { + html_element = ( + { + try { + // this.changePositivePrompt( + // event.target.value, + // store.data.current_index + // ) + // autoResize( + // event.target, + // store.data.positivePrompts[ + // store.data.current_index + // ] + // ) + inputs[name] = event.target.value + } catch (e) { + console.warn(e) + } + }} + placeholder={`${name}`} + value={inputs[name]} + > + ) + } else if (type === util.ComfyInputType.TextField) { + html_element = ( + <> + {name}: + + { + inputs[name] = e.target.value + console.log(`${name}: ${e.target.value}`) + }} + > + + ) + } + + return
{html_element}
+} + +export function swap(index1: number, index2: number) { + const { length } = store.data.nodes_order + if (index1 >= 0 && index1 < length && index2 >= 0 && index2 < length) { + ;[store.data.nodes_order[index1], store.data.nodes_order[index2]] = [ + store.data.nodes_order[index2], + store.data.nodes_order[index1], + ] + } +} + +export function saveWorkflowData( + workflow_name: string, + { prompt, nodes_order, nodes_label }: WorkflowData +) { + storage.localStorage.setItem( + workflow_name, + JSON.stringify({ prompt, nodes_order, nodes_label }) + ) +} +export function loadWorkflowData(workflow_name: string): WorkflowData { + const workflow_data: WorkflowData = JSON.parse( + storage.localStorage.getItem(workflow_name) + ) + return workflow_data +} +interface WorkflowData { + prompt: any + nodes_order: string[] + nodes_label: Record +} +function loadWorkflow2(workflow: any) { + const copyJson = (originalObject: any) => + JSON.parse(JSON.stringify(originalObject)) + //1) get prompt + store.data.current_prompt2 = copyJson(workflow) + + //2) get the original order + store.data.nodes_order = Object.keys(toJS(store.data.current_prompt2)) + + //3) get labels for each nodes + store.data.nodes_label = Object.fromEntries( + Object.entries(toJS(store.data.current_prompt2)).map( + ([node_id, node]: [string, any]) => { + return [ + node_id, + toJS(store.data.object_info[node.class_type]).display_name, + ] + } + ) + ) + + // parse the output nodes + // Note: we can't modify the node directly in the prompt like we do for input nodes. + //.. since this data doesn't exist on the prompt. so we create separate container for the output images + store.data.current_prompt2_output = Object.entries( + store.data.current_prompt2 + ).reduce( + ( + output_entries: Record, + [node_id, node]: [string, any] + ) => { + if (store.data.object_info[node.class_type].output_node) { + output_entries[node_id] = [] + } + return output_entries + }, + {} + ) + + //slider variables for output nodes + //TODO: delete store.data.output_thumbnail_image_size before loading a new workflow + for (let key in toJS(store.data.current_prompt2_output)) { + store.data.output_thumbnail_image_size[key] = 200 + } + + const workflow_name = store.data.selected_workflow_name + if (workflow_name) { + // check if the workflow has a name + + if (workflow_name in storage.localStorage) { + //load the workflow data from local storage + //1) load the last parameters used in generation + //2) load the order of the nodes + //3) load the labels of the nodes + + const workflow_data: WorkflowData = loadWorkflowData(workflow_name) + if ( + util.isSameStructure( + workflow_data.prompt, + toJS(store.data.current_prompt2) + ) + ) { + //load 1) + store.data.current_prompt2 = workflow_data.prompt + //load 2) + store.data.nodes_order = workflow_data.nodes_order + //load 3) + store.data.nodes_label = workflow_data.nodes_label + } else { + // do not load. instead override the localStorage with the new values + workflow_data.prompt = toJS(store.data.current_prompt2) + workflow_data.nodes_order = toJS(store.data.nodes_order) + workflow_data.nodes_label = toJS(store.data.nodes_label) + + saveWorkflowData(workflow_name, workflow_data) + } + } else { + // if workflow data is missing from local storage then save it for next time. + //1) save parameters values + //2) save nodes order + //3) save nodes label + + const prompt = toJS(store.data.current_prompt2) + const nodes_order = toJS(store.data.nodes_order) + const nodes_label = toJS(store.data.nodes_label) + saveWorkflowData(workflow_name, { + prompt, + nodes_order, + nodes_label, + }) + } + } +} +@observer +class ComfyWorkflowComponent extends React.Component<{}, { value?: number }> { + async componentDidMount(): Promise { + try { + store.data.object_info = await diffusion_chain.ComfyApi.objectInfo( + store.data.comfy_server + ) + + loadWorkflow2(util.lora_less_workflow) + + //convert all of comfyui loaded images into base64url that the plugin can use + const loaded_images = + store.data.object_info.LoadImage.input.required['image'][0] + const loaded_images_base64_url = await Promise.all( + loaded_images.map(async (filename: string) => { + try { + return await util.base64UrlFromComfy( + store.data.comfy_server, + { + filename: encodeURIComponent(filename), + type: 'input', + subfolder: '', + } + ) + } catch (e) { + console.warn(e) + } + }) + ) + store.data.loaded_images_list = + store.data.object_info.LoadImage.input.required['image'][0] + + store.data.loaded_images_base64_url = loaded_images_base64_url + } catch (e) { + console.error(e) + } + } + + render(): React.ReactNode { + const comfy_server = store.data.comfy_server + return ( +
+
+ {/* {util.getNodes(util.hi_res_workflow).map((node, index) => { + // return
{node.class_type}
+ return ( +
{this.renderNode(node)}
+ ) + })} */} + + +
+
+ +
+
+ { + store.data.selected_workflow_name = value.item + loadWorkflow2(store.data.workflows2[value.item]) + }} + >{' '} + +
+ + {store.data.object_info ? ( + <> +
+ {util + .getNodes(store.data.current_prompt2) + .sort( + ([node_id1, node1], [node_id2, node2]) => { + return ( + store.data.nodes_order.indexOf( + node_id1 + ) - + store.data.nodes_order.indexOf( + node_id2 + ) + ) + } + ) + + .map(([node_id, node], index) => { + return ( +
+
+
+ +
+
+ + {/* */} +
+
+ + "{node_id}":{' '} + + { + store.data.nodes_label[ + node_id + ] + } + {' '} + {' '} + + {node.class_type} + +
+ { + store.data.nodes_label[ + node_id + ] = event.target.value + }} + > +
+ {renderNode(node_id, node)} + {/* + */} +
+ ) + })} +
+ + ) : ( + void 0 + )} +
+ ) + } +} const container = document.getElementById('ComfyUIContainer')! const root = ReactDOM.createRoot(container) root.render( - - -
- - - -
- {/* */} -
-
+ // + +
+ + {/* */} + + + +
+ {/* */} +
+ //
) diff --git a/typescripts/comfyui/prompt.json b/typescripts/comfyui/hi_res_workflow.json similarity index 58% rename from typescripts/comfyui/prompt.json rename to typescripts/comfyui/hi_res_workflow.json index 25b43bb..e2075d0 100644 --- a/typescripts/comfyui/prompt.json +++ b/typescripts/comfyui/hi_res_workflow.json @@ -7,22 +7,10 @@ "sampler_name": "dpmpp_sde", "scheduler": "normal", "denoise": 1, - "model": [ - "16", - 0 - ], - "positive": [ - "6", - 0 - ], - "negative": [ - "7", - 0 - ], - "latent_image": [ - "5", - 0 - ] + "model": ["16", 0], + "positive": ["6", 0], + "negative": ["7", 0], + "latent_image": ["5", 0] }, "class_type": "KSampler" }, @@ -37,43 +25,28 @@ "6": { "inputs": { "text": "masterpiece HDR victorian portrait painting of woman, blonde hair, mountain nature, blue sky\n", - "clip": [ - "16", - 1 - ] + "clip": ["16", 1] }, "class_type": "CLIPTextEncode" }, "7": { "inputs": { "text": "bad hands, text, watermark\n", - "clip": [ - "16", - 1 - ] + "clip": ["16", 1] }, "class_type": "CLIPTextEncode" }, "8": { "inputs": { - "samples": [ - "3", - 0 - ], - "vae": [ - "16", - 2 - ] + "samples": ["3", 0], + "vae": ["16", 2] }, "class_type": "VAEDecode" }, "9": { "inputs": { "filename_prefix": "ComfyUI", - "images": [ - "8", - 0 - ] + "images": ["8", 0] }, "class_type": "SaveImage" }, @@ -83,10 +56,7 @@ "width": 1152, "height": 1152, "crop": "disabled", - "samples": [ - "3", - 0 - ] + "samples": ["3", 0] }, "class_type": "LatentUpscale" }, @@ -98,45 +68,24 @@ "sampler_name": "dpmpp_2m", "scheduler": "simple", "denoise": 0.5, - "model": [ - "16", - 0 - ], - "positive": [ - "6", - 0 - ], - "negative": [ - "7", - 0 - ], - "latent_image": [ - "10", - 0 - ] + "model": ["16", 0], + "positive": ["6", 0], + "negative": ["7", 0], + "latent_image": ["10", 0] }, "class_type": "KSampler" }, "12": { "inputs": { "filename_prefix": "ComfyUI", - "images": [ - "13", - 0 - ] + "images": ["13", 0] }, "class_type": "SaveImage" }, "13": { "inputs": { - "samples": [ - "11", - 0 - ], - "vae": [ - "16", - 2 - ] + "samples": ["11", 0], + "vae": ["16", 2] }, "class_type": "VAEDecode" }, @@ -146,4 +95,4 @@ }, "class_type": "CheckpointLoaderSimple" } -} \ No newline at end of file +} diff --git a/typescripts/comfyui/img2img_workflow.json b/typescripts/comfyui/img2img_workflow.json new file mode 100644 index 0000000..fd64421 --- /dev/null +++ b/typescripts/comfyui/img2img_workflow.json @@ -0,0 +1,65 @@ +{ + "3": { + "inputs": { + "seed": 280823642470253, + "steps": 20, + "cfg": 8, + "sampler_name": "dpmpp_2m", + "scheduler": "normal", + "denoise": 0.8700000000000001, + "model": ["14", 0], + "positive": ["6", 0], + "negative": ["7", 0], + "latent_image": ["12", 0] + }, + "class_type": "KSampler" + }, + "6": { + "inputs": { + "text": "photograph of victorian woman with wings, sky clouds, meadow grass\n", + "clip": ["14", 1] + }, + "class_type": "CLIPTextEncode" + }, + "7": { + "inputs": { + "text": "watermark, text\n", + "clip": ["14", 1] + }, + "class_type": "CLIPTextEncode" + }, + "8": { + "inputs": { + "samples": ["3", 0], + "vae": ["14", 2] + }, + "class_type": "VAEDecode" + }, + "9": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": ["8", 0] + }, + "class_type": "SaveImage" + }, + "10": { + "inputs": { + "image": "example.png", + "choose file to upload": "image" + }, + "class_type": "LoadImage" + }, + "12": { + "inputs": { + "pixels": ["10", 0], + "vae": ["14", 2] + }, + "class_type": "VAEEncode" + }, + "14": { + "inputs": { + "ckpt_name": "v1-5-pruned-emaonly.ckpt" + }, + "class_type": "CheckpointLoaderSimple" + } +} diff --git a/typescripts/comfyui/lora_less_workflow.json b/typescripts/comfyui/lora_less_workflow.json new file mode 100644 index 0000000..cbf12ae --- /dev/null +++ b/typescripts/comfyui/lora_less_workflow.json @@ -0,0 +1,187 @@ +{ + "8": { + "inputs": { + "vae_name": "klF8Anime2VAE_klF8Anime2VAE.ckpt" + }, + "class_type": "VAELoader" + }, + "16": { + "inputs": { + "ckpt_name": "juggernaut_final.safetensors" + }, + "class_type": "CheckpointLoaderSimple" + }, + "18": { + "inputs": { + "text": "(front view:1.2)", + "clip": ["16", 1] + }, + "class_type": "CLIPTextEncode" + }, + "21": { + "inputs": { + "width": 512, + "height": 512, + "batch_size": 2 + }, + "class_type": "EmptyLatentImage" + }, + "30": { + "inputs": { + "text": "(a dog sitting:1.3) on a tile floor with a blue eyes and a white nose and tail, looking at the camera, artist, extremely detailed oil painting, a photorealistic painting, photorealism", + "clip": ["16", 1] + }, + "class_type": "CLIPTextEncode" + }, + "31": { + "inputs": { + "tile_size": 512, + "samples": ["97", 0], + "vae": ["8", 0] + }, + "class_type": "VAEDecodeTiled" + }, + "48": { + "inputs": { + "weight": ["163", 0], + "model_name": "ip-adapter-plus_sd15.bin", + "dtype": "fp32", + "model": ["162", 0], + "image": ["168", 0], + "clip_vision": ["57", 0] + }, + "class_type": "IPAdapter" + }, + "50": { + "inputs": { + "strength": ["163", 0], + "noise_augmentation": 0, + "conditioning": ["30", 0], + "clip_vision_output": ["48", 1] + }, + "class_type": "unCLIPConditioning" + }, + "57": { + "inputs": { + "clip_name": "model.safetensors" + }, + "class_type": "CLIPVisionLoader" + }, + "97": { + "inputs": { + "seed": 229741993160779, + "steps": 32, + "cfg": 6.5, + "sampler_name": "dpmpp_2s_ancestral", + "scheduler": "karras", + "denoise": 1, + "model": ["48", 0], + "positive": ["50", 0], + "negative": ["18", 0], + "latent_image": ["21", 0] + }, + "class_type": "KSampler" + }, + "122": { + "inputs": { + "images": ["31", 0] + }, + "class_type": "PreviewImage" + }, + "155": { + "inputs": { + "seed": 216203953003378, + "steps": 40, + "cfg": 5, + "sampler_name": "ddim", + "scheduler": "normal", + "denoise": 0.45, + "model": ["48", 0], + "positive": ["50", 0], + "negative": ["18", 0], + "latent_image": ["156", 0] + }, + "class_type": "KSampler" + }, + "156": { + "inputs": { + "tile_size": 640, + "pixels": ["159", 0], + "vae": ["8", 0] + }, + "class_type": "VAEEncodeTiled" + }, + "157": { + "inputs": { + "upscale_model": ["158", 0], + "image": ["31", 0] + }, + "class_type": "ImageUpscaleWithModel" + }, + "158": { + "inputs": { + "model_name": "RealESRGAN_x4plus_anime_6B.pth" + }, + "class_type": "UpscaleModelLoader" + }, + "159": { + "inputs": { + "upscale_method": "nearest-exact", + "scale_by": 0.45, + "image": ["157", 0] + }, + "class_type": "ImageScaleBy" + }, + "160": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": ["161", 0] + }, + "class_type": "SaveImage" + }, + "161": { + "inputs": { + "tile_size": 512, + "samples": ["155", 0], + "vae": ["8", 0] + }, + "class_type": "VAEDecodeTiled" + }, + "162": { + "inputs": { + "b1": 1.1500000000000001, + "b2": 1.35, + "s1": 0.9500000000000001, + "s2": 0.18, + "model": ["16", 0] + }, + "class_type": "FreeU" + }, + "163": { + "inputs": { + "Value": 0.5 + }, + "class_type": "Float" + }, + "168": { + "inputs": { + "image": "Layer 4 (3).png", + "choose file to upload": "image" + }, + "class_type": "LoadImage" + }, + "169": { + "inputs": { + "image": "Layer 3 (1).png", + "choose file to upload": "image" + }, + "class_type": "LoadImage" + }, + "188": { + "inputs": { + "image": "01285-3246154361-a photo of charddim15 person looking happy, beautiful, ((cloth)), ((full body)), ((chest)), ((far away)), ((waist up)).png", + "choose file to upload": "image" + }, + "class_type": "LoadImage" + } +} diff --git a/typescripts/comfyui/util.ts b/typescripts/comfyui/util.ts new file mode 100644 index 0000000..d197086 --- /dev/null +++ b/typescripts/comfyui/util.ts @@ -0,0 +1,356 @@ +import hi_res_workflow from './hi_res_workflow.json' +import img2img_workflow from './img2img_workflow.json' +import animatediff_workflow from './animatediff_workflow.json' +import lora_less_workflow from './lora_less_workflow.json' +import { diffusion_chain } from '../entry' +import { ComfyPrompt } from 'diffusion-chain/dist/backends/comfyui-api.mjs' +// import { ComfyPrompt } from 'diffusion-chain/dist/backends/comfyui-api.mjs' +// import { ComfyPrompt } from 'diffusion-chain/' +export function getWorkflow() {} + +interface Workflow {} +export function getNodes(workflow: Workflow) { + // Object.values(workflow).forEach((node) => { + // console.log(node.class_type) + // }) + return Object.entries(workflow) +} + +export enum ComfyInputType { + TextField = 'TextField', + TextArea = 'TextArea', + Menu = 'Menu', + Number = 'Number', + Slider = 'Slider', + BigNumber = 'BigNumber', + TextFieldNumber = 'TextFieldNumber', + Skip = 'Skip', +} +export enum ComfyNodeType { + LoadImage = 'LoadImage', + Normal = 'Normal', + Skip = 'Skip', +} + +interface ComfyOutputImage { + filename: string + subfolder: string + type: string +} + +export function getNodeType(node_name: any) { + let node_type: ComfyNodeType = ComfyNodeType.Normal + switch (node_name) { + case 'LoadImage': + node_type = ComfyNodeType.LoadImage + break + default: + break + } + return node_type +} +export function parseComfyInput(input_info: any): { + type: ComfyInputType + config: any +} { + const value = input_info[0] + + let input_type: ComfyInputType = ComfyInputType.Skip + let input_config + + if (typeof value === 'string') { + if (value === 'FLOAT') { + input_type = ComfyInputType.Slider + input_config = input_info[1] + } else if (value === 'INT') { + if (input_info[1].max > Number.MAX_SAFE_INTEGER) { + input_type = ComfyInputType.BigNumber + input_config = input_info[1] + } else { + input_type = ComfyInputType.TextFieldNumber + input_config = input_info[1] + } + } else if (value === 'STRING') { + if (input_info[1]?.multiline) { + input_type = ComfyInputType.TextArea + input_config = input_info[1] + } else { + input_type = ComfyInputType.TextField + input_config = input_info[1] + } + } + } else if (Array.isArray(value)) { + input_type = ComfyInputType.Menu + input_config = value + } + + return { type: input_type, config: input_config } +} + +export function makeHtmlInput() {} + +export function nodeToUIConfig( + node: { inputs: { [key: string]: any }; class_type: string }, + object_info: any +) { + let comfy_node_info = object_info[node.class_type] + let node_ui_config = Object.entries(node.inputs).map( + ([name, value]: [string, any]) => { + const first_value = comfy_node_info[name][0] + let { type, config } = parseComfyInput(first_value) + + return + } + ) + // comfy_node_info.input.required[] +} + +async function getHistory(comfy_server: diffusion_chain.ComfyServer) { + while (true) { + const res = await diffusion_chain.ComfyApi.queue(comfy_server) + if (res.queue_pending.length || res.queue_running.length) { + await new Promise((resolve) => setTimeout(resolve, 500)) + } else { + break + } + await new Promise((resolve) => setTimeout(resolve, 500)) + } + const history = await diffusion_chain.ComfyApi.history(comfy_server) + return history +} +export async function postPromptAndGetBase64JsonResult( + comfy_server: diffusion_chain.ComfyServer, + prompt: Record +) { + try { + const res = await diffusion_chain.ComfyApi.prompt(comfy_server, { + prompt, + } as ComfyPrompt) + if (res.error) { + const readable_error = comfy_server.getReadableError(res) + throw new Error(readable_error) + } + const prompt_id = res.prompt_id + const history = await getHistory(comfy_server) + const promptInfo = history[prompt_id] + const store_output = await mapComfyOutputToStoreOutput( + comfy_server, + promptInfo.outputs + ) + // // [4][0] for output id. + // const fileName = promptInfo.outputs[promptInfo.prompt[4][0]].images[0].filename + // const resultB64 = await ComfyApi.view(this, fileName); + // resultImages.push(resultB64) + // if (option.imageFinishCallback) { + // try { option.imageFinishCallback(resultB64, index) } catch (e) { } + // } + // } + return store_output + } catch (e) { + console.error(e) + } +} +export const getFileFormat = (fileName: string): string => + fileName.includes('.') ? fileName.split('.').pop()! : '' + +export async function base64UrlFromComfy( + comfy_server: diffusion_chain.ComfyServer, + { filename, type, subfolder }: ComfyOutputImage +) { + const base64 = await diffusion_chain.ComfyApi.view( + comfy_server, + filename, + type, + subfolder + ) + return base64Url(base64, getFileFormat(filename)) +} +export function base64UrlFromFileName(base64: string, filename: string) { + return base64Url(base64, getFileFormat(filename)) +} +export function base64Url(base64: string, format: string = 'png') { + return `data:image/${format};base64,${base64}` +} +export function generatePrompt(prompt: Record) { + prompt +} +export function updateOutput(output: any, output_store_obj: any) { + // store.data.current_prompt2_output[26] = [image, image] + output_store_obj = output +} + +export async function mapComfyOutputToStoreOutput( + comfy_server: diffusion_chain.ComfyServer, + comfy_output: Record +) { + // const comfy_output: Record = { + // '12': { + // images: [ + // { + // filename: 'AA_readme_00506_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00507_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00508_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00509_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00510_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00511_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00512_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00513_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00514_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00515_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00516_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00517_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00518_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00519_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00520_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00521_.png', + // subfolder: '', + // type: 'output', + // }, + // ], + // }, + // '26': { + // images: [ + // { + // filename: 'AA_readme_gif_00079_.gif', + // subfolder: '', + // type: 'output', + // }, + // ], + // }, + // } + + const store_output: Record = {} + for (let key in comfy_output) { + if (comfy_output[key].hasOwnProperty('images')) { + let base64_url_list = await Promise.all( + comfy_output[key].images.map( + async (image: ComfyOutputImage) => + await base64UrlFromComfy(comfy_server, image) + ) + ) + store_output[key] = base64_url_list + } + } + return store_output +} + +interface LooseObject { + [key: string]: any +} + +function isSameStructure(obj1: LooseObject, obj2: LooseObject): boolean { + // Get keys + const keys1 = Object.keys(obj1) + const keys2 = Object.keys(obj2) + + // Check if both objects have the same number of keys + if (keys1.length !== keys2.length) { + return false + } + + // Check if all keys in obj1 exist in obj2 and have the same structure + for (let i = 0; i < keys1.length; i++) { + const key = keys1[i] + + // Check if the key exists in obj2 + if (!obj2.hasOwnProperty(key)) { + return false + } + + // If the value of this key is an object, check their structure recursively + if ( + typeof obj1[key] === 'object' && + obj1[key] !== null && + typeof obj2[key] === 'object' && + obj2[key] !== null + ) { + if (!isSameStructure(obj1[key], obj2[key])) { + return false + } + } + } + + // If all checks passed, the structures are the same + return true +} + +export default { + getNodes, + parseComfyInput, + getNodeType, + base64Url, + getFileFormat, + base64UrlFromComfy, + generatePrompt, + updateOutput, + getHistory, + mapComfyOutputToStoreOutput, + postPromptAndGetBase64JsonResult, + isSameStructure, + hi_res_workflow, + img2img_workflow, + animatediff_workflow, + lora_less_workflow, + ComfyInputType, + ComfyNodeType, +} diff --git a/typescripts/entry.ts b/typescripts/entry.ts index e9b152b..b25e760 100644 --- a/typescripts/entry.ts +++ b/typescripts/entry.ts @@ -40,3 +40,6 @@ export * as api_ts from './util/ts/api' export * as comfyui from './comfyui/comfyui' export { toJS } from 'mobx' export { default as node_fs } from 'fs' +export { default as comfyui_util } from './comfyui/util' + +export * as diffusion_chain from 'diffusion-chain'