cleanup todo

Signed-off-by: vladmandic <mandic00@live.com>

modules/apg/pipeline_stable_diffision_xl_apg.py

@@ -1081,7 +1081,7 @@ class StableDiffusionXLPipelineAPG(
             latents,
         )
 
-        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 6. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 7. Prepare added time ids & embeddings

modules/apg/pipeline_stable_diffusion_apg.py

@@ -965,7 +965,7 @@ class StableDiffusionPipelineAPG(
             latents,
         )
 
-        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 6. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 6.1 Add image embeds for IP-Adapter

modules/control/proc/dwpose/config/yolox_l_8xb8-300e_coco.py

@@ -194,7 +194,6 @@ param_scheduler = [
     dict(
         # use quadratic formula to warm up 5 epochs
         # and lr is updated by iteration
-        # TODO: fix default scope in get function
         type='mmdet.QuadraticWarmupLR',
         by_epoch=True,
         begin=0,

modules/control/proc/normalbae/nets/submodules/efficientnet_repo/geffnet/activations/activations.py

@@ -12,8 +12,6 @@ from torch.nn import functional as F
 def swish(x, inplace: bool = False):
     """Swish - Described originally as SiLU (https://arxiv.org/abs/1702.03118v3)
     and also as Swish (https://arxiv.org/abs/1710.05941).
-
-    TODO Rename to SiLU with addition to PyTorch
     """
     return x.mul_(x.sigmoid()) if inplace else x.mul(x.sigmoid())
 

modules/control/proc/normalbae/nets/submodules/efficientnet_repo/geffnet/activations/activations_jit.py

@@ -22,8 +22,6 @@ __all__ = ['swish_jit', 'SwishJit', 'mish_jit', 'MishJit',
 def swish_jit(x, inplace: bool = False):
     """Swish - Described originally as SiLU (https://arxiv.org/abs/1702.03118v3)
     and also as Swish (https://arxiv.org/abs/1710.05941).
-
-    TODO Rename to SiLU with addition to PyTorch
     """
     return x.mul(x.sigmoid())
 

modules/control/proc/normalbae/nets/submodules/efficientnet_repo/geffnet/activations/activations_me.py

@@ -36,8 +36,6 @@ class SwishJitAutoFn(torch.autograd.Function):
 
     Swish - Described originally as SiLU (https://arxiv.org/abs/1702.03118v3)
     and also as Swish (https://arxiv.org/abs/1710.05941).
-
-    TODO Rename to SiLU with addition to PyTorch
     """
 
     @staticmethod

modules/control/proc/normalbae/nets/submodules/efficientnet_repo/geffnet/efficientnet_builder.py

@@ -483,7 +483,7 @@ def _decode_block_str(block_str):
     Returns:
         A list of block args (dicts)
     Raises:
-        ValueError: if the string def not properly specified (TODO)
+        ValueError: if the string def not properly specified
     """
     assert isinstance(block_str, str)
     ops = block_str.split('_')

modules/control/proc/zoe/zoedepth/models/layers/localbins_layers.py

@@ -155,7 +155,7 @@ class LinearSplitter(nn.Module):
 
         b_prev = b_prev / b_prev.sum(dim=1, keepdim=True)  # renormalize for gurantees
         # print(b_prev.shape, S_normed.shape)
-        # if is_for_query:(1).expand(-1, b_prev.size(0)//n, -1, -1, -1, -1).flatten(0,1)  # TODO ? can replace all this with a single torch.repeat?
+        # if is_for_query:(1).expand(-1, b_prev.size(0)//n, -1, -1, -1, -1).flatten(0,1)
         b = b_prev.unsqueeze(2) * S_normed
         b = b.flatten(1,2)  # .shape n, prev_nbins * split_factor, h, w
 

modules/control/proc/zoe/zoedepth/utils/config.py

@@ -395,7 +395,7 @@ def get_config(model_name, mode='train', dataset=None, **overwrite_kwargs):
     overwrite_kwargs = split_combined_args(overwrite_kwargs)
     config = {**config, **overwrite_kwargs}
 
-    # Casting to bool   # TODO: Not necessary. Remove and test
+    # Casting to bool
     for key in KEYS_TYPE_BOOL:
         if key in config:
             config[key] = bool(config[key])

modules/face/instantid_model.py

@@ -882,7 +882,7 @@ class StableDiffusionXLInstantIDPipeline(StableDiffusionXLControlNetPipeline):
                 guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
             ).to(device=device, dtype=latents.dtype)
 
-        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 7. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 7.1 Create tensor stating which controlnets to keep

modules/face/photomaker_pipeline.py

@@ -679,7 +679,7 @@ class PhotoMakerStableDiffusionXLPipeline(StableDiffusionXLPipeline):
             latents,
         )
 
-        # 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 9. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 10. Prepare added time ids & embeddings

modules/facelib/detection/retinaface/retinaface_utils.py

@@ -181,7 +181,6 @@ def match(threshold, truths, priors, variances, labels, landms, loc_t, conf_t, l
     best_prior_idx_filter.squeeze_(1)
     best_prior_overlap.squeeze_(1)
     best_truth_overlap.index_fill_(0, best_prior_idx_filter, 2)  # ensure best prior
-    # TODO refactor: index  best_prior_idx with long tensor
     # ensure every gt matches with its prior of max overlap
     for j in range(best_prior_idx.size(0)):  # 判别此anchor是预测哪一个boxes
         best_truth_idx[best_prior_idx[j]] = j

modules/facelib/utils/face_utils.py

@@ -99,7 +99,7 @@ def align_crop_face_landmarks(img,
     #  - np.flipud(eye_to_mouth) * [-1, 1]: rotate 90 clockwise
     # norm with the hypotenuse: get the direction
     x /= np.hypot(*x)  # get the hypotenuse of a right triangle
-    rect_scale = 1  # TODO: you can edit it to get larger rect
+    rect_scale = 1
     x *= max(np.hypot(*eye_to_eye) * 2.0 * rect_scale, np.hypot(*eye_to_mouth) * 1.8 * rect_scale)
     # y: half height of the oriented crop rectangle
     y = np.flipud(x) * [-1, 1]
@@ -116,7 +116,6 @@ def align_crop_face_landmarks(img,
 
     quad_ori = np.copy(quad)
     # Shrink, for large face
-    # TODO: do we really need shrink
     shrink = int(np.floor(qsize / output_size * 0.5))
     if shrink > 1:
         h, w = img.shape[0:2]

modules/flash_attn_triton_amd/fwd_prefill.py

@@ -52,7 +52,6 @@ def _attn_fwd_inner(acc, l_i, m_i, q, k_ptrs, v_ptrs, bias_ptrs, stride_kn, stri
         qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=ACCUMULATOR_TYPE)
         # We start from end of seqlen_k so only the first iteration would need
         # to be checked for padding if it is not a multiple of block_n
-        # TODO: This can be optimized to only be true for the padded block.
         if MASK_STEPS:
             # If this is the last block / iteration, we want to
             # mask if the sequence length is not a multiple of block size
@@ -105,7 +104,7 @@ def _attn_fwd_inner(acc, l_i, m_i, q, k_ptrs, v_ptrs, bias_ptrs, stride_kn, stri
         # CAVEAT: Must update l_ij before applying dropout
         l_ij = tl.sum(p, 1)
         if ENABLE_DROPOUT:
-            rng_output = tl.rand(philox_seed, philox_ptrs)  # TODO: use tl.randint for better performance
+            rng_output = tl.rand(philox_seed, philox_ptrs)
             dropout_mask = rng_output > dropout_p
 
             # return scores with negative values for dropped vals
@@ -304,7 +303,6 @@ def attn_fwd(Q, K, V, bias, Cache_seqlens, Cache_batch_idx, # pylint: disable=un
             # softmax_lse = tl.where(lse_mask, 0.0, softmax_lse)
             l_ptrs_mask = offs_m < MAX_SEQLENS_Q
             tl.store(l_ptrs, l, mask=l_ptrs_mask)
-            # TODO: Should dropout and return encoded softmax be handled here too?
             return
 
     # If MQA / GQA, set the K and V head offsets appropriately.

modules/flash_attn_triton_amd/utils.py

@@ -116,7 +116,6 @@ class MetaData():
             assert self.cu_seqlens_q is not None
             assert self.cu_seqlens_k is not None
             assert len(self.cu_seqlens_q) == len(self.cu_seqlens_k)
-            # TODO: Remove once bias is supported with varlen
             assert self.bias is None
             # assert not self.return_scores
         else:
@@ -125,7 +124,6 @@ class MetaData():
             assert self.cu_seqlens_q is None and self.cu_seqlens_k is None
         assert k.shape == v.shape
         assert q.shape[-1] == k.shape[-1] and q.shape[-1] == v.shape[-1]
-        # TODO: Change assert if we support qkl f8 and v f16
         assert q.dtype == k.dtype and q.dtype == v.dtype
         assert o.shape == q.shape
         assert (nheads_q % nheads_k) == 0
@@ -243,7 +241,6 @@ def input_helper(
         equal_seqlens=False
 
         # gen tensors
-        # TODO: the gen functions should maybe have different gen modes like random, ones, increasing seqlen
         q, cu_seqlens_q, max_seqlen_q = generate_varlen_tensor(TOTAL_SEQLENS_Q, HQ, D_HEAD, batch_size=BATCH, dtype=dtype, device=device, equal_seqlens=equal_seqlens, DEBUG_INPUT=DEBUG_INPUT)
         k, cu_seqlens_k, max_seqlen_k = generate_varlen_tensor(TOTAL_SEQLENS_K, HK, D_HEAD, batch_size=BATCH, dtype=dtype, device=device, equal_seqlens=equal_seqlens, DEBUG_INPUT=DEBUG_INPUT)
         v, _, _ = generate_varlen_tensor(TOTAL_SEQLENS_K, HK, D_HEAD, batch_size=BATCH, dtype=dtype, device=device, equal_seqlens=equal_seqlens, DEBUG_INPUT=DEBUG_INPUT)

modules/hidiffusion/hidiffusion_controlnet.py

@@ -550,7 +550,7 @@ def make_diffusers_sdxl_contrtolnet_ppl(block_class):
                 # # scale the initial noise by the standard deviation required by the scheduler
                 # latents = latents * self.scheduler.init_noise_sigma
 
-            # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+            # 7. Prepare extra step kwargs. 
             extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
             # 7.1 Create tensor stating which controlnets to keep

modules/history.py

@@ -1,8 +1,5 @@
 """
-TODO:
-- apply metadata
-- preview
-- load/save
+TODO: apply metadata, preview, load/save
 """
 
 import sys

modules/masking.py

@@ -118,7 +118,7 @@ def fill(image, mask):
 
 """
 [docs](https://huggingface.co/docs/transformers/v4.36.1/en/model_doc/sam#overview)
-TODO:
+TODO: additional masking algorithms
 - PerSAM
 - REMBG
 - https://huggingface.co/docs/transformers/tasks/semantic_segmentation

modules/onnx_impl/pipelines/onnx_stable_diffusion_upscale_pipeline.py

@@ -128,7 +128,7 @@ class OnnxStableDiffusionUpscalePipeline(diffusers.OnnxStableDiffusionUpscalePip
                 " `pipeline.unet` or your `image` input."
             )
 
-        # 8. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 8. Prepare extra step kwargs. 
         accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
         extra_step_kwargs = {}
         if accepts_eta:

modules/pag/pipe_sd.py

@@ -1268,7 +1268,7 @@ class StableDiffusionPAGPipeline(
             latents,
         )
 
-        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 6. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 6.1 Add image embeds for IP-Adapter

modules/pag/pipe_sdxl.py

@@ -1366,7 +1366,7 @@ class StableDiffusionXLPAGPipeline(
             latents,
         )
 
-        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 6. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 7. Prepare added time ids & embeddings

modules/ras/ras_attention.py

@@ -126,7 +126,7 @@ class RASLuminaAttnProcessor2_0:
             else:
                 softmax_scale = attn.scale
 
-        # perform Grouped-qurey Attention (GQA)   # TODO replace with GQA
+        # perform Grouped-qurey Attention (GQA)
         n_rep = attn.heads // kv_heads
         if n_rep >= 1:
             key = key.unsqueeze(3).repeat(1, 1, 1, n_rep, 1).flatten(2, 3)

modules/schedulers/scheduler_flashflow.py

@@ -349,7 +349,6 @@ class FlashFlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin):
         """Constructs the noise schedule of Karras et al. (2022)."""
 
         # Hack to make sure that other schedulers which copy this function don't break
-        # TODO: Add this logic to the other schedulers
         if hasattr(self.config, "sigma_min"):
             sigma_min = self.config.sigma_min
         else:
@@ -375,7 +374,6 @@ class FlashFlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin):
         """Constructs an exponential noise schedule."""
 
         # Hack to make sure that other schedulers which copy this function don't break
-        # TODO: Add this logic to the other schedulers
         if hasattr(self.config, "sigma_min"):
             sigma_min = self.config.sigma_min
         else:
@@ -399,7 +397,6 @@ class FlashFlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin):
         """From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)"""
 
         # Hack to make sure that other schedulers which copy this function don't break
-        # TODO: Add this logic to the other schedulers
         if hasattr(self.config, "sigma_min"):
             sigma_min = self.config.sigma_min
         else:

modules/taesd/taehv.py

@@ -81,8 +81,6 @@ def apply_model_with_memblocks(model, x, parallel, show_progress_bar):
         T = NT // N
         x = x.view(N, T, C, H, W)
     else:
-        # TODO(oboerbohan): at least on macos this still gradually uses more memory during decode...
-        # need to fix :(
         out = []
         # iterate over input timesteps and also iterate over blocks.
         # because of the cursed TPool/TGrow blocks, this is not a nested loop,

modules/taesd/taem1.py

@@ -77,8 +77,6 @@ def apply_model_with_memblocks(model, x, parallel, show_progress_bar):
         T = NT // N
         x = x.view(N, T, C, H, W)
     else:
-        # TODO(oboerbohan): at least on macos this still gradually uses more memory during decode...
-        # need to fix :(
         out = []
         # iterate over input timesteps and also iterate over blocks.
         # because of the cursed TPool/TGrow blocks, this is not a nested loop,

package.json

@@ -33,7 +33,8 @@
     "lint": "npm run format && npm run eslint && npm run eslint-ui && npm run ruff && npm run pylint | grep -v TODO",
     "lint-win": "npm run format-win && npm run eslint-win && npm run eslint-ui-win && npm run ruff-win && npm run pylint-win",
     "test": ". venv/bin/activate; python launch.py --debug --test",
-    "todo": "npm run pylint | grep W0511 | awk -F'TODO ' '{print \"- \"$NF}' | sed 's/ (fixme)//g' | sort"
+    "todo": "grep -oIPR 'TODO.*' *.py modules/ pipelines/ | sort -u",
+    "debug": "grep -ohIPR 'SD_.*?_DEBUG' *.py modules/ pipelines/ | sort -u"
   },
   "devDependencies": {
     "@eslint/compat": "^2.0.0",

pipelines/bria/bria_pipeline.py

@@ -94,7 +94,6 @@ class BriaPipeline(FluxPipeline):
             scheduler=scheduler,
         )
 
-        # TODO - why different than offical flux (-1)
         self.vae_scale_factor = (
             2 ** (len(self.vae.config.block_out_channels)) if hasattr(self, "vae") and self.vae is not None else 16
         )

pipelines/hdm/hdm/data/base.py

@@ -12,7 +12,6 @@ class BaseDataset(Data.Dataset):
         samples = torch.stack([x["sample"] for x in batch])
         caption = [x["caption"] for x in batch]
         tokenizer_outs = [x["tokenizer_out"] for x in batch]
-        # TODO: change to stack and reduce dim?
         add_time_ids = [x["add_time_ids"] for x in batch]
         tokenizer_outputs = []
         for tokenizer_out in zip(*tokenizer_outs):

pipelines/hdm/hdm/loader.py

@@ -119,6 +119,5 @@ def load_all(conf: dict):
     trainer = load_trainer(
         conf.pop("trainer"), unet=unet, te=te, vae=vae, scheduler=scheduler
     )
-    # TODO: there might be a better way to handle this
     dataset.tokenizers = tokenizers
     return dataset, trainer, (unet, te, tokenizers, vae, scheduler)

pipelines/hdm/hdm/modules/unet_patch.py

@@ -141,7 +141,6 @@ class RoPEAttnProcessor2_0(AttnProcessor2_0):
             key = key.transpose(1, 2)
 
         # the output of sdp = (batch, num_heads, seq_len, head_dim)
-        # TODO: add support for attn.scale when we move to Torch 2.1
         hidden_states = F.scaled_dot_product_attention(
             query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
         )

scripts/consistory/consistory_pipeline.py

@@ -322,7 +322,7 @@ class ConsistoryExtendAttnSDXLPipeline(
             latents,
         )
 
-        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 6. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         if share_queries:

scripts/differential_diffusion.py

@@ -1756,7 +1756,7 @@ class StableDiffusionDiffImg2ImgPipeline(DiffusionPipeline):
         timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
 
 
-        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 7. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
         map = torchvision.transforms.Resize(tuple(s // self.vae_scale_factor for s in image.shape[2:]),antialias=None)(map)
 

scripts/freescale/freescale_pipeline.py

@@ -873,7 +873,7 @@ class StableDiffusionXLFreeScale(DiffusionPipeline, FromSingleFileMixin, LoraLoa
             latents,
         )
 
-        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 6. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 7. Prepare added time ids & embeddings

scripts/freescale/freescale_pipeline_img2img.py

@@ -902,7 +902,7 @@ class StableDiffusionXLFreeScaleImg2Img(DiffusionPipeline, FromSingleFileMixin,
             latents,
         )
 
-        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 6. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 7. Prepare added time ids & embeddings

scripts/instantir/sdxl_instantir.py

@@ -1405,7 +1405,7 @@ class InstantIRPipeline(
                 guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
             ).to(device=device, dtype=latents.dtype)
 
-        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 7. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 7.1 Create tensor stating which controlnets to keep

scripts/mod/__init__.py

@@ -1039,7 +1039,7 @@ class StableDiffusionXLTilingPipeline(
         if isinstance(self.scheduler, LMSDiscreteScheduler):
             latents = latents * self.scheduler.sigmas[0]
 
-        # 5. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 5. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 6. Prepare added time ids & embeddings

scripts/pixelsmith/pixelsmith_pipeline.py

@@ -1384,7 +1384,7 @@ class PixelSmithXLPipeline(
             latents,
         )
 
-        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 6. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 7. Prepare added time ids & embeddings

scripts/xadapter/pipeline_sd_xl_adapter.py

@@ -833,7 +833,7 @@ class StableDiffusionXLAdapterPipeline(DiffusionPipeline, FromSingleFileMixin, L
             latents_sd1_5,
         )
 
-        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 6. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 7. Prepare added time ids & embeddings

scripts/xadapter/pipeline_sd_xl_adapter_controlnet.py

@@ -934,7 +934,7 @@ class StableDiffusionXLAdapterControlnetPipeline(DiffusionPipeline, FromSingleFi
             latents_sd1_5,
         )
 
-        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 6. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 7. Prepare added time ids & embeddings

scripts/xadapter/pipeline_sd_xl_adapter_controlnet_img2img.py

@@ -941,7 +941,7 @@ class StableDiffusionXLAdapterControlnetI2IPipeline(DiffusionPipeline, FromSingl
             generator,
         )
 
-        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        # 6. Prepare extra step kwargs. 
         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
 
         # 7. Prepare added time ids & embeddings