regenerate locales and rerun all translations

Signed-off-by: Vladimir Mandic <mandic00@live.com>

javascript/setHints.js

@@ -63,18 +63,18 @@ async function tooltipHide(e) {
 async function validateHints(json, elements) {
   json.missing = [];
   const data = Object.values(json).flat().filter((e) => e.hint.length > 0);
-  for (const e of data) e.label = e.label.toLowerCase().trim();
-  let original = elements.map((e) => e.textContent.toLowerCase().trim()).sort();
-  let duplicateUI = original.filter((e, i, a) => a.indexOf(e) !== i).sort();
+  for (const e of data) e.label = e.label.trim();
+  let original = elements.map((e) => e.textContent.toLowerCase().trim()).sort(); // should be case sensitive
+  let duplicateUI = original.filter((e, i, a) => a.indexOf(e.toLowerCase()) !== i).sort();
   original = [...new Set(original)]; // remove duplicates
   duplicateUI = [...new Set(duplicateUI)]; // remove duplicates
-  const current = data.map((e) => e.label.toLowerCase().trim()).sort();
+  const current = data.map((e) => e.label.toLowerCase().trim()).sort(); // should be case sensitive
   log('all elements:', original);
   log('all hints:', current);
   log('hints-differences', { elements: original.length, hints: current.length });
-  const missingHints = original.filter((e) => !current.includes(e)).sort();
-  const orphanedHints = current.filter((e) => !original.includes(e)).sort();
-  const duplicateHints = current.filter((e, i, a) => a.indexOf(e) !== i).sort();
+  const missingHints = original.filter((e) => !current.includes(e.toLowerCase())).sort();
+  const orphanedHints = current.filter((e) => !original.includes(e.toLowerCase())).sort();
+  const duplicateHints = current.filter((e, i, a) => a.indexOf(e.toLowerCase()) !== i).sort();
   log('duplicate hints:', duplicateHints);
   log('duplicate labels:', duplicateUI);
   return [missingHints, orphanedHints];
@@ -93,8 +93,8 @@ async function addMissingHints(json, missingHints) {
 
 async function removeOrphanedHints(json, orphanedHints) {
   const data = Object.values(json).flat().filter((e) => e.hint.length > 0);
-  for (const e of data) e.label = e.label.toLowerCase().trim();
-  const orphaned = data.filter((e) => orphanedHints.includes(e.label));
+  for (const e of data) e.label = e.label.trim();
+  const orphaned = data.filter((e) => orphanedHints.includes(e.label.toLowerCase()));
   log('orphaned hints:', { orphaned });
 }
 
@@ -168,7 +168,6 @@ async function setHints(analyze = false) {
     overrideData = Object.values(json.override || {}).flat().filter((e) => e.hint.length > 0);
     const jsonData = Object.values(json).flat().filter((e) => e.hint.length > 0);
     localeData.data = [...overrideData, ...jsonData];
-    for (const e of localeData.data) e.label = e.label.toLowerCase().trim();
   }
   if (!localeData.hint) tooltipCreate();
   let localized = 0;
@@ -176,8 +175,8 @@ async function setHints(analyze = false) {
   const t0 = performance.now();
   for (const el of elements) {
     let found;
-    if (el.dataset.original) found = localeData.data.find((l) => l.label === el.dataset.original.toLowerCase().trim());
-    else found = localeData.data.find((l) => l.label === el.textContent.toLowerCase().trim());
+    if (el.dataset.original) found = localeData.data.find((l) => l.label.toLowerCase().trim() === el.dataset.original.toLowerCase().trim());
+    else found = localeData.data.find((l) => l.label.toLowerCase().trim() === el.textContent.toLowerCase().trim());
     if (found?.localized?.length > 0) {
       if (!el.dataset.original) el.dataset.original = el.textContent;
       localized++;

modules/interrogate/vqa.py

@@ -110,7 +110,10 @@ def qwen(question: str, image: Image.Image, repo: str = None):
     global processor, model, loaded # pylint: disable=global-statement
     if model is None or loaded != repo:
         shared.log.debug(f'Interrogate load: vlm="{repo}"')
-        model = transformers.Qwen2VLForConditionalGeneration.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
+        model = transformers.Qwen2VLForConditionalGeneration.from_pretrained(
+            repo,
+            cache_dir=shared.opts.hfcache_dir
+        )
         processor = transformers.AutoProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
         loaded = repo
     model = model.to(devices.device, devices.dtype)
@@ -151,7 +154,11 @@ def paligemma(question: str, image: Image.Image, repo: str = None):
     if model is None or loaded != repo:
         shared.log.debug(f'Interrogate load: vlm="{repo}"')
         processor = transformers.PaliGemmaProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
-        model = transformers.PaliGemmaForConditionalGeneration.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype)
+        model = transformers.PaliGemmaForConditionalGeneration.from_pretrained(
+            repo,
+            cache_dir=shared.opts.hfcache_dir,
+            torch_dtype=devices.dtype,
+        )
         loaded = repo
     model = model.to(devices.device, devices.dtype)
     question = question.replace('<', '').replace('>', '').replace('_', ' ')
@@ -176,7 +183,13 @@ def ovis(question: str, image: Image.Image, repo: str = None):
     global model, loaded # pylint: disable=global-statement
     if model is None or loaded != repo:
         shared.log.debug(f'Interrogate load: vlm="{repo}"')
-        model = transformers.AutoModelForCausalLM.from_pretrained(repo, torch_dtype=devices.dtype, multimodal_max_length=32768, trust_remote_code=True)
+        model = transformers.AutoModelForCausalLM.from_pretrained(
+            repo,
+            torch_dtype=devices.dtype,
+            multimodal_max_length=32768,
+            trust_remote_code=True,
+            cache_dir=shared.opts.hfcache_dir,
+        )
         loaded = repo
     model = model.to(devices.device, devices.dtype)
     text_tokenizer = model.get_text_tokenizer()
@@ -251,7 +264,10 @@ def git(question: str, image: Image.Image, repo: str = None):
     global processor, model, loaded # pylint: disable=global-statement
     if model is None or loaded != repo:
         shared.log.debug(f'Interrogate load: vlm="{repo}"')
-        model = transformers.GitForCausalLM.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
+        model = transformers.GitForCausalLM.from_pretrained(
+            repo,
+            cache_dir=shared.opts.hfcache_dir,
+        )
         processor = transformers.GitProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
         loaded = repo
     model.to(devices.device, devices.dtype)
@@ -273,7 +289,10 @@ def blip(question: str, image: Image.Image, repo: str = None):
     global processor, model, loaded # pylint: disable=global-statement
     if model is None or loaded != repo:
         shared.log.debug(f'Interrogate load: vlm="{repo}"')
-        model = transformers.BlipForQuestionAnswering.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
+        model = transformers.BlipForQuestionAnswering.from_pretrained(
+            repo,
+            cache_dir=shared.opts.hfcache_dir,
+        )
         processor = transformers.BlipProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
         loaded = repo
     model.to(devices.device, devices.dtype)
@@ -289,7 +308,10 @@ def vilt(question: str, image: Image.Image, repo: str = None):
     global processor, model, loaded # pylint: disable=global-statement
     if model is None or loaded != repo:
         shared.log.debug(f'Interrogate load: vlm="{repo}"')
-        model = transformers.ViltForQuestionAnswering.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
+        model = transformers.ViltForQuestionAnswering.from_pretrained(
+            repo,
+            cache_dir=shared.opts.hfcache_dir,
+        )
         processor = transformers.ViltProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
         loaded = repo
     model.to(devices.device)
@@ -307,7 +329,10 @@ def pix(question: str, image: Image.Image, repo: str = None):
     global processor, model, loaded # pylint: disable=global-statement
     if model is None or loaded != repo:
         shared.log.debug(f'Interrogate load: vlm="{repo}"')
-        model = transformers.Pix2StructForConditionalGeneration.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
+        model = transformers.Pix2StructForConditionalGeneration.from_pretrained(
+            repo,
+            cache_dir=shared.opts.hfcache_dir,
+        )
         processor = transformers.Pix2StructProcessor.from_pretrained(repo, cache_dir=shared.opts.hfcache_dir)
         loaded = repo
     model.to(devices.device)
@@ -356,7 +381,12 @@ def florence(question: str, image: Image.Image, repo: str = None, revision: str
     if model is None or loaded != repo:
         shared.log.debug(f'Interrogate load: vlm="{repo}" path="{shared.opts.hfcache_dir}"')
         transformers.dynamic_module_utils.get_imports = get_imports
-        model = transformers.AutoModelForCausalLM.from_pretrained(repo, trust_remote_code=True, revision=revision, cache_dir=shared.opts.hfcache_dir)
+        model = transformers.AutoModelForCausalLM.from_pretrained(
+            repo,
+            trust_remote_code=True,
+            revision=revision,
+            cache_dir=shared.opts.hfcache_dir,
+        )
         processor = transformers.AutoProcessor.from_pretrained(repo, trust_remote_code=True, revision=revision, cache_dir=shared.opts.hfcache_dir)
         transformers.dynamic_module_utils.get_imports = _get_imports
         loaded = repo

package.json

@@ -18,6 +18,7 @@
   "scripts": {
     "venv": ". venv/bin/activate",
     "start": "npm run venv; python launch.py --debug",
+    "localize": "node cli/localize.js",
     "ruff": ". venv/bin/activate && ruff check",
     "eslint": "eslint javascript/ extensions-builtin/sdnext-modernui/javascript/",
     "pylint": ". venv/bin/activate && pylint *.py modules/ extensions-builtin/ | grep -v '^*'",