Support multiple deploy template list URLs with cross-list dedupe

Add Settings UI to configure many Portainer-format template catalogs (not just one), merge them on fetch, drop duplicates by image/stack/title, and show source counts on the Deploy view.

client/templateLists.js

@@ -0,0 +1,254 @@
+/**
+ * Multi-source Portainer-format template catalogs.
+ * Users can configure many list URLs (Portainer only allows one).
+ * Results are merged and de-duplicated across sources.
+ */
+
+const SETTINGS_KEY = 'peardock.settings.v1'
+
+/** Default catalog (same as previous hard-coded URL) */
+export const DEFAULT_TEMPLATE_LIST_URLS = [
+  'https://raw.githubusercontent.com/Lissy93/portainer-templates/main/templates.json',
+]
+
+/**
+ * @returns {object}
+ */
+function readSettingsBlob() {
+  try {
+    if (typeof localStorage === 'undefined') return {}
+    return JSON.parse(localStorage.getItem(SETTINGS_KEY) || '{}')
+  } catch {
+    return {}
+  }
+}
+
+/**
+ * Normalize / validate a list of URLs (unique, https/http only).
+ * @param {unknown} urls
+ * @returns {string[]}
+ */
+export function normalizeTemplateListUrls(urls) {
+  const arr = Array.isArray(urls) ? urls : []
+  const seen = new Set()
+  const out = []
+  for (const raw of arr) {
+    const u = String(raw || '').trim()
+    if (!u) continue
+    let parsed
+    try {
+      parsed = new URL(u)
+    } catch {
+      continue
+    }
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') continue
+    const key = parsed.href
+    if (seen.has(key)) continue
+    seen.add(key)
+    out.push(key)
+  }
+  return out
+}
+
+/**
+ * Configured template list URLs (falls back to default).
+ * @returns {string[]}
+ */
+export function getTemplateListUrls() {
+  const s = readSettingsBlob()
+  const fromSettings = normalizeTemplateListUrls(s.templateListUrls)
+  if (fromSettings.length) return fromSettings
+  return [...DEFAULT_TEMPLATE_LIST_URLS]
+}
+
+/**
+ * Persist template list URLs into settings (merge with existing settings).
+ * @param {string[]} urls
+ * @returns {string[]}
+ */
+export function setTemplateListUrls(urls) {
+  const normalized = normalizeTemplateListUrls(urls)
+  const finalUrls = normalized.length ? normalized : [...DEFAULT_TEMPLATE_LIST_URLS]
+  try {
+    const s = readSettingsBlob()
+    s.templateListUrls = finalUrls
+    localStorage.setItem(SETTINGS_KEY, JSON.stringify(s))
+  } catch (err) {
+    console.warn('[WARN] Failed to save template list URLs', err?.message || err)
+  }
+  return finalUrls
+}
+
+/**
+ * Stable key for de-duplication across catalogs.
+ * Prefer image identity, then stack repository, then title.
+ * @param {object} t
+ * @returns {string}
+ */
+export function templateDedupeKey(t) {
+  if (!t || typeof t !== 'object') return ''
+  const image = String(t.image || t.Image || '')
+    .toLowerCase()
+    .trim()
+    .replace(/:latest$/, '')
+  if (image) return `image:${image}`
+
+  const repoUrl = String(
+    t.repository?.url || t.repository?.URL || t.repository || t.repo || ''
+  )
+    .toLowerCase()
+    .trim()
+  const stackfile = String(
+    t.repository?.stackfile || t.stackfile || t.composeFile || ''
+  )
+    .toLowerCase()
+    .trim()
+  if (repoUrl) return `stack:${repoUrl}:${stackfile}`
+
+  const title = String(t.title || t.name || t.Name || '')
+    .toLowerCase()
+    .trim()
+  const type = String(t.type ?? t.Type ?? '')
+  if (title) return `title:${type}:${title}`
+  return ''
+}
+
+/**
+ * Extract template array from Portainer-format JSON (v2/v3 variants).
+ * @param {unknown} data
+ * @returns {object[]}
+ */
+export function extractTemplatesArray(data) {
+  if (!data) return []
+  if (Array.isArray(data)) return data.filter((t) => t && typeof t === 'object')
+  if (Array.isArray(data.templates)) {
+    return data.templates.filter((t) => t && typeof t === 'object')
+  }
+  // Some catalogs nest under versioned keys
+  if (data.templates && typeof data.templates === 'object' && !Array.isArray(data.templates)) {
+    return Object.values(data.templates).filter((t) => t && typeof t === 'object')
+  }
+  return []
+}
+
+/**
+ * Merge template arrays from multiple sources; drop duplicates (first wins).
+ * @param {Array<{ url: string, templates: object[] }>} batches
+ * @returns {{ templates: object[], stats: { sources: number, fetched: number, unique: number, duplicates: number, errors: Array<{url:string,error:string}> } }}
+ */
+export function mergeAndDedupeTemplates(batches) {
+  const seen = new Set()
+  const templates = []
+  let fetched = 0
+  let duplicates = 0
+  const errors = []
+
+  for (const batch of batches || []) {
+    if (batch?.error) {
+      errors.push({ url: batch.url || '', error: String(batch.error) })
+    }
+    const list = Array.isArray(batch?.templates) ? batch.templates : []
+    fetched += list.length
+    for (const t of list) {
+      const key = templateDedupeKey(t) || `anon:${templates.length}`
+      if (seen.has(key)) {
+        duplicates += 1
+        continue
+      }
+      seen.add(key)
+      // Annotate source for UI (non-breaking extra field)
+      templates.push({
+        ...t,
+        _sourceUrl: batch.url || null,
+      })
+    }
+  }
+
+  return {
+    templates,
+    stats: {
+      sources: (batches || []).filter((b) => !b?.error).length,
+      fetched,
+      unique: templates.length,
+      duplicates,
+      errors,
+    },
+  }
+}
+
+/**
+ * Fetch one catalog URL.
+ * @param {string} url
+ * @param {{ signal?: AbortSignal }} [opts]
+ * @returns {Promise<{ url: string, templates: object[], error?: string }>}
+ */
+export async function fetchTemplateList(url, opts = {}) {
+  try {
+    const res = await fetch(url, {
+      signal: opts.signal,
+      headers: { Accept: 'application/json' },
+    })
+    if (!res.ok) {
+      return { url, templates: [], error: `HTTP ${res.status}` }
+    }
+    const data = await res.json()
+    return { url, templates: extractTemplatesArray(data) }
+  } catch (err) {
+    return { url, templates: [], error: err?.message || String(err) }
+  }
+}
+
+/** @type {{ templates: object[], stats: object, urls: string[], at: number }|null} */
+let mergedCache = null
+const MERGED_CACHE_TTL_MS = 60_000
+
+export function clearMergedTemplateCache() {
+  mergedCache = null
+}
+
+/**
+ * Fetch all configured lists in parallel, merge + dedupe.
+ * @param {string[]} [urls]
+ * @param {{ signal?: AbortSignal, force?: boolean }} [opts]
+ * @returns {Promise<{ templates: object[], stats: object, urls: string[] }>}
+ */
+export async function fetchMergedTemplates(urls, opts = {}) {
+  const list = normalizeTemplateListUrls(urls ?? getTemplateListUrls())
+  const effective = list.length ? list : [...DEFAULT_TEMPLATE_LIST_URLS]
+  const cacheKey = effective.join('\n')
+
+  if (
+    !opts.force &&
+    mergedCache &&
+    Date.now() - mergedCache.at < MERGED_CACHE_TTL_MS &&
+    mergedCache.urls.join('\n') === cacheKey
+  ) {
+    return {
+      templates: mergedCache.templates,
+      stats: mergedCache.stats,
+      urls: mergedCache.urls,
+    }
+  }
+
+  const batches = await Promise.all(effective.map((url) => fetchTemplateList(url, opts)))
+  const merged = mergeAndDedupeTemplates(batches)
+  const result = {
+    ...merged,
+    urls: effective,
+  }
+  mergedCache = { ...result, at: Date.now() }
+  return result
+}
+
+export default {
+  DEFAULT_TEMPLATE_LIST_URLS,
+  getTemplateListUrls,
+  setTemplateListUrls,
+  normalizeTemplateListUrls,
+  templateDedupeKey,
+  extractTemplatesArray,
+  mergeAndDedupeTemplates,
+  fetchTemplateList,
+  fetchMergedTemplates,
+  clearMergedTemplateCache,
+}