AI/crawler_platform/app/web/static/app.js

const state = {
  projects: [],
  selectedProject: null,
  projectDetail: null,
  ontology: null,
};

const $ = (id) => document.getElementById(id);

function csv(value) {
  return value.split(",").map((item) => item.trim()).filter(Boolean);
}

async function api(path, options = {}) {
  const response = await fetch(path, {
    headers: { "Content-Type": "application/json" },
    ...options,
  });
  if (!response.ok) {
    let detail = `${response.status} ${response.statusText}`;
    try {
      const body = await response.json();
      detail = body.detail || body.error || detail;
    } catch {
      // Keep status text.
    }
    throw new Error(detail);
  }
  return response.json();
}

function toast(message) {
  const node = $("toast");
  node.textContent = message;
  node.classList.add("show");
  window.setTimeout(() => node.classList.remove("show"), 2400);
}

function requestBase() {
  return {
    config_path: $("configPath").value.trim(),
    source_name: $("sourceSelect").value,
    url: $("crawlUrl").value.trim(),
    extractor_provider: $("extractorProvider").value,
    extractor_model: $("extractorModel").value.trim() || null,
    extractor_base_url: $("extractorBaseUrl").value.trim() || null,
  };
}

async function loadProjects() {
  state.projects = await api("/projects");
  if (!state.selectedProject && state.projects.length) {
    state.selectedProject = state.projects[0].name;
  }
  renderProjects();
  if (state.selectedProject) {
    await selectProject(state.selectedProject);
  }
}

function renderProjects() {
  const list = $("projectList");
  list.innerHTML = "";
  state.projects.forEach((project) => {
    const button = document.createElement("button");
    button.className = `project-item ${state.selectedProject === project.name ? "active" : ""}`;
    button.innerHTML = `<strong>${escapeHtml(project.name)}</strong><span>${escapeHtml(project.domain)}</span>`;
    button.addEventListener("click", () => selectProject(project.name));
    list.appendChild(button);
  });
}

async function selectProject(projectName) {
  state.selectedProject = projectName;
  state.projectDetail = await api(`/projects/${encodeURIComponent(projectName)}`);
  state.ontology = await api(`/ontology/${encodeURIComponent(state.projectDetail.domain)}`);
  renderProjects();
  renderOverview();
  renderOntology();
  await Promise.all([loadEntities(), loadClaims(), loadTags()]);
}

function renderOverview() {
  const detail = state.projectDetail;
  $("metricProject").textContent = detail?.name ?? "-";
  $("metricDomain").textContent = detail?.domain ?? "-";
  $("metricSources").textContent = detail?.sources?.length ?? 0;

  const sourceSelect = $("sourceSelect");
  sourceSelect.innerHTML = "";
  (detail?.sources ?? []).forEach((source) => {
    const option = document.createElement("option");
    option.value = source.name;
    option.textContent = `${source.name} (${source.type})`;
    sourceSelect.appendChild(option);
  });
  $("sourceTable").innerHTML = table(
    ["Name", "Type", "Trust", "Robots", "Rate"],
    (detail?.sources ?? []).map((source) => [
      source.name,
      source.type,
      source.trust_level,
      source.respect_robots_txt ? "on" : "off",
      `${source.rate_limit_per_minute}/min`,
    ])
  );
}

function renderOntology() {
  const entityTypes = state.ontology?.entity_types ?? [];
  const predicates = state.ontology?.predicates ?? [];
  $("ontologyEntities").innerHTML = entityTypes.map(chip).join("");
  $("ontologyPredicates").innerHTML = predicates.map(chip).join("");
  $("entityTypeFilter").innerHTML = `<option value="">All types</option>${entityTypes
    .map((type) => `<option value="${escapeHtml(type)}">${escapeHtml(type)}</option>`)
    .join("")}`;
}

async function createProject() {
  const configPath = $("configPath").value.trim();
  if (!configPath) return;
  const result = await api("/projects", {
    method: "POST",
    body: JSON.stringify({ config_path: configPath }),
  });
  toast(`Project created: ${result.name}`);
  state.selectedProject = result.name;
  await loadProjects();
}

async function crawl() {
  if (!state.selectedProject) return;
  $("crawlResult").textContent = "Crawling one URL...";
  try {
    const result = await api("/crawl", {
      method: "POST",
      body: JSON.stringify(requestBase()),
    });
    $("crawlResult").textContent = `URL done: page ${result.page_id}, claims ${result.claim_count}, entities ${result.entity_count}`;
    toast("URL crawl completed");
    await Promise.all([loadEntities(), loadClaims(), loadTags()]);
  } catch (error) {
    $("crawlResult").textContent = `Crawl failed: ${error.message}`;
    toast("Crawl failed");
  }
}

async function crawlSite() {
  if (!state.selectedProject) return;
  $("crawlResult").textContent = "Crawling site from seed...";
  $("discoveredLinks").innerHTML = "";
  try {
    const result = await api("/crawl-site", {
      method: "POST",
      body: JSON.stringify({
        ...requestBase(),
        max_depth: Number($("siteMaxDepth").value || 0),
        max_pages: Number($("siteMaxPages").value || 1),
        same_domain_only: $("sameDomainOnly").checked,
        analyze_page_types: ["product", "brand", "review"],
      }),
    });
    $("crawlResult").textContent =
      `Site done: visited ${result.visited_count}, analyzed ${result.analyzed_count}, skipped ${result.skipped_count}, queued ${result.queued_count}`;
    $("discoveredLinks").innerHTML = result.pages.map(renderSitePage).join("");
    toast("Site crawl completed");
    await Promise.all([loadEntities(), loadClaims(), loadTags()]);
  } catch (error) {
    $("crawlResult").textContent = `Site crawl failed: ${error.message}`;
    toast("Site crawl failed");
  }
}

function renderSitePage(page) {
  return `
    <button class="discovered-link" data-discovered-url="${escapeHtml(page.url)}">
      <strong>${escapeHtml(page.status)} · ${escapeHtml(page.page_type)} · depth ${page.depth}</strong>
      <span>${escapeHtml(page.url)}</span>
      <span>claims ${page.claim_count}, entities ${page.entity_count}, links ${page.discovered_count}${page.error ? `, error: ${escapeHtml(page.error)}` : ""}</span>
    </button>
  `;
}

async function discover() {
  $("crawlResult").textContent = "Discovering links...";
  $("discoveredLinks").innerHTML = "";
  try {
    const result = await api("/discover", {
      method: "POST",
      body: JSON.stringify({
        config_path: $("configPath").value.trim(),
        source_name: $("sourceSelect").value,
        url: $("crawlUrl").value.trim(),
        limit: 30,
      }),
    });
    if (!result.ok) {
      $("crawlResult").textContent = result.error ?? "Discovery failed";
      return;
    }
    $("crawlResult").textContent = `Discovered ${result.links.length} links`;
    $("discoveredLinks").innerHTML = result.links.map(renderDiscoveredLink).join("");
    document.querySelectorAll("[data-discovered-url]").forEach((button) => {
      button.addEventListener("click", () => {
        $("crawlUrl").value = button.dataset.discoveredUrl;
        toast("URL copied to input");
      });
    });
  } catch (error) {
    $("crawlResult").textContent = `Discovery failed: ${error.message}`;
    toast("Discovery failed");
  }
}

function renderDiscoveredLink(link) {
  return `
    <button class="discovered-link" data-discovered-url="${escapeHtml(link.url)}">
      <strong>${escapeHtml(link.label)}</strong>
      <span>${escapeHtml(link.kind)} · ${escapeHtml(link.url)}</span>
    </button>
  `;
}

function updateExtractorOptions() {
  const provider = $("extractorProvider").value;
  $("extractorOptions").classList.toggle("active", provider !== "rule_based");
  if (provider === "ollama" && !$("extractorBaseUrl").value.trim()) {
    $("extractorBaseUrl").placeholder = "http://localhost:11434/api/chat";
  } else if (provider === "lm_studio" && !$("extractorBaseUrl").value.trim()) {
    $("extractorBaseUrl").placeholder = "http://localhost:1234/v1";
  } else {
    $("extractorBaseUrl").placeholder = "optional provider endpoint";
  }
}

async function testExtractor() {
  const provider = $("extractorProvider").value;
  const baseUrl = $("extractorBaseUrl").value.trim();
  $("crawlResult").textContent = "Testing analyzer...";
  const result = await api("/extractors/models", {
    method: "POST",
    body: JSON.stringify({ provider, base_url: baseUrl || null }),
  });
  if (!result.ok) {
    $("crawlResult").textContent = `Analyzer failed: ${result.error}`;
    toast("Analyzer failed");
    return;
  }
  const models = result.models ?? [];
  if (models.length && !$("extractorModel").value.trim()) {
    $("extractorModel").value = models[0].id;
  }
  $("crawlResult").textContent = models.length
    ? `Analyzer connected. Models: ${models.map((model) => model.id).join(", ")}`
    : "Analyzer connected. No models returned.";
  toast("Analyzer connected");
}

async function loadEntities() {
  if (!state.selectedProject) return;
  const type = $("entityTypeFilter").value;
  const query = type ? `?entity_type=${encodeURIComponent(type)}&limit=100` : "?limit=100";
  const entities = await api(`/projects/${encodeURIComponent(state.selectedProject)}/entities${query}`);
  $("metricEntities").textContent = entities.length;
  $("entityTable").innerHTML = table(
    ["ID", "Type", "Name", "Metadata"],
    entities.map((entity) => [
      entity.id,
      entity.type,
      entity.name,
      `<code>${escapeHtml(JSON.stringify(entity.metadata ?? {}))}</code>`,
    ])
  );
}

async function mergeEntities() {
  if (!state.selectedProject) return;
  const sourceId = Number($("mergeSourceId").value);
  const targetId = Number($("mergeTargetId").value);
  if (!sourceId || !targetId || sourceId === targetId) {
    toast("Check entity IDs");
    return;
  }
  const result = await api("/entities/merge", {
    method: "POST",
    body: JSON.stringify({
      project_name: state.selectedProject,
      source_entity_id: sourceId,
      target_entity_id: targetId,
    }),
  });
  if (!result.ok) {
    toast(result.error ?? "Merge failed");
    return;
  }
  toast("Entity merged");
  $("mergeSourceId").value = "";
  $("mergeTargetId").value = "";
  await Promise.all([loadEntities(), loadClaims(), loadTags()]);
}

async function loadClaims() {
  if (!state.selectedProject) return;
  const claims = await api(`/projects/${encodeURIComponent(state.selectedProject)}/claims?limit=100`);
  $("claimTable").innerHTML = claims.map(renderClaim).join("");
  document.querySelectorAll("[data-save-claim]").forEach((button) => {
    button.addEventListener("click", () => updateClaimConfidence(button.dataset.saveClaim));
  });
}

function renderClaim(claim) {
  const object = claim.object ?? JSON.stringify(claim.object_value ?? "");
  return `
    <article class="claim-card">
      <div class="claim-main">
        <strong>${escapeHtml(claim.subject)}</strong>
        <span class="predicate">${escapeHtml(claim.predicate)}</span>
        <span>${escapeHtml(object)}</span>
        <span class="confidence">${Math.round(claim.confidence * 100)}%</span>
      </div>
      <div class="evidence">${escapeHtml(claim.evidence_text ?? "")}</div>
      <div class="evidence">${escapeHtml(claim.source)} · ${escapeHtml(claim.page_url ?? "")}</div>
      <div class="claim-actions">
        <input id="confidence-${claim.id}" type="number" min="0" max="1" step="0.01" value="${claim.confidence}" aria-label="confidence" />
        <input id="reason-${claim.id}" value="${escapeHtml(claim.confidence_reason ?? "")}" aria-label="reason" />
        <button data-save-claim="${claim.id}">Save</button>
      </div>
    </article>
  `;
}

async function updateClaimConfidence(claimId) {
  const confidence = Number($(`confidence-${claimId}`).value);
  const reason = $(`reason-${claimId}`).value.trim();
  await api(`/claims/${claimId}/confidence`, {
    method: "PATCH",
    body: JSON.stringify({ confidence, reason }),
  });
  toast("Claim updated");
  await loadClaims();
}

async function loadTags() {
  if (!state.selectedProject) return;
  const tags = await api(`/projects/${encodeURIComponent(state.selectedProject)}/recommendation-tags`);
  $("tagTable").innerHTML = table(
    ["Predicate", "Type", "Name", "Support", "Confidence"],
    tags.map((tag) => [
      tag.predicate,
      tag.type,
      tag.name,
      tag.support_count,
      Math.round(tag.max_confidence * 100) + "%",
    ])
  );
}

async function recommend() {
  if (!state.selectedProject) return;
  const result = await api("/recommend", {
    method: "POST",
    body: JSON.stringify({
      project_name: state.selectedProject,
      target_entity_type: state.projectDetail?.config?.recommendation?.target_entity_type ?? "Perfume",
      preferences: {
        preferred_notes: csv($("preferredNotes").value),
        avoided_notes: csv($("avoidedNotes").value),
        preferred_moods: csv($("preferredMoods").value),
        season_context: $("seasonContext").value.trim() || null,
        occasion_context: $("occasionContext").value.trim() || null,
      },
      limit: 10,
    }),
  });
  $("recommendTable").innerHTML = table(
    ["Name", "Type", "Score", "Reasons"],
    result.map((item) => [item.name, item.entity_type, item.score, item.reasons.join(", ")])
  );
}

function chip(value) {
  return `<span class="chip">${escapeHtml(value)}</span>`;
}

function table(headers, rows) {
  if (!rows.length) {
    return `<table><tbody><tr><td>No data.</td></tr></tbody></table>`;
  }
  return `
    <table>
      <thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead>
      <tbody>${rows.map((row) => `<tr>${row.map((cell) => `<td>${String(cell)}</td>`).join("")}</tr>`).join("")}</tbody>
    </table>
  `;
}

function escapeHtml(value) {
  return String(value)
    .replaceAll("&", "&amp;")
    .replaceAll("<", "&lt;")
    .replaceAll(">", "&gt;")
    .replaceAll('"', "&quot;")
    .replaceAll("'", "&#039;");
}

document.querySelectorAll(".tab").forEach((tab) => {
  tab.addEventListener("click", () => {
    document.querySelectorAll(".tab").forEach((node) => node.classList.remove("active"));
    document.querySelectorAll(".tab-panel").forEach((node) => node.classList.remove("active"));
    tab.classList.add("active");
    $(tab.dataset.tab).classList.add("active");
  });
});

$("refreshBtn").addEventListener("click", loadProjects);
$("createProjectBtn").addEventListener("click", createProject);
$("discoverBtn").addEventListener("click", discover);
$("crawlBtn").addEventListener("click", crawl);
$("siteCrawlBtn").addEventListener("click", crawlSite);
$("extractorProvider").addEventListener("change", updateExtractorOptions);
$("testExtractorBtn").addEventListener("click", testExtractor);
$("loadEntitiesBtn").addEventListener("click", loadEntities);
$("mergeEntitiesBtn").addEventListener("click", mergeEntities);
$("loadClaimsBtn").addEventListener("click", loadClaims);
$("loadTagsBtn").addEventListener("click", loadTags);
$("recommendBtn").addEventListener("click", recommend);

updateExtractorOptions();
loadProjects().catch((error) => toast(error.message));