lasta/AI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[crawler]

Browse Source
This commit is contained in:
LASTA_DEV01\lasta
2026-05-08 17:41:15 +09:00
commit 4158789869
54 changed files with 4355 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

405
crawler_platform/app/web/static/app.js Normal file
View File

@@ -0,0 +1,405 @@
const state = {
projects: [],
selectedProject: null,
projectDetail: null,
ontology: null,
};
const $ = (id) => document.getElementById(id);
function csv(value) {
return value
.split(",")
.map((item) => item.trim())
.filter(Boolean);
}
async function api(path, options = {}) {
const response = await fetch(path, {
headers: { "Content-Type": "application/json" },
...options,
});
if (!response.ok) {
let detail = `${response.status} ${response.statusText}`;
try {
const body = await response.json();
detail = body.detail || body.error || detail;
} catch {
// Keep the HTTP status text.
}
throw new Error(detail);
}
return response.json();
}
function toast(message) {
const node = $("toast");
node.textContent = message;
node.classList.add("show");
window.setTimeout(() => node.classList.remove("show"), 2400);
}
function renderProjects() {
const list = $("projectList");
list.innerHTML = "";
state.projects.forEach((project) => {
const button = document.createElement("button");
button.className = `project-item ${state.selectedProject === project.name ? "active" : ""}`;
button.innerHTML = `<strong>${project.name}</strong><span>${project.domain}</span>`;
button.addEventListener("click", () => selectProject(project.name));
list.appendChild(button);
});
}
async function loadProjects() {
state.projects = await api("/projects");
if (!state.selectedProject && state.projects.length) {
state.selectedProject = state.projects[0].name;
}
renderProjects();
if (state.selectedProject) {
await selectProject(state.selectedProject);
}
}
async function selectProject(projectName) {
state.selectedProject = projectName;
state.projectDetail = await api(`/projects/${encodeURIComponent(projectName)}`);
state.ontology = await api(`/ontology/${encodeURIComponent(state.projectDetail.domain)}`);
renderProjects();
renderOverview();
renderOntology();
await Promise.all([loadEntities(), loadClaims(), loadTags()]);
}
function renderOverview() {
const detail = state.projectDetail;
$("metricProject").textContent = detail?.name ?? "-";
$("metricDomain").textContent = detail?.domain ?? "-";
$("metricSources").textContent = detail?.sources?.length ?? 0;
const sourceSelect = $("sourceSelect");
sourceSelect.innerHTML = "";
(detail?.sources ?? []).forEach((source) => {
const option = document.createElement("option");
option.value = source.name;
option.textContent = `${source.name} (${source.type})`;
sourceSelect.appendChild(option);
});
$("sourceTable").innerHTML = table(
["Name", "Type", "Trust", "Robots", "Rate"],
(detail?.sources ?? []).map((source) => [
source.name,
source.type,
source.trust_level,
source.respect_robots_txt ? "on" : "off",
`${source.rate_limit_per_minute}/min`,
])
);
}
function renderOntology() {
const entityTypes = state.ontology?.entity_types ?? [];
const predicates = state.ontology?.predicates ?? [];
$("ontologyEntities").innerHTML = entityTypes.map(chip).join("");
$("ontologyPredicates").innerHTML = predicates.map(chip).join("");
const filter = $("entityTypeFilter");
filter.innerHTML = `<option value="">All types</option>${entityTypes
.map((type) => `<option value="${escapeHtml(type)}">${escapeHtml(type)}</option>`)
.join("")}`;
}
async function createProject() {
const configPath = $("configPath").value.trim();
if (!configPath) return;
const result = await api("/projects", {
method: "POST",
body: JSON.stringify({ config_path: configPath }),
});
toast(`프로젝트 생성: ${result.name}`);
state.selectedProject = result.name;
await loadProjects();
}
async function crawl() {
if (!state.selectedProject) return;
const sourceName = $("sourceSelect").value;
const url = $("crawlUrl").value.trim();
const provider = $("extractorProvider").value;
const model = $("extractorModel").value.trim();
const baseUrl = $("extractorBaseUrl").value.trim();
$("crawlResult").textContent = "수집 중...";
try {
const result = await api("/crawl", {
method: "POST",
body: JSON.stringify({
config_path: $("configPath").value.trim(),
source_name: sourceName,
url,
extractor_provider: provider,
extractor_model: model || null,
extractor_base_url: baseUrl || null,
}),
});
$("crawlResult").textContent = `analyzer ${provider}, page ${result.page_id}, claims ${result.claim_count}, entities ${result.entity_count}`;
toast("수집 완료");
await Promise.all([loadEntities(), loadClaims(), loadTags()]);
} catch (error) {
$("crawlResult").textContent = `수집 실패: ${error.message}`;
toast("수집 실패");
}
}
function updateExtractorOptions() {
const provider = $("extractorProvider").value;
$("extractorOptions").classList.toggle("active", provider !== "rule_based");
if (provider === "ollama" && !$("extractorBaseUrl").value.trim()) {
$("extractorBaseUrl").placeholder = "http://localhost:11434/api/chat";
} else if (provider === "lm_studio" && !$("extractorBaseUrl").value.trim()) {
$("extractorBaseUrl").placeholder = "http://localhost:1234/v1";
} else {
$("extractorBaseUrl").placeholder = "optional provider endpoint";
}
}
async function testExtractor() {
const provider = $("extractorProvider").value;
const baseUrl = $("extractorBaseUrl").value.trim();
$("crawlResult").textContent = "분석기 연결 확인 중...";
const result = await api("/extractors/models", {
method: "POST",
body: JSON.stringify({
provider,
base_url: baseUrl || null,
}),
});
if (!result.ok) {
$("crawlResult").textContent = `분석기 연결 실패: ${result.error}`;
toast("분석기 연결 실패");
return;
}
const models = result.models ?? [];
if (models.length && !$("extractorModel").value.trim()) {
$("extractorModel").value = models[0].id;
}
$("crawlResult").textContent = models.length
? `연결됨. 모델 ${models.length}개: ${models.map((model) => model.id).join(", ")}`
: "연결됨. 모델 목록은 비어 있습니다.";
toast("분석기 연결 확인 완료");
}
async function discover() {
const sourceName = $("sourceSelect").value;
const url = $("crawlUrl").value.trim();
$("crawlResult").textContent = "주소 발견 중...";
$("discoveredLinks").innerHTML = "";
try {
const result = await api("/discover", {
method: "POST",
body: JSON.stringify({
config_path: $("configPath").value.trim(),
source_name: sourceName,
url,
limit: 30,
}),
});
if (!result.ok) {
$("crawlResult").textContent = result.error ?? "주소 발견 실패";
return;
}
$("crawlResult").textContent = `발견된 주소 ${result.links.length}개`;
$("discoveredLinks").innerHTML = result.links.map(renderDiscoveredLink).join("");
document.querySelectorAll("[data-discovered-url]").forEach((button) => {
button.addEventListener("click", () => {
$("crawlUrl").value = button.dataset.discoveredUrl;
toast("URL 입력칸에 넣었습니다.");
});
});
} catch (error) {
$("crawlResult").textContent = `주소 발견 실패: ${error.message}`;
toast("주소 발견 실패");
}
}
function renderDiscoveredLink(link) {
return `
<button class="discovered-link" data-discovered-url="${escapeHtml(link.url)}">
<strong>${escapeHtml(link.label)}</strong>
<span>${escapeHtml(link.kind)} · ${escapeHtml(link.url)}</span>
</button>
`;
}
async function loadEntities() {
if (!state.selectedProject) return;
const type = $("entityTypeFilter").value;
const query = type ? `?entity_type=${encodeURIComponent(type)}&limit=100` : "?limit=100";
const entities = await api(`/projects/${encodeURIComponent(state.selectedProject)}/entities${query}`);
$("metricEntities").textContent = entities.length;
$("entityTable").innerHTML = table(
["ID", "Type", "Name", "Metadata"],
entities.map((entity) => [
entity.id,
entity.type,
entity.name,
`<code>${escapeHtml(JSON.stringify(entity.metadata ?? {}))}</code>`,
])
);
}
async function mergeEntities() {
if (!state.selectedProject) return;
const sourceId = Number($("mergeSourceId").value);
const targetId = Number($("mergeTargetId").value);
if (!sourceId || !targetId || sourceId === targetId) {
toast("병합할 ID와 남길 ID를 확인하세요.");
return;
}
const result = await api("/entities/merge", {
method: "POST",
body: JSON.stringify({
project_name: state.selectedProject,
source_entity_id: sourceId,
target_entity_id: targetId,
}),
});
if (!result.ok) {
toast(result.error ?? "병합 실패");
return;
}
toast("Entity 병합 완료");
$("mergeSourceId").value = "";
$("mergeTargetId").value = "";
await Promise.all([loadEntities(), loadClaims(), loadTags()]);
}
async function loadClaims() {
if (!state.selectedProject) return;
const claims = await api(`/projects/${encodeURIComponent(state.selectedProject)}/claims?limit=100`);
$("claimTable").innerHTML = claims.map(renderClaim).join("");
document.querySelectorAll("[data-save-claim]").forEach((button) => {
button.addEventListener("click", () => updateClaimConfidence(button.dataset.saveClaim));
});
}
function renderClaim(claim) {
const object = claim.object ?? JSON.stringify(claim.object_value ?? "");
return `
<article class="claim-card">
<div class="claim-main">
<strong>${escapeHtml(claim.subject)}</strong>
<span class="predicate">${escapeHtml(claim.predicate)}</span>
<span>${escapeHtml(object)}</span>
<span class="confidence">${Math.round(claim.confidence * 100)}%</span>
</div>
<div class="evidence">${escapeHtml(claim.evidence_text ?? "")}</div>
<div class="evidence">${escapeHtml(claim.source)} · ${escapeHtml(claim.page_url ?? "")}</div>
<div class="claim-actions">
<input id="confidence-${claim.id}" type="number" min="0" max="1" step="0.01" value="${claim.confidence}" aria-label="confidence" />
<input id="reason-${claim.id}" value="${escapeHtml(claim.confidence_reason ?? "")}" aria-label="reason" />
<button data-save-claim="${claim.id}">저장</button>
</div>
</article>
`;
}
async function updateClaimConfidence(claimId) {
const confidence = Number($(`confidence-${claimId}`).value);
const reason = $(`reason-${claimId}`).value.trim();
await api(`/claims/${claimId}/confidence`, {
method: "PATCH",
body: JSON.stringify({ confidence, reason }),
});
toast("신뢰도 수정 완료");
await loadClaims();
}
async function loadTags() {
if (!state.selectedProject) return;
const tags = await api(`/projects/${encodeURIComponent(state.selectedProject)}/recommendation-tags`);
$("tagTable").innerHTML = table(
["Predicate", "Type", "Name", "Support", "Confidence"],
tags.map((tag) => [
tag.predicate,
tag.type,
tag.name,
tag.support_count,
Math.round(tag.max_confidence * 100) + "%",
])
);
}
async function recommend() {
if (!state.selectedProject) return;
const result = await api("/recommend", {
method: "POST",
body: JSON.stringify({
project_name: state.selectedProject,
target_entity_type: state.projectDetail?.config?.recommendation?.target_entity_type ?? "Perfume",
preferences: {
preferred_notes: csv($("preferredNotes").value),
avoided_notes: csv($("avoidedNotes").value),
preferred_moods: csv($("preferredMoods").value),
season_context: $("seasonContext").value.trim() || null,
occasion_context: $("occasionContext").value.trim() || null,
},
limit: 10,
}),
});
$("recommendTable").innerHTML = table(
["Name", "Type", "Score", "Reasons"],
result.map((item) => [item.name, item.entity_type, item.score, item.reasons.join(", ")])
);
}
function chip(value) {
return `<span class="chip">${escapeHtml(value)}</span>`;
}
function table(headers, rows) {
if (!rows.length) {
return `<table><tbody><tr><td>데이터가 없습니다.</td></tr></tbody></table>`;
}
return `
<table>
<thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead>
<tbody>
${rows
.map((row) => `<tr>${row.map((cell) => `<td>${String(cell)}</td>`).join("")}</tr>`)
.join("")}
</tbody>
</table>
`;
}
function escapeHtml(value) {
return String(value)
.replaceAll("&", "&amp;")
.replaceAll("<", "&lt;")
.replaceAll(">", "&gt;")
.replaceAll('"', "&quot;")
.replaceAll("'", "&#039;");
}
document.querySelectorAll(".tab").forEach((tab) => {
tab.addEventListener("click", () => {
document.querySelectorAll(".tab").forEach((node) => node.classList.remove("active"));
document.querySelectorAll(".tab-panel").forEach((node) => node.classList.remove("active"));
tab.classList.add("active");
$(tab.dataset.tab).classList.add("active");
});
});
$("refreshBtn").addEventListener("click", loadProjects);
$("createProjectBtn").addEventListener("click", createProject);
$("discoverBtn").addEventListener("click", discover);
$("crawlBtn").addEventListener("click", crawl);
$("extractorProvider").addEventListener("change", updateExtractorOptions);
$("testExtractorBtn").addEventListener("click", testExtractor);
$("loadEntitiesBtn").addEventListener("click", loadEntities);
$("mergeEntitiesBtn").addEventListener("click", mergeEntities);
$("loadClaimsBtn").addEventListener("click", loadClaims);
$("loadTagsBtn").addEventListener("click", loadTags);
$("recommendBtn").addEventListener("click", recommend);
updateExtractorOptions();
loadProjects().catch((error) => toast(error.message));