domain builder

This commit is contained in:
LASTA_DEV01\lasta
2026-05-20 20:59:40 +09:00
parent 6a6befc485
commit de46b657c2
17 changed files with 5111 additions and 1243 deletions

View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import csv import csv
import io import io
import json import json
import re
from dataclasses import asdict from dataclasses import asdict
from typing import Any from typing import Any
@@ -28,7 +29,8 @@ from crawler_platform.app.core.database.repository import (
from crawler_platform.app.core.database.session import session_scope from crawler_platform.app.core.database.session import session_scope
from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
from crawler_platform.app.core.extractor.factory import extractor_for_domain from crawler_platform.app.core.extractor.factory import extractor_for_domain
from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, ontology_for_domain from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, Ontology, ontology_for_domain
from crawler_platform.app.core.ontology.domain_discovery import DomainDiscoveryService
from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
from crawler_platform.app.core.ontology.mapper import ontology_to_dict from crawler_platform.app.core.ontology.mapper import ontology_to_dict
from crawler_platform.app.core.ontology.registry import OntologyRegistry from crawler_platform.app.core.ontology.registry import OntologyRegistry
@@ -38,6 +40,9 @@ from crawler_platform.app.core.research.graph_research_loop import GraphResearch
from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload
DOMAIN_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{1,79}$")
class CrawlRequest(BaseModel): class CrawlRequest(BaseModel):
config_path: str config_path: str
source_name: str source_name: str
@@ -149,6 +154,41 @@ class CreateProjectInlineRequest(BaseModel):
) )
class DomainDefinitionRequest(BaseModel):
domain: str
description: str | None = None
entity_types: list[str] = Field(default_factory=list)
predicates: list[str] = Field(default_factory=list)
attributes: list[str] = Field(default_factory=list)
aliases: dict[str, str] = Field(default_factory=dict)
class DomainDefinitionUpdateRequest(BaseModel):
description: str | None = None
entity_types: list[str] = Field(default_factory=list)
predicates: list[str] = Field(default_factory=list)
attributes: list[str] = Field(default_factory=list)
aliases: dict[str, str] = Field(default_factory=dict)
class DomainDiscoveryRequest(BaseModel):
seed_urls: list[str] = Field(default_factory=list)
max_pages: int = 20
max_depth: int = 1
same_domain_only: bool = True
fetcher: str = "requests"
respect_robots_txt: bool = False
force_recrawl: bool = False
class DomainCandidateApplyRequest(BaseModel):
candidate_ids: list[int] = Field(default_factory=list)
class DomainCandidateStatusRequest(BaseModel):
status: str
class ResetProjectRequest(BaseModel): class ResetProjectRequest(BaseModel):
config_path: str config_path: str
project_name: str | None = None project_name: str | None = None
@@ -165,6 +205,250 @@ def source_model_to_config(source: models.Source) -> SourceConfig:
) )
def normalize_domain_name(domain: str) -> str:
normalized = domain.strip().lower()
if not DOMAIN_NAME_PATTERN.match(normalized):
raise HTTPException(
status_code=400,
detail="Domain must be 2-80 chars using lowercase letters, numbers, underscore, or hyphen.",
)
return normalized
def clean_string_list(values: list[str]) -> list[str]:
cleaned: list[str] = []
seen: set[str] = set()
for value in values:
item = value.strip()
if not item or item in seen:
continue
cleaned.append(item)
seen.add(item)
return cleaned
def clean_aliases(values: dict[str, str]) -> dict[str, str]:
return {
key.strip(): value.strip()
for key, value in values.items()
if key.strip() and value.strip()
}
def domain_row_to_ontology(row: models.DomainDefinition):
return Ontology(
domain=row.domain,
entity_types=clean_string_list(list(row.entity_types or [])),
predicates=clean_string_list(list(row.predicates or [])),
attributes=clean_string_list(list(row.attributes or [])),
aliases=clean_aliases(dict(row.aliases or {})),
)
def domain_summary_payload(ont, *, is_builtin: bool, is_custom: bool, description: str | None = None):
return {
"domain": ont.domain,
"description": description,
"entity_types": list(ont.entity_types),
"predicates": list(ont.predicates),
"attribute_count": len(ont.attributes),
"attributes": list(ont.attributes),
"aliases": dict(ont.aliases),
"is_builtin": is_builtin,
"is_custom": is_custom,
}
def upsert_domain_definition(
session,
domain: str,
payload: DomainDefinitionUpdateRequest,
) -> models.DomainDefinition:
normalized_domain = normalize_domain_name(domain)
row = session.scalar(
select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
)
if row is None:
row = models.DomainDefinition(domain=normalized_domain)
session.add(row)
session.flush()
row.description = payload.description.strip() if payload.description else None
row.entity_types = clean_string_list(payload.entity_types)
row.predicates = clean_string_list(payload.predicates)
row.attributes = clean_string_list(payload.attributes)
row.aliases = clean_aliases(payload.aliases)
row.is_builtin_override = normalized_domain in DOMAIN_ONTOLOGIES
row.updated_at = models.utcnow()
return row
def ensure_domain_definition(session, domain: str) -> models.DomainDefinition:
normalized_domain = normalize_domain_name(domain)
row = session.scalar(
select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
)
if row is not None:
return row
base = DOMAIN_ONTOLOGIES.get(normalized_domain)
row = models.DomainDefinition(
domain=normalized_domain,
entity_types=list(base.entity_types) if base else [],
predicates=list(base.predicates) if base else [],
attributes=list(base.attributes) if base else [],
aliases=dict(base.aliases) if base else {},
is_builtin_override=normalized_domain in DOMAIN_ONTOLOGIES,
)
session.add(row)
session.flush()
return row
def ontology_for_domain_from_db(session, domain: str):
normalized_domain = normalize_domain_name(domain)
row = session.scalar(
select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
)
if row is not None:
return domain_row_to_ontology(row)
return ontology_for_domain(normalized_domain)
def clean_seed_urls(values: list[str]) -> list[str]:
cleaned: list[str] = []
seen: set[str] = set()
for value in values:
url = value.strip()
if not url or url in seen:
continue
if not (url.startswith("http://") or url.startswith("https://") or url.startswith("file://")):
raise HTTPException(status_code=400, detail=f"Unsupported URL: {url}")
cleaned.append(url)
seen.add(url)
if not cleaned:
raise HTTPException(status_code=400, detail="At least one reference URL is required.")
return cleaned
def domain_discovery_job_payload(job: models.DomainDiscoveryJob) -> dict[str, Any]:
return {
"job_id": job.id,
"domain": job.domain,
"status": job.status,
"seed_urls": list(job.seed_urls or []),
"max_pages": job.max_pages,
"max_depth": job.max_depth,
"same_domain_only": job.same_domain_only,
"fetcher": job.fetcher,
"respect_robots_txt": job.respect_robots_txt,
"error": job.error,
"progress": job.progress or {},
"result_summary": job.result_summary or {},
"created_at": job.created_at.isoformat() if job.created_at else None,
"started_at": job.started_at.isoformat() if job.started_at else None,
"finished_at": job.finished_at.isoformat() if job.finished_at else None,
}
def domain_reference_source_payload(row: models.DomainReferenceSource) -> dict[str, Any]:
return {
"id": row.id,
"domain": row.domain,
"url": row.url,
"label": row.label,
"status": row.status,
"last_crawled_at": row.last_crawled_at.isoformat() if row.last_crawled_at else None,
"metadata": row.metadata_json or {},
"created_at": row.created_at.isoformat() if row.created_at else None,
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
}
def already_crawled_reference_urls(session, domain: str, seed_urls: list[str]) -> set[str]:
if not seed_urls:
return set()
rows = session.scalars(
select(models.DomainReferenceSource).where(
models.DomainReferenceSource.domain == domain,
models.DomainReferenceSource.url.in_(seed_urls),
models.DomainReferenceSource.status == "active",
models.DomainReferenceSource.last_crawled_at.is_not(None),
)
).all()
return {row.url for row in rows}
def domain_candidate_payload(row: models.DomainSchemaCandidate, evidence_rows: list[models.DomainCandidateEvidence]) -> dict[str, Any]:
return {
"id": row.id,
"domain": row.domain,
"job_id": row.job_id,
"candidate_type": row.candidate_type,
"name": row.name,
"description": row.description,
"confidence": row.confidence,
"occurrence_count": row.occurrence_count,
"status": row.status,
"metadata": row.metadata_json or {},
"created_at": row.created_at.isoformat() if row.created_at else None,
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
"evidence": [
{
"id": evidence.id,
"job_id": evidence.job_id,
"url": evidence.url,
"title": evidence.title,
"snippet": evidence.snippet,
"metadata": evidence.metadata_json or {},
"created_at": evidence.created_at.isoformat() if evidence.created_at else None,
}
for evidence in evidence_rows
],
}
def run_domain_discovery_job(database_url: str, job_id: int) -> None:
try:
with session_scope(database_url) as session:
DomainDiscoveryService(session).run_job(job_id)
except Exception as exc:
with session_scope(database_url) as session:
job = session.get(models.DomainDiscoveryJob, job_id)
if job is not None:
job.status = "failed"
job.error = str(exc)
job.finished_at = models.utcnow()
progress = dict(job.progress or {})
progress["errors"] = [*progress.get("errors", []), str(exc)]
job.progress = progress
def recover_interrupted_domain_discovery_jobs(database_url: str) -> None:
"""Mark domain discovery jobs that cannot survive a server restart."""
with session_scope(database_url) as session:
rows = session.scalars(
select(models.DomainDiscoveryJob).where(
models.DomainDiscoveryJob.status.in_(
["pending", "running", "cancel_requested"]
)
)
).all()
for job in rows:
progress = dict(job.progress or {})
errors = list(progress.get("errors") or [])
if job.status == "cancel_requested":
job.status = "canceled"
progress["completion_reason"] = "canceled"
else:
job.status = "failed"
job.error = "Server restarted before this background analysis finished."
progress["completion_reason"] = "interrupted"
errors.append(job.error)
progress["errors"] = errors[-10:]
job.progress = progress
job.finished_at = models.utcnow()
def project_config_from_project_row(session, project: models.Project) -> ProjectConfig: def project_config_from_project_row(session, project: models.Project) -> ProjectConfig:
config_dict = dict(project.config or {}) config_dict = dict(project.config or {})
if not config_dict: if not config_dict:
@@ -195,6 +479,32 @@ def project_config_to_dict(config: ProjectConfig) -> dict[str, Any]:
} }
def delete_project_data(session, project_id: int, include_project: bool = False) -> dict[str, int]:
repo = KnowledgeRepository(session)
deleted = repo.reset_project_runtime_data(project_id)
for table_model in [
models.OntologyProposal,
models.OntologyTriple,
models.OntologyRelationType,
models.OntologyEntityType,
models.Source,
]:
count = (
session.query(table_model)
.filter(table_model.project_id == project_id)
.delete(synchronize_session=False)
)
deleted[table_model.__tablename__] = deleted.get(table_model.__tablename__, 0) + int(count or 0)
if include_project:
project_count = (
session.query(models.Project)
.filter(models.Project.id == project_id)
.delete(synchronize_session=False)
)
deleted[models.Project.__tablename__] = int(project_count or 0)
return deleted
class CreateEntityRequest(BaseModel): class CreateEntityRequest(BaseModel):
entity_type: str entity_type: str
name: str name: str
@@ -485,6 +795,8 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A
def register_routes(app, database_url: str) -> None: def register_routes(app, database_url: str) -> None:
recover_interrupted_domain_discovery_jobs(database_url)
@app.get("/health") @app.get("/health")
def health(): def health():
return {"ok": True} return {"ok": True}
@@ -520,16 +832,298 @@ def register_routes(app, database_url: str) -> None:
@app.get("/domains") @app.get("/domains")
def list_domains(): def list_domains():
"""Available pre-defined ontology domains for project creation.""" """Available ontology domains for project creation."""
return [ with session_scope(database_url) as session:
{ rows = session.scalars(select(models.DomainDefinition)).all()
"domain": ont.domain, custom_by_domain = {row.domain: row for row in rows}
"entity_types": list(ont.entity_types), payloads = []
"predicates": list(ont.predicates), for domain, built_in in DOMAIN_ONTOLOGIES.items():
"attribute_count": len(ont.attributes), row = custom_by_domain.pop(domain, None)
if row is not None:
payloads.append(
domain_summary_payload(
domain_row_to_ontology(row),
is_builtin=True,
is_custom=True,
description=row.description,
)
)
else:
payloads.append(
domain_summary_payload(
built_in,
is_builtin=True,
is_custom=False,
)
)
for row in sorted(custom_by_domain.values(), key=lambda item: item.domain):
payloads.append(
domain_summary_payload(
domain_row_to_ontology(row),
is_builtin=False,
is_custom=True,
description=row.description,
)
)
return payloads
@app.post("/domains")
def create_domain(request: DomainDefinitionRequest):
domain = normalize_domain_name(request.domain)
with session_scope(database_url) as session:
existing = session.scalar(
select(models.DomainDefinition).where(models.DomainDefinition.domain == domain)
)
if existing is not None or domain in DOMAIN_ONTOLOGIES:
raise HTTPException(status_code=409, detail=f"Domain '{domain}' already exists.")
row = upsert_domain_definition(
session,
domain,
DomainDefinitionUpdateRequest(
description=request.description,
entity_types=request.entity_types,
predicates=request.predicates,
attributes=request.attributes,
aliases=request.aliases,
),
)
return domain_summary_payload(
domain_row_to_ontology(row),
is_builtin=domain in DOMAIN_ONTOLOGIES,
is_custom=True,
description=row.description,
)
@app.put("/domains/{domain}")
def update_domain(domain: str, request: DomainDefinitionUpdateRequest):
with session_scope(database_url) as session:
row = upsert_domain_definition(session, domain, request)
return domain_summary_payload(
domain_row_to_ontology(row),
is_builtin=row.domain in DOMAIN_ONTOLOGIES,
is_custom=True,
description=row.description,
)
@app.get("/domains/{domain}/discovery/jobs")
def list_domain_discovery_jobs(domain: str, limit: int = 20):
normalized_domain = normalize_domain_name(domain)
with session_scope(database_url) as session:
rows = session.scalars(
select(models.DomainDiscoveryJob)
.where(models.DomainDiscoveryJob.domain == normalized_domain)
.order_by(models.DomainDiscoveryJob.created_at.desc())
.limit(max(min(limit, 100), 1))
).all()
return [domain_discovery_job_payload(row) for row in rows]
@app.get("/domains/{domain}/reference-sources")
def list_domain_reference_sources(domain: str):
normalized_domain = normalize_domain_name(domain)
with session_scope(database_url) as session:
rows = session.scalars(
select(models.DomainReferenceSource)
.where(models.DomainReferenceSource.domain == normalized_domain)
.order_by(models.DomainReferenceSource.updated_at.desc())
).all()
return [domain_reference_source_payload(row) for row in rows]
@app.post("/domains/{domain}/discovery/jobs")
def create_domain_discovery_job(
domain: str,
request: DomainDiscoveryRequest,
background_tasks: BackgroundTasks,
):
normalized_domain = normalize_domain_name(domain)
seed_urls = clean_seed_urls(request.seed_urls)
with session_scope(database_url) as session:
ensure_domain_definition(session, normalized_domain)
skipped_seed_urls: list[str] = []
if not request.force_recrawl:
skipped = already_crawled_reference_urls(session, normalized_domain, seed_urls)
skipped_seed_urls = [url for url in seed_urls if url in skipped]
seed_urls = [url for url in seed_urls if url not in skipped]
if not seed_urls:
job = models.DomainDiscoveryJob(
domain=normalized_domain,
status="completed",
seed_urls=[],
max_pages=max(min(request.max_pages, 200), 1),
max_depth=max(min(request.max_depth, 5), 0),
same_domain_only=request.same_domain_only,
fetcher=request.fetcher if request.fetcher in {"requests", "playwright", "browser"} else "requests",
respect_robots_txt=request.respect_robots_txt,
progress={
"visited_count": 0,
"queued_count": 0,
"candidate_count": 0,
"observation_count": 0,
"max_pages": max(min(request.max_pages, 200), 1),
"skipped_seed_urls": skipped_seed_urls,
"errors": [],
"pages": [],
},
result_summary={
"visited_count": 0,
"candidate_count": 0,
"skipped_seed_urls": skipped_seed_urls,
},
finished_at=models.utcnow(),
)
session.add(job)
session.flush()
return domain_discovery_job_payload(job)
job = models.DomainDiscoveryJob(
domain=normalized_domain,
status="pending",
seed_urls=seed_urls,
max_pages=max(min(request.max_pages, 200), 1),
max_depth=max(min(request.max_depth, 5), 0),
same_domain_only=request.same_domain_only,
fetcher=request.fetcher if request.fetcher in {"requests", "playwright", "browser"} else "requests",
respect_robots_txt=request.respect_robots_txt,
progress={
"visited_count": 0,
"queued_count": len(seed_urls),
"candidate_count": 0,
"observation_count": 0,
"max_pages": max(min(request.max_pages, 200), 1),
"skipped_seed_urls": skipped_seed_urls,
"errors": [],
"pages": [],
},
)
session.add(job)
session.flush()
response = domain_discovery_job_payload(job)
background_tasks.add_task(run_domain_discovery_job, database_url, response["job_id"])
return response
@app.get("/domains/{domain}/discovery/jobs/{job_id}")
def domain_discovery_job(domain: str, job_id: int):
normalized_domain = normalize_domain_name(domain)
with session_scope(database_url) as session:
job = session.get(models.DomainDiscoveryJob, job_id)
if job is None or job.domain != normalized_domain:
raise HTTPException(status_code=404, detail="domain discovery job not found")
return domain_discovery_job_payload(job)
@app.post("/domains/{domain}/discovery/jobs/{job_id}/cancel")
def cancel_domain_discovery_job(domain: str, job_id: int):
normalized_domain = normalize_domain_name(domain)
with session_scope(database_url) as session:
job = session.get(models.DomainDiscoveryJob, job_id)
if job is None or job.domain != normalized_domain:
raise HTTPException(status_code=404, detail="domain discovery job not found")
if job.status in {"completed", "failed", "canceled"}:
return domain_discovery_job_payload(job)
job.status = "cancel_requested"
job.error = "cancel requested by user"
return domain_discovery_job_payload(job)
@app.get("/domains/{domain}/candidates")
def list_domain_candidates(domain: str, status: str | None = None, limit: int = 200):
normalized_domain = normalize_domain_name(domain)
with session_scope(database_url) as session:
query = select(models.DomainSchemaCandidate).where(
models.DomainSchemaCandidate.domain == normalized_domain
)
if status:
query = query.where(models.DomainSchemaCandidate.status == status)
rows = session.scalars(
query.order_by(
models.DomainSchemaCandidate.status,
models.DomainSchemaCandidate.candidate_type,
models.DomainSchemaCandidate.confidence.desc(),
models.DomainSchemaCandidate.name,
).limit(max(min(limit, 500), 1))
).all()
payload = []
for row in rows:
evidence = session.scalars(
select(models.DomainCandidateEvidence)
.where(models.DomainCandidateEvidence.candidate_id == row.id)
.order_by(models.DomainCandidateEvidence.created_at.desc())
.limit(3)
).all()
payload.append(domain_candidate_payload(row, evidence))
return payload
@app.post("/domains/{domain}/candidates/apply")
def apply_domain_candidates(domain: str, request: DomainCandidateApplyRequest):
normalized_domain = normalize_domain_name(domain)
if not request.candidate_ids:
raise HTTPException(status_code=400, detail="No candidates selected.")
with session_scope(database_url) as session:
definition = ensure_domain_definition(session, normalized_domain)
rows = session.scalars(
select(models.DomainSchemaCandidate).where(
models.DomainSchemaCandidate.domain == normalized_domain,
models.DomainSchemaCandidate.id.in_(request.candidate_ids),
)
).all()
entity_types = set(definition.entity_types or [])
predicates = set(definition.predicates or [])
attributes = set(definition.attributes or [])
aliases = dict(definition.aliases or {})
applied = {"entity_type": 0, "predicate": 0, "attribute": 0, "alias": 0}
for row in rows:
if row.candidate_type == "entity_type":
entity_types.add(row.name)
applied["entity_type"] += 1
elif row.candidate_type == "predicate":
predicates.add(row.name)
applied["predicate"] += 1
elif row.candidate_type == "attribute":
attributes.add(row.name)
applied["attribute"] += 1
elif row.candidate_type == "alias":
metadata = dict(row.metadata_json or {})
targets = metadata.get("targets") or []
target = metadata.get("target") or (targets[0] if targets else None)
if target:
aliases[row.name] = str(target)
applied["alias"] += 1
row.status = "approved"
row.updated_at = models.utcnow()
definition.entity_types = sorted(entity_types)
definition.predicates = sorted(predicates)
definition.attributes = sorted(attributes)
definition.aliases = aliases
definition.is_builtin_override = normalized_domain in DOMAIN_ONTOLOGIES
definition.updated_at = models.utcnow()
return {
"ok": True,
"domain": normalized_domain,
"applied": applied,
"definition": domain_summary_payload(
domain_row_to_ontology(definition),
is_builtin=normalized_domain in DOMAIN_ONTOLOGIES,
is_custom=True,
description=definition.description,
),
} }
for ont in DOMAIN_ONTOLOGIES.values()
] @app.post("/domains/{domain}/candidates/{candidate_id}/status")
def update_domain_candidate_status(domain: str, candidate_id: int, request: DomainCandidateStatusRequest):
normalized_domain = normalize_domain_name(domain)
if request.status not in {"pending_review", "approved", "rejected"}:
raise HTTPException(status_code=400, detail="Unsupported candidate status.")
with session_scope(database_url) as session:
row = session.get(models.DomainSchemaCandidate, candidate_id)
if row is None or row.domain != normalized_domain:
raise HTTPException(status_code=404, detail="candidate not found")
row.status = request.status
row.updated_at = models.utcnow()
evidence = session.scalars(
select(models.DomainCandidateEvidence)
.where(models.DomainCandidateEvidence.candidate_id == row.id)
.order_by(models.DomainCandidateEvidence.created_at.desc())
.limit(3)
).all()
return domain_candidate_payload(row, evidence)
@app.post("/projects/reset") @app.post("/projects/reset")
def reset_project(request: ResetProjectRequest): def reset_project(request: ResetProjectRequest):
@@ -568,6 +1162,35 @@ def register_routes(app, database_url: str) -> None:
"deleted": deleted, "deleted": deleted,
} }
@app.post("/projects/{project_name}/reset")
def reset_project_by_name(project_name: str):
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(project_name)
config = project_config_from_project_row(session, project)
deleted = repo.reset_project_runtime_data(project.id)
project = repo.upsert_project(config)
return {
"ok": True,
"name": project.name,
"domain": project.domain,
"reset": True,
"deleted": deleted,
}
@app.delete("/projects/{project_name}")
def delete_project(project_name: str):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
domain = project.domain
deleted = delete_project_data(session, project.id, include_project=True)
return {
"ok": True,
"name": project_name,
"domain": domain,
"deleted": deleted,
}
@app.get("/projects/{project_name}") @app.get("/projects/{project_name}")
def project_detail(project_name: str): def project_detail(project_name: str):
with session_scope(database_url) as session: with session_scope(database_url) as session:
@@ -630,7 +1253,8 @@ def register_routes(app, database_url: str) -> None:
@app.get("/ontology/{domain}") @app.get("/ontology/{domain}")
def ontology(domain: str): def ontology(domain: str):
return ontology_to_dict(ontology_for_domain(domain)) with session_scope(database_url) as session:
return ontology_to_dict(ontology_for_domain_from_db(session, domain))
@app.get("/projects/{project_name}/ontology/registry") @app.get("/projects/{project_name}/ontology/registry")
def ontology_registry(project_name: str): def ontology_registry(project_name: str):

View File

@@ -56,6 +56,91 @@ class Source(Base):
project = relationship("Project", back_populates="sources") project = relationship("Project", back_populates="sources")
class DomainDefinition(Base):
__tablename__ = "domain_definitions"
id = Column(Integer, primary_key=True)
domain = Column(String(80), unique=True, nullable=False, index=True)
description = Column(Text)
entity_types = Column(JSON, nullable=False, default=list)
predicates = Column(JSON, nullable=False, default=list)
attributes = Column(JSON, nullable=False, default=list)
aliases = Column(JSON, nullable=False, default=dict)
is_builtin_override = Column(Boolean, nullable=False, default=False)
created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
class DomainReferenceSource(Base):
__tablename__ = "domain_reference_sources"
__table_args__ = (
UniqueConstraint("domain", "url", name="uq_domain_reference_source"),
)
id = Column(Integer, primary_key=True)
domain = Column(String(80), nullable=False, index=True)
url = Column(Text, nullable=False)
label = Column(String(160))
status = Column(String(40), nullable=False, default="active", index=True)
last_crawled_at = Column(DateTime(timezone=True), nullable=True)
metadata_json = Column(JSON, nullable=False, default=dict)
created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
class DomainDiscoveryJob(Base):
__tablename__ = "domain_discovery_jobs"
id = Column(Integer, primary_key=True)
domain = Column(String(80), nullable=False, index=True)
status = Column(String(40), nullable=False, default="pending", index=True)
seed_urls = Column(JSON, nullable=False, default=list)
max_pages = Column(Integer, nullable=False, default=20)
max_depth = Column(Integer, nullable=False, default=1)
same_domain_only = Column(Boolean, nullable=False, default=True)
fetcher = Column(String(80), nullable=False, default="requests")
respect_robots_txt = Column(Boolean, nullable=False, default=False)
error = Column(Text)
progress = Column(JSON, nullable=False, default=dict)
result_summary = Column(JSON, nullable=False, default=dict)
created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
started_at = Column(DateTime(timezone=True), nullable=True)
finished_at = Column(DateTime(timezone=True), nullable=True)
class DomainSchemaCandidate(Base):
__tablename__ = "domain_schema_candidates"
__table_args__ = (
UniqueConstraint("domain", "candidate_type", "name", name="uq_domain_schema_candidate"),
)
id = Column(Integer, primary_key=True)
domain = Column(String(80), nullable=False, index=True)
job_id = Column(Integer, ForeignKey("domain_discovery_jobs.id"), nullable=True, index=True)
candidate_type = Column(String(40), nullable=False, index=True)
name = Column(String(160), nullable=False, index=True)
description = Column(Text)
confidence = Column(Float, nullable=False, default=0.5)
occurrence_count = Column(Integer, nullable=False, default=1)
status = Column(String(40), nullable=False, default="pending_review", index=True)
metadata_json = Column(JSON, nullable=False, default=dict)
created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
class DomainCandidateEvidence(Base):
__tablename__ = "domain_candidate_evidence"
id = Column(Integer, primary_key=True)
candidate_id = Column(Integer, ForeignKey("domain_schema_candidates.id"), nullable=False, index=True)
job_id = Column(Integer, ForeignKey("domain_discovery_jobs.id"), nullable=True, index=True)
url = Column(Text, nullable=False)
title = Column(Text)
snippet = Column(Text, nullable=False)
metadata_json = Column(JSON, nullable=False, default=dict)
created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
class Page(Base): class Page(Base):
__tablename__ = "pages" __tablename__ = "pages"
__table_args__ = (UniqueConstraint("project_id", "url", name="uq_page_project_url"),) __table_args__ = (UniqueConstraint("project_id", "url", name="uq_page_project_url"),)

View File

@@ -0,0 +1,643 @@
from __future__ import annotations
from collections import Counter, deque
from dataclasses import dataclass, field
import json
import re
from typing import Any
from urllib.parse import urlparse
from sqlalchemy import select
from sqlalchemy.orm import Session
from crawler_platform.app.core.crawler.discovery import discover_links
from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
from crawler_platform.app.core.crawler.page_classifier import classify_page
from crawler_platform.app.core.crawler.plugins import default_parser_registry
from crawler_platform.app.core.database import models
SCHEMA_TYPES = {"entity_type", "predicate", "attribute", "alias"}
RELATION_HINTS = {
"brand": ("Brand", "hasBrand"),
"브랜드": ("Brand", "hasBrand"),
"category": ("Category", "hasCategory"),
"카테고리": ("Category", "hasCategory"),
"ingredient": ("Ingredient", "hasIngredient"),
"ingredients": ("Ingredient", "hasIngredient"),
"성분": ("Ingredient", "hasIngredient"),
"effect": ("Effect", "hasEffect"),
"benefit": ("Effect", "hasEffect"),
"효과": ("Effect", "hasEffect"),
"효능": ("Effect", "hasEffect"),
"warning": ("Warning", "hasWarning"),
"주의": ("Warning", "hasWarning"),
"review": ("Review", "hasReview"),
"rating": ("Rating", "hasRating"),
"image": ("Image", "hasImage"),
"price": ("Price", "hasPrice"),
"가격": ("Price", "hasPrice"),
"offer": ("Offer", "hasOffer"),
}
@dataclass(slots=True)
class CandidateEvidence:
url: str
title: str | None
snippet: str
metadata: dict[str, Any] = field(default_factory=dict)
@dataclass(slots=True)
class CandidateObservation:
candidate_type: str
name: str
description: str
confidence: float
evidence: CandidateEvidence
metadata: dict[str, Any] = field(default_factory=dict)
class DomainDiscoveryService:
def __init__(self, session: Session):
self.session = session
self.parser = default_parser_registry().get("generic")
def run_job(self, job_id: int) -> None:
job = self.session.get(models.DomainDiscoveryJob, job_id)
if job is None:
return
job.status = "running"
job.started_at = models.utcnow()
job.progress = {
"visited_count": 0,
"queued_count": len(job.seed_urls or []),
"candidate_count": 0,
"errors": [],
"pages": [],
}
self.session.commit()
visited: set[str] = set()
queued: set[str] = set()
queue: deque[tuple[str, int, str]] = deque()
for seed_url in job.seed_urls or []:
normalized = normalize_url(str(seed_url))
if normalized:
queue.append((normalized, 0, seed_host(normalized)))
queued.add(normalized)
self._upsert_reference_source(job.domain, normalized)
fetcher = make_fetcher(job.fetcher, rate_limit_per_minute=30)
robots = RobotsPolicy()
observation_count = 0
errors: list[str] = []
page_payloads: list[dict[str, Any]] = []
while queue and len(visited) < max(min(job.max_pages, 200), 1):
if self._cancel_requested(job):
job.status = "canceled"
job.finished_at = models.utcnow()
self.session.commit()
return
url, depth, root_host = queue.popleft()
if url in visited:
continue
visited.add(url)
if depth > max(job.max_depth, 0):
continue
if job.same_domain_only and seed_host(url) != root_host:
continue
decision = robots.check(url, job.respect_robots_txt)
if not decision.allowed:
errors.append(f"{decision.reason}: {url}")
self._save_progress(job, visited, queue, observation_count, errors, page_payloads)
continue
try:
fetched = fetcher.fetch(url)
parsed = self.parser.parse(fetched.analysis_html, fetched.final_url or url)
page_type = classify_page(
fetched.final_url or url,
parsed.title or fetched.title,
parsed.text or parsed.raw_text,
html=fetched.analysis_html,
source_zones=parsed.source_zones or [],
)
page_observations = mine_schema_candidates(
url=fetched.final_url or url,
title=parsed.title or fetched.title,
text=parsed.text or parsed.raw_text,
html=fetched.analysis_html,
page_type=page_type,
)
observation_count += len(page_observations)
if page_observations:
self._save_observations(job, page_observations)
page_payloads.append(
{
"url": fetched.final_url or url,
"title": parsed.title or fetched.title,
"page_type": page_type,
"candidate_count": len(page_observations),
"text_length": len(parsed.text or ""),
"warnings": [*fetched.warnings, *(parsed.extraction_warnings or [])],
}
)
self._enqueue_links(
html=fetched.analysis_html,
base_url=fetched.final_url or url,
root_host=root_host,
depth=depth,
max_depth=max(job.max_depth, 0),
same_domain_only=job.same_domain_only,
queue=queue,
queued=queued,
visited=visited,
)
except Exception as exc:
errors.append(f"{url}: {exc}")
self._save_progress(job, visited, queue, observation_count, errors, page_payloads)
reached_page_limit = len(visited) >= max(min(job.max_pages, 200), 1)
completion_reason = "max_pages_reached" if reached_page_limit else "no_more_links"
saved = self._candidate_count(job.domain)
job.status = "completed"
job.finished_at = models.utcnow()
job.result_summary = {
"visited_count": len(visited),
"candidate_count": saved,
"completion_reason": completion_reason,
"errors": errors[-20:],
"pages": page_payloads[-50:],
}
self._save_progress(
job,
visited,
queue,
observation_count,
errors,
page_payloads,
completion_reason=completion_reason,
)
self.session.commit()
def _save_progress(
self,
job: models.DomainDiscoveryJob,
visited: set[str],
queue: deque[tuple[str, int, str]],
observation_count: int,
errors: list[str],
page_payloads: list[dict[str, Any]],
completion_reason: str | None = None,
) -> None:
job.progress = {
"visited_count": len(visited),
"queued_count": len(queue),
"candidate_count": self._candidate_count(job.domain),
"observation_count": observation_count,
"max_pages": job.max_pages,
"errors": errors[-10:],
"pages": page_payloads[-10:],
}
if completion_reason:
job.progress["completion_reason"] = completion_reason
self.session.commit()
def _cancel_requested(self, job: models.DomainDiscoveryJob) -> bool:
self.session.expire(job)
return job.status == "cancel_requested"
def _candidate_count(self, domain: str) -> int:
return int(
self.session.query(models.DomainSchemaCandidate)
.filter(models.DomainSchemaCandidate.domain == domain)
.count()
)
def _save_observations(self, job: models.DomainDiscoveryJob, observations: list[CandidateObservation]) -> int:
if not observations:
return 0
by_key: dict[tuple[str, str], list[CandidateObservation]] = {}
for observation in observations:
if observation.candidate_type not in SCHEMA_TYPES:
continue
key = (observation.candidate_type, observation.name)
by_key.setdefault(key, []).append(observation)
saved_count = 0
for (candidate_type, name), rows in sorted(by_key.items()):
row = self.session.scalar(
select(models.DomainSchemaCandidate).where(
models.DomainSchemaCandidate.domain == job.domain,
models.DomainSchemaCandidate.candidate_type == candidate_type,
models.DomainSchemaCandidate.name == name,
)
)
confidence = min(max(max(item.confidence for item in rows) + min(len(rows), 10) * 0.02, 0.0), 0.98)
metadata = merge_metadata(item.metadata for item in rows)
if row is None:
row = models.DomainSchemaCandidate(
domain=job.domain,
job_id=job.id,
candidate_type=candidate_type,
name=name,
description=rows[0].description,
confidence=round(confidence, 4),
occurrence_count=len(rows),
metadata_json=metadata,
)
self.session.add(row)
self.session.flush()
else:
row.job_id = job.id
row.description = row.description or rows[0].description
row.confidence = max(row.confidence, round(confidence, 4))
row.occurrence_count += len(rows)
row.metadata_json = {**(row.metadata_json or {}), **metadata}
row.updated_at = models.utcnow()
for evidence in rows[:8]:
self.session.add(
models.DomainCandidateEvidence(
candidate_id=row.id,
job_id=job.id,
url=evidence.evidence.url,
title=evidence.evidence.title,
snippet=evidence.evidence.snippet[:1000],
metadata_json=evidence.evidence.metadata,
)
)
saved_count += 1
return saved_count
def _upsert_reference_source(self, domain: str, url: str) -> models.DomainReferenceSource:
row = self.session.scalar(
select(models.DomainReferenceSource).where(
models.DomainReferenceSource.domain == domain,
models.DomainReferenceSource.url == url,
)
)
if row is None:
row = models.DomainReferenceSource(domain=domain, url=url)
self.session.add(row)
self.session.flush()
row.last_crawled_at = models.utcnow()
row.updated_at = models.utcnow()
return row
def _enqueue_links(
self,
*,
html: str,
base_url: str,
root_host: str,
depth: int,
max_depth: int,
same_domain_only: bool,
queue: deque[tuple[str, int, str]],
queued: set[str],
visited: set[str],
) -> None:
if depth >= max_depth:
return
for link in discover_links(html, base_url, limit=100):
next_url = normalize_url(link.url)
if not next_url or next_url in queued or next_url in visited:
continue
if same_domain_only and seed_host(next_url) != root_host:
continue
queue.append((next_url, depth + 1, root_host))
queued.add(next_url)
def mine_schema_candidates(
*,
url: str,
title: str | None,
text: str,
html: str,
page_type: str,
) -> list[CandidateObservation]:
observations: list[CandidateObservation] = []
evidence_base = {"page_type": page_type}
page_type_entity = entity_type_from_page_type(page_type)
if page_type_entity:
observations.append(
observation(
"entity_type",
page_type_entity,
f"Detected from page type {page_type}.",
0.72,
url,
title,
title or first_sentence(text),
evidence_base,
)
)
soup = soup_from_html(html)
labels = collect_structural_labels(soup, text)
json_ld_types, json_ld_keys = collect_json_ld_schema(soup)
for schema_type, count in json_ld_types.items():
observations.append(
observation(
"entity_type",
pascal_case(schema_type),
"Detected from JSON-LD schema type.",
min(0.86 + count * 0.02, 0.95),
url,
title,
f"JSON-LD @type: {schema_type}",
{**evidence_base, "source": "json_ld"},
)
)
for key, count in json_ld_keys.items():
attr = attribute_name(key)
if attr:
observations.append(
observation(
"attribute",
attr,
"Detected from structured data property.",
min(0.78 + count * 0.02, 0.92),
url,
title,
f"Structured data property: {key}",
{**evidence_base, "source": "json_ld", "raw_label": key},
)
)
for label, count in labels.items():
attr = attribute_name(label)
if not attr:
continue
observations.append(
observation(
"attribute",
attr,
"Detected from page labels, tables, metadata, or key-value text.",
min(0.62 + count * 0.04, 0.88),
url,
title,
label,
{**evidence_base, "source": "page_label", "raw_label": label},
)
)
hint = relation_hint(label)
if hint:
entity_type, predicate = hint
observations.append(
observation(
"entity_type",
entity_type,
f"Detected from repeated label '{label}'.",
min(0.68 + count * 0.03, 0.9),
url,
title,
label,
{**evidence_base, "source": "relation_hint", "raw_label": label},
)
)
observations.append(
observation(
"predicate",
predicate,
f"Suggested relation for label '{label}'.",
min(0.7 + count * 0.03, 0.92),
url,
title,
label,
{
**evidence_base,
"source": "relation_hint",
"raw_label": label,
"object_type": entity_type,
},
)
)
observations.append(
observation(
"alias",
attr,
f"Map source field '{attr}' to relation '{predicate}'.",
0.66,
url,
title,
label,
{**evidence_base, "source": "relation_hint", "target": predicate},
)
)
return observations
def observation(
candidate_type: str,
name: str,
description: str,
confidence: float,
url: str,
title: str | None,
snippet: str,
metadata: dict[str, Any],
) -> CandidateObservation:
return CandidateObservation(
candidate_type=candidate_type,
name=name[:160],
description=description,
confidence=confidence,
evidence=CandidateEvidence(url=url, title=title, snippet=clean_snippet(snippet), metadata=metadata),
metadata=metadata,
)
def soup_from_html(html: str):
from bs4 import BeautifulSoup
return BeautifulSoup(html or "", "html.parser")
def collect_json_ld_schema(soup) -> tuple[Counter[str], Counter[str]]:
types: Counter[str] = Counter()
keys: Counter[str] = Counter()
for tag in soup.find_all("script"):
script_type = " ".join(str(item) for item in tag.get("type", "").split()).lower()
if "ld+json" not in script_type:
continue
raw = tag.string or tag.get_text(" ", strip=True)
try:
payload = json.loads(raw)
except Exception:
continue
visit_json_ld(payload, types, keys)
return types, keys
def visit_json_ld(value: Any, types: Counter[str], keys: Counter[str]) -> None:
if isinstance(value, list):
for item in value:
visit_json_ld(item, types, keys)
return
if not isinstance(value, dict):
return
for key, nested in value.items():
if key == "@type":
for item in ensure_list(nested):
if isinstance(item, str) and is_schema_name(item):
types[item] += 1
elif not key.startswith("@"):
keys[key] += 1
visit_json_ld(nested, types, keys)
def collect_structural_labels(soup, text: str) -> Counter[str]:
labels: Counter[str] = Counter()
for selector in ["th", "dt", "label", "[itemprop]", "[property]", "[name]"]:
for node in soup.select(selector):
label = ""
if selector == "[itemprop]":
label = node.get("itemprop", "")
elif selector == "[property]":
label = str(node.get("property", "")).split(":")[-1]
elif selector == "[name]":
label = str(node.get("name", "")).split(":")[-1]
else:
label = node.get_text(" ", strip=True)
add_label(labels, label)
for line in text.splitlines()[:500]:
match = re.match(r"^\s*([^:|]{2,40})\s*[:|]\s*(.{1,200})$", line)
if match:
add_label(labels, match.group(1))
return labels
def add_label(labels: Counter[str], label: str) -> None:
cleaned = clean_label(label)
if cleaned and not is_noisy_label(cleaned):
labels[cleaned] += 1
def relation_hint(label: str) -> tuple[str, str] | None:
lowered = label.lower()
for token, hint in RELATION_HINTS.items():
if token in lowered:
return hint
return None
def entity_type_from_page_type(page_type: str) -> str | None:
mapping = {
"ProductPage": "Product",
"CategoryPage": "Category",
"BrandStoryPage": "Brand",
"NoticePage": "Notice",
"PromotionPage": "Promotion",
"ReviewPage": "Review",
"BoardPage": "Article",
}
return mapping.get(page_type)
def attribute_name(label: str) -> str:
cleaned = clean_label(label)
if not cleaned or is_noisy_label(cleaned):
return ""
if re.search(r"[가-힣]", cleaned):
return re.sub(r"[\s/()-]+", "_", cleaned).strip("_")[:80]
words = re.findall(r"[A-Za-z0-9]+", cleaned)
if not words:
return ""
return "_".join(word.lower() for word in words)[:80]
def pascal_case(value: str) -> str:
cleaned = clean_label(value)
parts = re.findall(r"[A-Za-z0-9가-힣]+", cleaned)
if not parts:
return ""
return "".join(part[:1].upper() + part[1:] for part in parts)[:80]
def clean_label(value: str) -> str:
return re.sub(r"\s+", " ", str(value or "").replace("\xa0", " ")).strip(" -_*:/|")
def is_schema_name(value: str) -> bool:
return 1 <= len(value) <= 80 and not value.startswith("http")
def is_noisy_label(value: str) -> bool:
lowered = value.lower().strip()
if len(lowered) < 2 or len(lowered) > 80:
return True
noisy = {
"class",
"style",
"viewport",
"charset",
"csrf-token",
"description",
"keywords",
"robots",
"generator",
"theme-color",
}
return lowered in noisy or lowered.startswith("twitter:")
def clean_snippet(value: str) -> str:
return re.sub(r"\s+", " ", str(value or "")).strip()[:1000]
def first_sentence(text: str) -> str:
cleaned = clean_snippet(text)
for delimiter in [". ", "\n", "", "다."]:
if delimiter in cleaned:
return cleaned.split(delimiter, 1)[0][:240]
return cleaned[:240]
def normalize_url(url: str) -> str:
return str(url or "").strip().rstrip("/")
def seed_host(url: str) -> str:
return urlparse(url).netloc.lower()
def ensure_list(value: Any) -> list[Any]:
if isinstance(value, list):
return value
return [value]
def merge_metadata(items) -> dict[str, Any]:
merged: dict[str, Any] = {}
sources: set[str] = set()
targets: set[str] = set()
object_types: set[str] = set()
for item in items:
merged.update(item or {})
if item.get("source"):
sources.add(str(item["source"]))
if item.get("target"):
targets.add(str(item["target"]))
if item.get("object_type"):
object_types.add(str(item["object_type"]))
if sources:
merged["sources"] = sorted(sources)
if targets:
merged["targets"] = sorted(targets)
if object_types:
merged["object_types"] = sorted(object_types)
return merged

View File

@@ -1,6 +1,7 @@
import { Routes, Route } from "react-router-dom"; import { Routes, Route } from "react-router-dom";
import AppShell from "@/components/layout/AppShell"; import AppShell from "@/components/layout/AppShell";
import OnboardingPage from "@/pages/OnboardingPage"; import OnboardingPage from "@/pages/OnboardingPage";
import DomainBuilderPage from "@/pages/DomainBuilderPage";
import ConfigureSourcesPage from "@/pages/ConfigureSourcesPage"; import ConfigureSourcesPage from "@/pages/ConfigureSourcesPage";
import CrawlPage from "@/pages/CrawlPage"; import CrawlPage from "@/pages/CrawlPage";
import ResearchPage from "@/pages/ResearchPage"; import ResearchPage from "@/pages/ResearchPage";
@@ -20,6 +21,7 @@ function App() {
<Route element={<AppShell />}> <Route element={<AppShell />}>
<Route path="/" element={<DashboardPage />} /> <Route path="/" element={<DashboardPage />} />
<Route path="/onboard" element={<OnboardingPage />} /> <Route path="/onboard" element={<OnboardingPage />} />
<Route path="/domains" element={<DomainBuilderPage />} />
<Route path="/sources/:projectId" element={<ConfigureSourcesPage />} /> <Route path="/sources/:projectId" element={<ConfigureSourcesPage />} />
<Route path="/crawl/:projectId" element={<CrawlPage />} /> <Route path="/crawl/:projectId" element={<CrawlPage />} />
<Route path="/pipeline/:projectId" element={<BuildPipelinePage />} /> <Route path="/pipeline/:projectId" element={<BuildPipelinePage />} />

View File

@@ -5,6 +5,7 @@ import {
ClipboardCheck, ClipboardCheck,
Code2, Code2,
GitBranch, GitBranch,
Globe2,
Layers3, Layers3,
LayoutDashboard, LayoutDashboard,
ListChecks, ListChecks,
@@ -66,6 +67,15 @@ export const NAV_GROUPS: NavGroup[] = [
keywords: ["create", "start", "wizard"], keywords: ["create", "start", "wizard"],
shortcut: "g n", shortcut: "g n",
}, },
{
id: "domains",
to: "/domains",
labelKey: "nav.domains",
label: "Domain Builder",
icon: Globe2,
keywords: ["domain", "schema", "reference", "crawler"],
shortcut: "g m",
},
], ],
}, },
{ {

View File

@@ -808,6 +808,7 @@ function TableShell<T>({
scope="col" scope="col"
style={{ style={{
width: header.getSize(), width: header.getSize(),
maxWidth: header.getSize(),
...getPinStyle(col, "header"), ...getPinStyle(col, "header"),
}} }}
aria-sort={ aria-sort={
@@ -921,16 +922,19 @@ function TableShell<T>({
key={cell.id} key={cell.id}
style={{ style={{
width: col.getSize(), width: col.getSize(),
maxWidth: col.getSize(),
...getPinStyle(col, "cell"), ...getPinStyle(col, "cell"),
}} }}
className={cn( className={cn(
cellPadClass, cellPadClass,
alignClass, alignClass,
"text-foreground", "overflow-hidden text-foreground align-middle",
pinned && "bg-surface", pinned && "bg-surface",
)} )}
> >
{flexRender(col.columnDef.cell, cell.getContext())} <div className="min-w-0 break-words">
{flexRender(col.columnDef.cell, cell.getContext())}
</div>
</td> </td>
); );
})} })}

View File

@@ -92,7 +92,7 @@ export const DialogPanel = React.forwardRef<HTMLDivElement, DialogPanelProps>(
"fixed left-1/2 top-1/2 z-50 -translate-x-1/2 -translate-y-1/2", "fixed left-1/2 top-1/2 z-50 -translate-x-1/2 -translate-y-1/2",
"flex max-h-[90vh] w-full max-w-lg flex-col overflow-hidden", "flex max-h-[90vh] w-full max-w-lg flex-col overflow-hidden",
"rounded-xl border border-border bg-surface-raised shadow-xl outline-none", "rounded-xl border border-border bg-surface-raised shadow-xl outline-none",
"data-[state=open]:animate-fade-in-up", "data-[state=open]:animate-fade-in",
className, className,
)} )}
{...props} {...props}

View File

@@ -1,5 +1,14 @@
import { useQuery } from "@tanstack/react-query"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { domainsApi, DomainSummary } from "@/lib/api/domains"; import {
domainsApi,
DomainDefinitionRequest,
DomainDefinitionUpdateRequest,
DomainDiscoveryJob,
DomainDiscoveryRequest,
DomainReferenceSource,
DomainCandidate,
DomainSummary,
} from "@/lib/api/domains";
import { ontologyApi, OntologyDetail } from "@/lib/api/ontology"; import { ontologyApi, OntologyDetail } from "@/lib/api/ontology";
import { queryKeys } from "./queryKeys"; import { queryKeys } from "./queryKeys";
@@ -11,6 +20,151 @@ export function useDomains() {
}); });
} }
export function useCreateDomain() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: (body: DomainDefinitionRequest) => domainsApi.create(body),
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: queryKeys.domains.all });
queryClient.invalidateQueries({ queryKey: queryKeys.ontology.all });
},
});
}
export function useUpdateDomain() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: ({
domain,
body,
}: {
domain: string;
body: DomainDefinitionUpdateRequest;
}) => domainsApi.update(domain, body),
onSuccess: (_data, variables) => {
queryClient.invalidateQueries({ queryKey: queryKeys.domains.all });
queryClient.invalidateQueries({
queryKey: queryKeys.ontology.byDomain(variables.domain),
});
},
});
}
export function useDomainDiscoveryJobs(domain: string | undefined) {
return useQuery<DomainDiscoveryJob[]>({
queryKey: [...queryKeys.domains.all, "discoveryJobs", domain ?? ""],
queryFn: () => domainsApi.discoveryJobs(domain!),
enabled: Boolean(domain),
refetchInterval: (query) => {
const jobs = query.state.data;
return jobs?.some((job) => ["pending", "running", "cancel_requested"].includes(job.status))
? 2000
: false;
},
});
}
export function useDomainReferenceSources(domain: string | undefined) {
return useQuery<DomainReferenceSource[]>({
queryKey: [...queryKeys.domains.all, "referenceSources", domain ?? ""],
queryFn: () => domainsApi.referenceSources(domain!),
enabled: Boolean(domain),
});
}
export function useDomainCandidates(domain: string | undefined) {
return useQuery<DomainCandidate[]>({
queryKey: [...queryKeys.domains.all, "candidates", domain ?? ""],
queryFn: () => domainsApi.candidates(domain!),
enabled: Boolean(domain),
refetchInterval: 2000,
});
}
export function useStartDomainDiscovery() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: ({
domain,
body,
}: {
domain: string;
body: DomainDiscoveryRequest;
}) => domainsApi.startDiscovery(domain, body),
onSuccess: (_data, variables) => {
queryClient.invalidateQueries({
queryKey: [...queryKeys.domains.all, "discoveryJobs", variables.domain],
});
queryClient.invalidateQueries({
queryKey: [...queryKeys.domains.all, "referenceSources", variables.domain],
});
queryClient.invalidateQueries({
queryKey: [...queryKeys.domains.all, "candidates", variables.domain],
});
},
});
}
export function useCancelDomainDiscovery() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: ({
domain,
jobId,
}: {
domain: string;
jobId: string;
}) => domainsApi.cancelDiscovery(domain, jobId),
onSuccess: (_data, variables) => {
queryClient.invalidateQueries({
queryKey: [...queryKeys.domains.all, "discoveryJobs", variables.domain],
});
},
});
}
export function useApplyDomainCandidates() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: ({
domain,
candidateIds,
}: {
domain: string;
candidateIds: string[];
}) => domainsApi.applyCandidates(domain, candidateIds),
onSuccess: (_data, variables) => {
queryClient.invalidateQueries({ queryKey: queryKeys.domains.all });
queryClient.invalidateQueries({
queryKey: queryKeys.ontology.byDomain(variables.domain),
});
queryClient.invalidateQueries({
queryKey: [...queryKeys.domains.all, "candidates", variables.domain],
});
},
});
}
export function useUpdateDomainCandidateStatus() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: ({
domain,
candidateId,
status,
}: {
domain: string;
candidateId: string;
status: "pending_review" | "approved" | "rejected";
}) => domainsApi.updateCandidateStatus(domain, candidateId, status),
onSuccess: (_data, variables) => {
queryClient.invalidateQueries({
queryKey: [...queryKeys.domains.all, "candidates", variables.domain],
});
},
});
}
export function useOntology(domain: string | undefined) { export function useOntology(domain: string | undefined) {
return useQuery<OntologyDetail>({ return useQuery<OntologyDetail>({
queryKey: queryKeys.ontology.byDomain(domain ?? ""), queryKey: queryKeys.ontology.byDomain(domain ?? ""),

View File

@@ -43,3 +43,31 @@ export function useCreateProjectInline() {
}, },
}); });
} }
export function useResetProject() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: (projectName: string) => projectsApi.reset(projectName),
onSuccess: (_data, projectName) => {
queryClient.invalidateQueries({ queryKey: queryKeys.projects.all });
queryClient.invalidateQueries({
queryKey: queryKeys.projects.detail(projectName),
});
queryClient.invalidateQueries({ queryKey: queryKeys.platform.all });
},
});
}
export function useDeleteProject() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: (projectName: string) => projectsApi.delete(projectName),
onSuccess: (_data, projectName) => {
queryClient.invalidateQueries({ queryKey: queryKeys.projects.all });
queryClient.removeQueries({
queryKey: queryKeys.projects.detail(projectName),
});
queryClient.invalidateQueries({ queryKey: queryKeys.platform.all });
},
});
}

View File

@@ -3,15 +3,177 @@ import { apiClient } from "./client";
export const domainSummarySchema = z.object({ export const domainSummarySchema = z.object({
domain: z.string(), domain: z.string(),
description: z.string().nullable().optional(),
entity_types: z.array(z.string()), entity_types: z.array(z.string()),
predicates: z.array(z.string()), predicates: z.array(z.string()),
attribute_count: z.number().int(), attribute_count: z.number().int(),
attributes: z.array(z.string()).default([]),
aliases: z.record(z.string(), z.string()).default({}),
is_builtin: z.boolean().default(false),
is_custom: z.boolean().default(false),
}); });
export const domainListSchema = z.array(domainSummarySchema); export const domainListSchema = z.array(domainSummarySchema);
export type DomainSummary = z.infer<typeof domainSummarySchema>; export type DomainSummary = z.infer<typeof domainSummarySchema>;
export const domainDiscoveryJobSchema = z.object({
job_id: z.union([z.string(), z.number()]).transform(String),
domain: z.string(),
status: z.string(),
seed_urls: z.array(z.string()).default([]),
max_pages: z.number(),
max_depth: z.number(),
same_domain_only: z.boolean(),
fetcher: z.string(),
respect_robots_txt: z.boolean(),
error: z.string().nullable().optional(),
progress: z.record(z.unknown()).default({}),
result_summary: z.record(z.unknown()).default({}),
created_at: z.string().nullable().optional(),
started_at: z.string().nullable().optional(),
finished_at: z.string().nullable().optional(),
});
export const domainReferenceSourceSchema = z.object({
id: z.union([z.string(), z.number()]).transform(String),
domain: z.string(),
url: z.string(),
label: z.string().nullable().optional(),
status: z.string(),
last_crawled_at: z.string().nullable().optional(),
metadata: z.record(z.unknown()).default({}),
created_at: z.string().nullable().optional(),
updated_at: z.string().nullable().optional(),
});
export const domainCandidateEvidenceSchema = z.object({
id: z.union([z.string(), z.number()]).transform(String),
job_id: z.union([z.string(), z.number()]).nullable().optional().transform((v) =>
v == null ? null : String(v),
),
url: z.string(),
title: z.string().nullable().optional(),
snippet: z.string(),
metadata: z.record(z.unknown()).default({}),
created_at: z.string().nullable().optional(),
});
export const domainCandidateSchema = z.object({
id: z.union([z.string(), z.number()]).transform(String),
domain: z.string(),
job_id: z.union([z.string(), z.number()]).nullable().optional().transform((v) =>
v == null ? null : String(v),
),
candidate_type: z.string(),
name: z.string(),
description: z.string().nullable().optional(),
confidence: z.number(),
occurrence_count: z.number(),
status: z.string(),
metadata: z.record(z.unknown()).default({}),
evidence: z.array(domainCandidateEvidenceSchema).default([]),
created_at: z.string().nullable().optional(),
updated_at: z.string().nullable().optional(),
});
export const domainDiscoveryJobListSchema = z.array(domainDiscoveryJobSchema);
export const domainReferenceSourceListSchema = z.array(domainReferenceSourceSchema);
export const domainCandidateListSchema = z.array(domainCandidateSchema);
export const domainCandidateApplyResponseSchema = z.object({
ok: z.boolean(),
domain: z.string(),
applied: z.record(z.number()),
definition: domainSummarySchema,
});
export type DomainDiscoveryJob = z.infer<typeof domainDiscoveryJobSchema>;
export type DomainReferenceSource = z.infer<typeof domainReferenceSourceSchema>;
export type DomainCandidate = z.infer<typeof domainCandidateSchema>;
export type DomainCandidateApplyResponse = z.infer<
typeof domainCandidateApplyResponseSchema
>;
export interface DomainDefinitionRequest {
domain: string;
description?: string | null;
entity_types: string[];
predicates: string[];
attributes: string[];
aliases?: Record<string, string>;
}
export type DomainDefinitionUpdateRequest = Omit<
DomainDefinitionRequest,
"domain"
>;
export interface DomainDiscoveryRequest {
seed_urls: string[];
max_pages?: number;
max_depth?: number;
same_domain_only?: boolean;
fetcher?: string;
respect_robots_txt?: boolean;
force_recrawl?: boolean;
}
export const domainsApi = { export const domainsApi = {
list: () => apiClient.get("/domains", domainListSchema), list: () => apiClient.get("/domains", domainListSchema),
create: (body: DomainDefinitionRequest) =>
apiClient.post("/domains", domainSummarySchema, body),
update: (domain: string, body: DomainDefinitionUpdateRequest) =>
apiClient.put(
`/domains/${encodeURIComponent(domain)}`,
domainSummarySchema,
body,
),
discoveryJobs: (domain: string) =>
apiClient.get(
`/domains/${encodeURIComponent(domain)}/discovery/jobs`,
domainDiscoveryJobListSchema,
),
referenceSources: (domain: string) =>
apiClient.get(
`/domains/${encodeURIComponent(domain)}/reference-sources`,
domainReferenceSourceListSchema,
),
startDiscovery: (domain: string, body: DomainDiscoveryRequest) =>
apiClient.post(
`/domains/${encodeURIComponent(domain)}/discovery/jobs`,
domainDiscoveryJobSchema,
body,
),
discoveryJob: (domain: string, jobId: string) =>
apiClient.get(
`/domains/${encodeURIComponent(domain)}/discovery/jobs/${encodeURIComponent(jobId)}`,
domainDiscoveryJobSchema,
),
cancelDiscovery: (domain: string, jobId: string) =>
apiClient.post(
`/domains/${encodeURIComponent(domain)}/discovery/jobs/${encodeURIComponent(jobId)}/cancel`,
domainDiscoveryJobSchema,
),
candidates: (domain: string, status?: string) =>
apiClient.get(
`/domains/${encodeURIComponent(domain)}/candidates`,
domainCandidateListSchema,
{ query: { status } },
),
applyCandidates: (domain: string, candidateIds: string[]) =>
apiClient.post(
`/domains/${encodeURIComponent(domain)}/candidates/apply`,
domainCandidateApplyResponseSchema,
{ candidate_ids: candidateIds.map(Number) },
),
updateCandidateStatus: (
domain: string,
candidateId: string,
status: "pending_review" | "approved" | "rejected",
) =>
apiClient.post(
`/domains/${encodeURIComponent(domain)}/candidates/${encodeURIComponent(candidateId)}/status`,
domainCandidateSchema,
{ status },
),
}; };

View File

@@ -35,9 +35,17 @@ export const createProjectResponseSchema = z.object({
domain: z.string(), domain: z.string(),
}); });
export const projectActionResponseSchema = z.object({
ok: z.boolean(),
name: z.string(),
domain: z.string().optional(),
deleted: z.record(z.number()).optional(),
});
export type ProjectSummary = z.infer<typeof projectSummarySchema>; export type ProjectSummary = z.infer<typeof projectSummarySchema>;
export type ProjectDetail = z.infer<typeof projectDetailSchema>; export type ProjectDetail = z.infer<typeof projectDetailSchema>;
export type ProjectSource = z.infer<typeof projectSourceSchema>; export type ProjectSource = z.infer<typeof projectSourceSchema>;
export type ProjectActionResponse = z.infer<typeof projectActionResponseSchema>;
export interface CreateProjectRequest { export interface CreateProjectRequest {
config_path: string; config_path: string;
@@ -77,4 +85,14 @@ export const projectsApi = {
apiClient.post("/projects", createProjectResponseSchema, body), apiClient.post("/projects", createProjectResponseSchema, body),
createInline: (body: CreateProjectInlineRequest) => createInline: (body: CreateProjectInlineRequest) =>
apiClient.post("/projects/inline", createProjectResponseSchema, body), apiClient.post("/projects/inline", createProjectResponseSchema, body),
reset: (projectName: string) =>
apiClient.post(
`/projects/${encodeURIComponent(projectName)}/reset`,
projectActionResponseSchema,
),
delete: (projectName: string) =>
apiClient.delete(
`/projects/${encodeURIComponent(projectName)}`,
projectActionResponseSchema,
),
}; };

View File

@@ -1,6 +1,15 @@
import { useNavigate } from "react-router-dom"; import { useNavigate } from "react-router-dom";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import { Plus, FolderOpen, AlertCircle, Clock } from "lucide-react"; import { useState } from "react";
import { toast } from "sonner";
import {
Plus,
FolderOpen,
AlertCircle,
Clock,
DatabaseZap,
Trash2,
} from "lucide-react";
import { import {
Card, Card,
CardContent, CardContent,
@@ -9,8 +18,21 @@ import {
CardTitle, CardTitle,
} from "@/components/ui/card"; } from "@/components/ui/card";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import {
Dialog,
DialogBody,
DialogDescription,
DialogFooter,
DialogHeader,
DialogPanel,
DialogTitle,
} from "@/components/ui/dialog";
import { Skeleton } from "@/components/ui/skeleton"; import { Skeleton } from "@/components/ui/skeleton";
import { useProjects } from "@/hooks/useProjects"; import {
useDeleteProject,
useProjects,
useResetProject,
} from "@/hooks/useProjects";
import { usePipeline } from "@/hooks/usePlatform"; import { usePipeline } from "@/hooks/usePlatform";
import { formatPercent } from "@/lib/display"; import { formatPercent } from "@/lib/display";
import { ProjectSummary } from "@/lib/api/projects"; import { ProjectSummary } from "@/lib/api/projects";
@@ -128,6 +150,11 @@ function ProjectCard({
onOpen: (path: string) => void; onOpen: (path: string) => void;
}) { }) {
const pipeline = usePipeline(project.name); const pipeline = usePipeline(project.name);
const resetProject = useResetProject();
const deleteProject = useDeleteProject();
const [confirmAction, setConfirmAction] = useState<"reset" | "delete" | null>(
null,
);
const claimStage = pipeline.data?.stages.find((stage) => stage.key === "claims"); const claimStage = pipeline.data?.stages.find((stage) => stage.key === "claims");
const approvedStage = pipeline.data?.stages.find( const approvedStage = pipeline.data?.stages.find(
(stage) => stage.key === "validated", (stage) => stage.key === "validated",
@@ -136,35 +163,136 @@ function ProjectCard({
const claimCount = claimStage?.count ?? 0; const claimCount = claimStage?.count ?? 0;
const approvedCount = approvedStage?.count ?? 0; const approvedCount = approvedStage?.count ?? 0;
const approvalRate = claimCount > 0 ? approvedCount / claimCount : 0; const approvalRate = claimCount > 0 ? approvedCount / claimCount : 0;
const isDeleting = deleteProject.isPending;
const isResetting = resetProject.isPending;
const handleProjectAction = async () => {
if (!confirmAction) return;
try {
if (confirmAction === "reset") {
await resetProject.mutateAsync(project.name);
toast.success(`${project.name} 데이터가 초기화되었습니다`);
} else {
await deleteProject.mutateAsync(project.name);
toast.success(`${project.name} 프로젝트가 삭제되었습니다`);
}
setConfirmAction(null);
} catch (error) {
toast.error((error as Error).message);
}
};
return ( return (
<Card <>
role="button" <Card
tabIndex={0} role="button"
onClick={() => onOpen(`/sources/${project.name}`)} tabIndex={0}
onKeyDown={(e) => { onClick={() => onOpen(`/sources/${project.name}`)}
if (e.key === "Enter" || e.key === " ") { onKeyDown={(e) => {
onOpen(`/sources/${project.name}`); if (e.key === "Enter" || e.key === " ") {
} onOpen(`/sources/${project.name}`);
}} }
className="cursor-pointer transition-colors hover:bg-accent/40" }}
> className="relative cursor-pointer transition-colors hover:bg-accent/40"
<CardHeader> >
<CardTitle>{project.name}</CardTitle> <div className="absolute right-3 top-3 z-10 flex items-center gap-1">
<CardDescription>{project.domain}</CardDescription> <Button
</CardHeader> type="button"
<CardContent> variant="ghost"
<div className="grid grid-cols-3 gap-2 text-xs"> size="icon"
<ProjectStat label="Pages" value={pageStage?.count ?? 0} /> className="h-8 w-8 text-muted-foreground hover:text-foreground"
<ProjectStat label="Claims" value={claimCount} /> aria-label={`${project.name} 데이터 초기화`}
<ProjectStat label="Approved" value={formatPercent(approvalRate)} /> title="데이터 초기화"
disabled={isDeleting || isResetting}
onClick={(e) => {
e.stopPropagation();
setConfirmAction("reset");
}}
>
<DatabaseZap className="h-4 w-4" />
</Button>
<Button
type="button"
variant="ghost"
size="icon"
className="h-8 w-8 text-muted-foreground hover:bg-destructive/10 hover:text-destructive"
aria-label={`${project.name} 프로젝트 삭제`}
title="프로젝트 삭제"
disabled={isDeleting || isResetting}
onClick={(e) => {
e.stopPropagation();
setConfirmAction("delete");
}}
>
<Trash2 className="h-4 w-4" />
</Button>
</div> </div>
<div className="mt-3 flex items-center gap-2 text-xs text-muted-foreground"> <CardHeader className="pr-24">
<Clock className="h-3 w-3" /> <CardTitle>{project.name}</CardTitle>
{formatRelative(project.updated_at)} <CardDescription>{project.domain}</CardDescription>
</div> </CardHeader>
</CardContent> <CardContent>
</Card> <div className="grid grid-cols-3 gap-2 text-xs">
<ProjectStat label="Pages" value={pageStage?.count ?? 0} />
<ProjectStat label="Claims" value={claimCount} />
<ProjectStat label="Approved" value={formatPercent(approvalRate)} />
</div>
<div className="mt-3 flex items-center gap-2 text-xs text-muted-foreground">
<Clock className="h-3 w-3" />
{formatRelative(project.updated_at)}
</div>
</CardContent>
</Card>
<Dialog
open={confirmAction !== null}
onOpenChange={(open) => {
if (!open && !isDeleting && !isResetting) setConfirmAction(null);
}}
ariaLabel="프로젝트 작업 확인"
>
<DialogPanel>
<DialogHeader onClose={() => setConfirmAction(null)}>
<DialogTitle>
{confirmAction === "delete" ? "프로젝트 삭제" : "데이터 초기화"}
</DialogTitle>
<DialogDescription>
{confirmAction === "delete"
? "프로젝트와 연결된 모든 데이터가 삭제됩니다."
: "프로젝트는 유지하고 크롤링/엔티티/클레임 데이터를 초기화합니다."}
</DialogDescription>
</DialogHeader>
<DialogBody>
<p className="text-sm">
<span className="font-semibold">{project.name}</span>
{confirmAction === "delete"
? " 프로젝트를 삭제할까요?"
: " 데이터를 초기화할까요?"}
</p>
</DialogBody>
<DialogFooter>
<Button
variant={confirmAction === "delete" ? "destructive" : "default"}
onClick={handleProjectAction}
disabled={isDeleting || isResetting}
>
{isDeleting || isResetting
? "처리 중..."
: confirmAction === "delete"
? "삭제"
: "초기화"}
</Button>
<Button
variant="outline"
onClick={() => setConfirmAction(null)}
disabled={isDeleting || isResetting}
>
</Button>
</DialogFooter>
</DialogPanel>
</Dialog>
</>
); );
} }

File diff suppressed because it is too large Load Diff

View File

@@ -1,10 +1,11 @@
import { useEffect, useState } from "react";
import { useNavigate } from "react-router-dom"; import { useNavigate } from "react-router-dom";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import { useForm } from "react-hook-form"; import { useForm } from "react-hook-form";
import { zodResolver } from "@hookform/resolvers/zod"; import { zodResolver } from "@hookform/resolvers/zod";
import { z } from "zod"; import { z } from "zod";
import { toast } from "sonner"; import { toast } from "sonner";
import { AlertCircle, ArrowLeft, Loader2 } from "lucide-react"; import { AlertCircle, ArrowLeft, Loader2, Pencil, Plus } from "lucide-react";
import { import {
Card, Card,
CardContent, CardContent,
@@ -12,30 +13,235 @@ import {
CardHeader, CardHeader,
CardTitle, CardTitle,
} from "@/components/ui/card"; } from "@/components/ui/card";
import {
Dialog,
DialogBody,
DialogDescription,
DialogFooter,
DialogHeader,
DialogPanel,
DialogTitle,
} from "@/components/ui/dialog";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input"; import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label"; import { Label } from "@/components/ui/label";
import { Skeleton } from "@/components/ui/skeleton"; import { Skeleton } from "@/components/ui/skeleton";
import { useDomains } from "@/hooks/useDomains"; import { Textarea } from "@/components/ui/textarea";
import {
useCreateDomain,
useDomains,
useUpdateDomain,
} from "@/hooks/useDomains";
import { useCreateProjectInline } from "@/hooks/useProjects"; import { useCreateProjectInline } from "@/hooks/useProjects";
import { DomainSummary } from "@/lib/api/domains";
import { cn } from "@/lib/utils"; import { cn } from "@/lib/utils";
const projectNameRegex = /^[a-zA-Z0-9_-]+$/; const projectNameRegex = /^[a-zA-Z0-9_-]+$/;
const domainNameRegex = /^[a-z0-9][a-z0-9_-]{1,79}$/;
const schema = z.object({ const schema = z.object({
project_name: z project_name: z
.string() .string()
.min(2, "최소 2자 이상") .min(2, "At least 2 characters")
.max(64, "최대 64자") .max(64, "Up to 64 characters")
.regex(projectNameRegex, "영문/숫자/_/- 만 허용"), .regex(projectNameRegex, "Use letters, numbers, underscore, or hyphen"),
domain: z.string().min(1, "도메인을 선택하세요"), domain: z.string().min(1, "Select a domain"),
}); });
type FormValues = z.infer<typeof schema>; type FormValues = z.infer<typeof schema>;
interface DomainEditorState {
mode: "create" | "edit";
domain?: DomainSummary;
}
function parseList(value: string): string[] {
const seen = new Set<string>();
return value
.split(/[\n,]/)
.map((item) => item.trim())
.filter((item) => {
if (!item || seen.has(item)) return false;
seen.add(item);
return true;
});
}
function formatList(values: string[] | undefined): string {
return (values ?? []).join("\n");
}
function parseAliases(value: string): Record<string, string> {
const aliases: Record<string, string> = {};
for (const line of value.split("\n")) {
const [rawKey, ...rawValue] = line.split("=");
const key = rawKey?.trim();
const aliasValue = rawValue.join("=").trim();
if (key && aliasValue) aliases[key] = aliasValue;
}
return aliases;
}
function formatAliases(values: Record<string, string> | undefined): string {
return Object.entries(values ?? {})
.map(([key, value]) => `${key}=${value}`)
.join("\n");
}
function DomainEditorDialog({
state,
onClose,
onSelect,
}: {
state: DomainEditorState | null;
onClose: () => void;
onSelect: (domain: string) => void;
}) {
const createDomain = useCreateDomain();
const updateDomain = useUpdateDomain();
const editing = state?.mode === "edit";
const domain = state?.domain;
const [name, setName] = useState("");
const [description, setDescription] = useState("");
const [entityTypes, setEntityTypes] = useState("");
const [predicates, setPredicates] = useState("");
const [attributes, setAttributes] = useState("");
const [aliases, setAliases] = useState("");
useEffect(() => {
if (!state) return;
setName(domain?.domain ?? "");
setDescription(domain?.description ?? "");
setEntityTypes(formatList(domain?.entity_types));
setPredicates(formatList(domain?.predicates));
setAttributes(formatList(domain?.attributes));
setAliases(formatAliases(domain?.aliases));
}, [domain, state]);
const saving = createDomain.isPending || updateDomain.isPending;
const save = async () => {
const normalized = name.trim().toLowerCase();
if (!domainNameRegex.test(normalized)) {
toast.error("도메인은 소문자, 숫자, _, - 조합의 2~80자로 입력하세요.");
return;
}
const body = {
description: description.trim() || null,
entity_types: parseList(entityTypes),
predicates: parseList(predicates),
attributes: parseList(attributes),
aliases: parseAliases(aliases),
};
if (body.entity_types.length === 0) {
toast.error("엔티티 타입을 최소 1개 이상 입력하세요.");
return;
}
try {
const saved = editing
? await updateDomain.mutateAsync({ domain: normalized, body })
: await createDomain.mutateAsync({ domain: normalized, ...body });
toast.success(editing ? "도메인을 수정했습니다." : "도메인을 생성했습니다.");
onSelect(saved.domain);
onClose();
} catch (e) {
toast.error((e as Error).message);
}
};
return (
<Dialog open={Boolean(state)} onOpenChange={(open) => !open && onClose()}>
<DialogPanel ariaLabel={editing ? "도메인 편집" : "도메인 생성"} className="max-w-2xl">
<DialogHeader onClose={onClose}>
<DialogTitle>{editing ? "도메인 편집" : "새 도메인 생성"}</DialogTitle>
<DialogDescription>
, , 릿 .
</DialogDescription>
</DialogHeader>
<DialogBody className="space-y-4">
<div className="grid gap-4 sm:grid-cols-2">
<div className="space-y-2">
<Label htmlFor="domain-name"> ID</Label>
<Input
id="domain-name"
value={name}
disabled={editing}
placeholder="cosmetic_ingredient"
onChange={(event) => setName(event.target.value)}
/>
</div>
<div className="space-y-2">
<Label htmlFor="domain-description"></Label>
<Input
id="domain-description"
value={description}
placeholder="화장품 성분 지식 그래프"
onChange={(event) => setDescription(event.target.value)}
/>
</div>
</div>
<div className="grid gap-4 sm:grid-cols-2">
<div className="space-y-2">
<Label htmlFor="domain-entities"> </Label>
<Textarea
id="domain-entities"
value={entityTypes}
className="min-h-[160px]"
placeholder={"Ingredient\nBrand\nProduct\nSkinConcern"}
onChange={(event) => setEntityTypes(event.target.value)}
/>
</div>
<div className="space-y-2">
<Label htmlFor="domain-predicates"></Label>
<Textarea
id="domain-predicates"
value={predicates}
className="min-h-[160px]"
placeholder={"hasIngredient\nsuitableForSkinType\nhasWarning"}
onChange={(event) => setPredicates(event.target.value)}
/>
</div>
</div>
<div className="grid gap-4 sm:grid-cols-2">
<div className="space-y-2">
<Label htmlFor="domain-attributes"></Label>
<Textarea
id="domain-attributes"
value={attributes}
placeholder={"name\nsource_url\nupdated_at\nconfidence"}
onChange={(event) => setAttributes(event.target.value)}
/>
</div>
<div className="space-y-2">
<Label htmlFor="domain-aliases"> </Label>
<Textarea
id="domain-aliases"
value={aliases}
placeholder={"ingredients=hasIngredient\nwarnings=hasWarning"}
onChange={(event) => setAliases(event.target.value)}
/>
</div>
</div>
</DialogBody>
<DialogFooter>
<Button type="button" onClick={save} disabled={saving}>
{saving && <Loader2 className="h-4 w-4 animate-spin" />}
</Button>
<Button type="button" variant="outline" onClick={onClose} disabled={saving}>
</Button>
</DialogFooter>
</DialogPanel>
</Dialog>
);
}
export default function OnboardingPage() { export default function OnboardingPage() {
const navigate = useNavigate(); const navigate = useNavigate();
const { t } = useTranslation(); const { t } = useTranslation();
const [editorState, setEditorState] = useState<DomainEditorState | null>(null);
const { const {
data: domains, data: domains,
isLoading: domainsLoading, isLoading: domainsLoading,
@@ -65,6 +271,15 @@ export default function OnboardingPage() {
project_name: values.project_name, project_name: values.project_name,
domain: values.domain, domain: values.domain,
target_entities: selected?.entity_types ?? [], target_entities: selected?.entity_types ?? [],
fields: selected?.attributes ?? [],
ontology: selected
? {
entity_types: selected.entity_types,
predicates: selected.predicates,
attributes: selected.attributes,
aliases: selected.aliases,
}
: {},
}); });
toast.success( toast.success(
t("onboarding.created", "프로젝트가 생성되었습니다: {{name}}", { t("onboarding.created", "프로젝트가 생성되었습니다: {{name}}", {
@@ -99,29 +314,33 @@ export default function OnboardingPage() {
<Card> <Card>
<CardHeader> <CardHeader>
<CardTitle> <div className="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
{t("onboarding.formTitle", "온톨로지 도메인 선택")} <div>
</CardTitle> <CardTitle>{t("onboarding.formTitle", "온톨로지 도메인 선택")}</CardTitle>
<CardDescription> <CardDescription>
{t( .
"onboarding.formDesc", </CardDescription>
"어떤 종류의 온톨로지를 구축할지 도메인을 선택하고 프로젝트 이름을 정해주세요.", </div>
)} <div className="flex flex-wrap gap-2">
</CardDescription> <Button type="button" variant="outline" onClick={() => navigate("/domains")}>
</Button>
<Button type="button" variant="outline" onClick={() => setEditorState({ mode: "create" })}>
<Plus className="h-4 w-4" />
</Button>
</div>
</div>
</CardHeader> </CardHeader>
<CardContent> <CardContent>
<form <form onSubmit={handleSubmit(onSubmit)} className="space-y-6" noValidate>
onSubmit={handleSubmit(onSubmit)}
className="space-y-6"
noValidate
>
<div className="space-y-2"> <div className="space-y-2">
<Label htmlFor="project_name"> <Label htmlFor="project_name">
{t("onboarding.projectName", "프로젝트 이름")} {t("onboarding.projectName", "프로젝트 이름")}
</Label> </Label>
<Input <Input
id="project_name" id="project_name"
placeholder="my_perfume_project" placeholder="my_ontology_project"
aria-invalid={Boolean(errors.project_name)} aria-invalid={Boolean(errors.project_name)}
{...register("project_name")} {...register("project_name")}
/> />
@@ -131,37 +350,27 @@ export default function OnboardingPage() {
</p> </p>
)} )}
<p className="text-xs text-muted-foreground"> <p className="text-xs text-muted-foreground">
{t( , , _, - (2~64)
"onboarding.projectNameHint",
"영문, 숫자, _ , - 만 사용 (2~64자)",
)}
</p> </p>
</div> </div>
<div className="space-y-2"> <div className="space-y-2">
<Label>{t("onboarding.domain", "도메인")}</Label> <Label>{t("onboarding.domain", "도메인")}</Label>
{isError && ( {isError && (
<Card className="border-destructive"> <div className="flex items-center justify-between gap-3 rounded-md border border-destructive px-4 py-3">
<CardContent className="flex items-center justify-between gap-3 py-4"> <div className="flex items-center gap-2 text-sm text-destructive">
<div className="flex items-center gap-2 text-sm text-destructive"> <AlertCircle className="h-4 w-4" />
<AlertCircle className="h-4 w-4" /> <span>{(error as Error).message}</span>
<span>{(error as Error).message}</span> </div>
</div> <Button variant="outline" size="sm" type="button" onClick={() => refetch()}>
<Button {t("common.retry", "다시 시도")}
variant="outline" </Button>
size="sm" </div>
type="button"
onClick={() => refetch()}
>
{t("common.retry", "다시 시도")}
</Button>
</CardContent>
</Card>
)} )}
{domainsLoading && ( {domainsLoading && (
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2"> <div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
{Array.from({ length: 4 }).map((_, i) => ( {Array.from({ length: 4 }).map((_, i) => (
<Skeleton key={i} className="h-24" /> <Skeleton key={i} className="h-28" />
))} ))}
</div> </div>
)} )}
@@ -170,43 +379,56 @@ export default function OnboardingPage() {
{domains.map((d) => { {domains.map((d) => {
const active = selectedDomain === d.domain; const active = selectedDomain === d.domain;
return ( return (
<button <div
key={d.domain} key={d.domain}
type="button"
onClick={() =>
setValue("domain", d.domain, {
shouldValidate: true,
})
}
className={cn( className={cn(
"rounded-lg border bg-card p-4 text-left transition-colors hover:bg-accent/40 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring", "rounded-lg border bg-card transition-colors",
active && "border-primary bg-accent/60", active && "border-primary bg-accent/60",
)} )}
> >
<div className="mb-1 font-semibold capitalize"> <div className="flex items-start gap-2 p-4">
{d.domain} <button
type="button"
onClick={() =>
setValue("domain", d.domain, {
shouldValidate: true,
})
}
className="min-w-0 flex-1 text-left focus-visible:outline-none"
>
<div className="mb-1 flex items-center gap-2">
<span className="font-semibold capitalize">{d.domain}</span>
{d.is_custom && (
<span className="rounded border px-1.5 py-0.5 text-[10px] text-muted-foreground">
custom
</span>
)}
</div>
<div className="text-xs text-muted-foreground">
{d.entity_types.length} · {d.predicates.length} · {d.attribute_count}
</div>
<div className="mt-2 line-clamp-2 text-xs text-muted-foreground/70">
{d.entity_types.slice(0, 6).join(", ")}
{d.entity_types.length > 6 && " ..."}
</div>
</button>
<Button
type="button"
variant="ghost"
size="icon"
aria-label={`${d.domain} 편집`}
onClick={() => setEditorState({ mode: "edit", domain: d })}
>
<Pencil className="h-4 w-4" />
</Button>
</div> </div>
<div className="text-xs text-muted-foreground"> </div>
{t("onboarding.domainSummary", {
entities: d.entity_types.length,
predicates: d.predicates.length,
defaultValue:
"엔티티 {{entities}}종 · 관계 {{predicates}}개",
})}
</div>
<div className="mt-2 line-clamp-2 text-xs text-muted-foreground/70">
{d.entity_types.slice(0, 5).join(", ")}
{d.entity_types.length > 5 && " …"}
</div>
</button>
); );
})} })}
</div> </div>
)} )}
{errors.domain && ( {errors.domain && (
<p className="text-sm text-destructive"> <p className="text-sm text-destructive">{errors.domain.message}</p>
{errors.domain.message}
</p>
)} )}
</div> </div>
@@ -219,19 +441,20 @@ export default function OnboardingPage() {
> >
{t("common.cancel", "취소")} {t("common.cancel", "취소")}
</Button> </Button>
<Button <Button type="submit" disabled={isSubmitting || createProject.isPending}>
type="submit" {createProject.isPending && <Loader2 className="h-4 w-4 animate-spin" />}
disabled={isSubmitting || createProject.isPending}
>
{createProject.isPending && (
<Loader2 className="h-4 w-4 animate-spin" />
)}
{t("onboarding.submit", "프로젝트 만들기")} {t("onboarding.submit", "프로젝트 만들기")}
</Button> </Button>
</div> </div>
</form> </form>
</CardContent> </CardContent>
</Card> </Card>
<DomainEditorDialog
state={editorState}
onClose={() => setEditorState(null)}
onSelect={(domain) => setValue("domain", domain, { shouldValidate: true })}
/>
</div> </div>
); );
} }

File diff suppressed because it is too large Load Diff

View File

@@ -1,30 +1,31 @@
import * as React from "react";
import { useMemo, useState } from "react"; import { useMemo, useState } from "react";
import { useNavigate, useParams } from "react-router-dom"; import { useNavigate, useParams } from "react-router-dom";
import { toast } from "sonner"; import { toast } from "sonner";
import { import {
AlertCircle,
ArrowLeft,
ArrowRight, ArrowRight,
Check, Check,
Eye, ExternalLink,
Filter,
ListChecks, ListChecks,
Search, ShieldCheck,
X, X,
} from "lucide-react"; } from "lucide-react";
import { cn } from "@/lib/utils"; import { Card, CardContent } from "@/components/ui/card";
import {
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
} from "@/components/ui/card";
import { Badge, BadgeProps } from "@/components/ui/badge"; import { Badge, BadgeProps } from "@/components/ui/badge";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input"; import { DataTable, type ColumnDef } from "@/components/ui/data-table";
import { Select } from "@/components/ui/select"; import { Drawer } from "@/components/ui/drawer";
import { Skeleton } from "@/components/ui/skeleton"; import { EmptyState } from "@/components/ui/empty-state";
import { ErrorState } from "@/components/ui/error-state";
import { Tooltip } from "@/components/ui/tooltip";
import {
FilterPanel,
FilterSearch,
FilterSelect,
FilterChips,
type ActiveChip,
} from "@/components/ui/filter-panel";
import { PageHeader } from "@/components/layout/PageHeader";
import { useClaims, useUpdateClaimStatus } from "@/hooks/useClaims"; import { useClaims, useUpdateClaimStatus } from "@/hooks/useClaims";
import { useProject } from "@/hooks/useProjects"; import { useProject } from "@/hooks/useProjects";
import { import {
@@ -50,15 +51,21 @@ function statusVariant(status: string | undefined): BadgeProps["variant"] {
function confidenceBucket(claim: Claim): string { function confidenceBucket(claim: Claim): string {
if (typeof claim.confidence !== "number") return "unknown"; if (typeof claim.confidence !== "number") return "unknown";
if (claim.confidence >= 0.9) return "auto-approve candidate"; if (claim.confidence >= 0.9) return "auto-approve";
if (claim.confidence >= 0.7) return "normal review"; if (claim.confidence >= 0.7) return "normal";
return "priority review"; return "priority";
} }
function claimObject(claim: Claim): string { function claimObjectText(claim: Claim): string {
return humanizeValue(claim.object ?? claim.object_value); return humanizeValue(claim.object ?? claim.object_value);
} }
const STATUS_LABELS: Record<string, string> = {
candidate: "Candidate",
approved: "Approved",
rejected: "Rejected",
};
export default function ReviewPage() { export default function ReviewPage() {
const navigate = useNavigate(); const navigate = useNavigate();
const { projectId } = useParams<{ projectId: string }>(); const { projectId } = useParams<{ projectId: string }>();
@@ -66,24 +73,28 @@ export default function ReviewPage() {
const { data: project } = useProject(projectName); const { data: project } = useProject(projectName);
const claims = useClaims(projectName, { const claims = useClaims(projectName, {
includeCandidates: true, includeCandidates: true,
limit: 300, limit: 500,
}); });
const updateStatus = useUpdateClaimStatus(projectName); const updateStatus = useUpdateClaimStatus(projectName);
const [statusFilter, setStatusFilter] = useState("all");
const [statusFilter, setStatusFilter] = useState<string>("");
const [search, setSearch] = useState(""); const [search, setSearch] = useState("");
const [selectedId, setSelectedId] = useState<string | null>(null); const [selectedId, setSelectedId] = useState<string | null>(null);
const [drawerOpen, setDrawerOpen] = useState(false);
/* --------------------- Derived data --------------------- */
const filteredClaims = useMemo(() => { const filteredClaims = useMemo(() => {
const query = search.trim().toLowerCase(); const query = search.trim().toLowerCase();
return (claims.data ?? []).filter((claim) => { return (claims.data ?? []).filter((claim) => {
if (statusFilter !== "all" && reviewLabel(claim.status) !== statusFilter) { if (statusFilter && reviewLabel(claim.status) !== statusFilter) {
return false; return false;
} }
if (!query) return true; if (!query) return true;
return [ return [
claim.subject, claim.subject,
claim.predicate, claim.predicate,
claimObject(claim), claimObjectText(claim),
claim.evidence_text, claim.evidence_text,
claim.page_url, claim.page_url,
claim.source, claim.source,
@@ -94,11 +105,9 @@ export default function ReviewPage() {
}, [claims.data, search, statusFilter]); }, [claims.data, search, statusFilter]);
const selectedClaim = useMemo(() => { const selectedClaim = useMemo(() => {
return ( if (!selectedId) return null;
filteredClaims.find((claim) => claim.id === selectedId) ?? return claims.data?.find((claim) => claim.id === selectedId) ?? null;
filteredClaims[0] }, [claims.data, selectedId]);
);
}, [filteredClaims, selectedId]);
const counts = useMemo(() => { const counts = useMemo(() => {
const all = claims.data ?? []; const all = claims.data ?? [];
@@ -113,6 +122,8 @@ export default function ReviewPage() {
}; };
}, [claims.data]); }, [claims.data]);
/* --------------------- Actions --------------------- */
const applyStatus = async (claim: Claim, status: string) => { const applyStatus = async (claim: Claim, status: string) => {
try { try {
await updateStatus.mutateAsync({ await updateStatus.mutateAsync({
@@ -123,334 +134,491 @@ export default function ReviewPage() {
? "Rejected from Claim Review" ? "Rejected from Claim Review"
: "Updated from Claim Review", : "Updated from Claim Review",
}); });
toast.success(`Claim ${status}`); toast.success(
status === "validated_claim"
? `Approved · ${humanizeValue(claim.subject)}`
: `Rejected · ${humanizeValue(claim.subject)}`,
);
} catch (error) { } catch (error) {
toast.error((error as Error).message); toast.error((error as Error).message);
} }
}; };
return ( const openDetail = (claim: Claim) => {
<div className="mx-auto max-w-7xl px-6 py-8"> setSelectedId(claim.id);
<div className="mb-6 flex items-center gap-3"> setDrawerOpen(true);
<Button };
variant="ghost"
size="icon" /* --------------------- Filter chips --------------------- */
onClick={() => navigate(`/schema/${projectName}`)}
aria-label="Back" const chips: ActiveChip[] = [];
> if (search)
<ArrowLeft className="h-4 w-4" /> chips.push({
</Button> id: "search",
<div className="flex-1"> label: "Search",
<h1 className="flex items-center gap-2 text-2xl font-bold tracking-tight"> value: `"${search}"`,
<ListChecks className="h-6 w-6 text-primary" /> onClear: () => setSearch(""),
Claim Review });
</h1> if (statusFilter)
<p className="text-sm text-muted-foreground"> chips.push({
{project?.name ?? projectName} · , , id: "status",
</p> label: "Status",
value: STATUS_LABELS[statusFilter] ?? statusFilter,
onClear: () => setStatusFilter(""),
});
const clearAll = () => {
setSearch("");
setStatusFilter("");
};
/* --------------------- Table columns --------------------- */
const columns: ColumnDef<Claim, any>[] = [
{
id: "status",
header: "Status",
accessorFn: (row) => reviewLabel(row.status),
cell: ({ row }) => (
<Badge variant={statusVariant(row.original.status)}>
{reviewLabel(row.original.status)}
</Badge>
),
size: 110,
enablePinning: true,
},
{
id: "subject",
header: "Subject",
accessorFn: (row) => humanizeValue(row.subject),
cell: ({ row }) => (
<span className="line-clamp-2 break-words font-medium text-foreground">
{humanizeValue(row.original.subject)}
</span>
),
size: 220,
},
{
id: "predicate",
header: "Predicate",
accessorFn: (row) => row.predicate,
cell: ({ row }) => (
<Badge variant="secondary" className="font-mono">
{row.original.predicate}
</Badge>
),
size: 160,
},
{
id: "object",
header: "Object",
accessorFn: (row) => claimObjectText(row),
cell: ({ row }) => (
<span className="line-clamp-2 break-words text-foreground">
{claimObjectText(row.original)}
</span>
),
size: 280,
enableSorting: false,
},
{
id: "confidence",
header: "Confidence",
accessorFn: (row) => row.confidence ?? 0,
cell: ({ row }) => <ConfidenceBar value={row.original.confidence} />,
size: 140,
meta: { align: "right" },
},
{
id: "bucket",
header: "Bucket",
accessorFn: (row) => confidenceBucket(row),
cell: ({ row }) => {
const b = confidenceBucket(row.original);
const tone =
b === "auto-approve"
? "pill-success"
: b === "priority"
? "pill-warning"
: "pill-info";
return <span className={tone}>{b}</span>;
},
size: 130,
},
{
id: "source",
header: "Source",
accessorFn: (row) => row.source ?? "",
cell: ({ row }) => (
<span className="text-xs text-muted-foreground">
{humanizeValue(row.original.source)}
</span>
),
size: 120,
},
{
id: "evidence",
header: "Evidence",
accessorFn: (row) => (row.evidence_text ? 1 : 0),
cell: ({ row }) =>
row.original.evidence_text ? (
<span className="text-xs font-medium text-success">yes</span>
) : (
<span className="text-xs font-medium text-warning">missing</span>
),
size: 90,
meta: { align: "center" },
},
{
id: "actions",
header: "",
enableSorting: false,
enableHiding: false,
enableResizing: false,
cell: ({ row }) => (
<div className="flex items-center justify-end gap-1">
<Tooltip content="Approve" shortcut="A">
<Button
variant="ghost"
size="icon"
onClick={(e) => {
e.stopPropagation();
applyStatus(row.original, "validated_claim");
}}
disabled={updateStatus.isPending}
className="h-7 w-7"
>
<Check className="h-3.5 w-3.5 text-success" />
</Button>
</Tooltip>
<Tooltip content="Reject" shortcut="X">
<Button
variant="ghost"
size="icon"
onClick={(e) => {
e.stopPropagation();
applyStatus(row.original, "rejected");
}}
disabled={updateStatus.isPending}
className="h-7 w-7"
>
<X className="h-3.5 w-3.5 text-danger" />
</Button>
</Tooltip>
</div> </div>
<Button onClick={() => navigate(`/quality/${projectName}`)}> ),
Quality Inspector size: 90,
</Button> meta: { align: "right" },
},
];
/* --------------------- Render --------------------- */
return (
<>
<PageHeader
icon={ListChecks}
breadcrumbs={[
{ label: "Dashboard", to: "/" },
{ label: projectName, to: `/sources/${projectName}` },
{ label: "Claim Review" },
]}
title="Claim Review"
description={
project &&
`${project.name} · 후보·승인·반려 상태 분리 검토 · ${counts.total.toLocaleString()} claims`
}
actions={
<Button onClick={() => navigate(`/quality/${projectName}`)}>
<ShieldCheck className="h-4 w-4" />
Quality Inspector
</Button>
}
/>
<div className="mx-auto max-w-7xl px-6 py-6">
{claims.isError && (
<ErrorState
className="mb-4"
inline
severity="error"
title="Failed to load claims"
description={claims.error as Error}
onRetry={() => claims.refetch()}
/>
)}
{/* Metrics */}
<div className="mb-4 grid gap-3 sm:grid-cols-2 lg:grid-cols-4">
<Metric label="Total" value={counts.total} tone="muted" />
<Metric label="Candidate" value={counts.candidate} tone="warning" />
<Metric label="Approved" value={counts.approved} tone="success" />
<Metric label="Rejected" value={counts.rejected} tone="danger" />
</div>
{/* Filter panel */}
<FilterPanel
className="mb-4"
chips={chips.length > 0 ? <FilterChips chips={chips} /> : undefined}
onClearAll={chips.length > 0 ? clearAll : undefined}
>
<FilterSearch
value={search}
onChange={setSearch}
placeholder="Search subject, predicate, object, evidence…"
/>
<FilterSelect
label="Status"
value={statusFilter}
onChange={setStatusFilter}
placeholder="All statuses"
options={[
{ value: "candidate", label: "Candidate", hint: counts.candidate, swatch: "hsl(var(--warning))" },
{ value: "approved", label: "Approved", hint: counts.approved, swatch: "hsl(var(--success))" },
{ value: "rejected", label: "Rejected", hint: counts.rejected, swatch: "hsl(var(--danger))" },
]}
/>
</FilterPanel>
<DataTable<Claim>
tableId="review"
columns={columns}
data={filteredClaims}
getRowId={(row) => row.id}
loading={claims.isLoading}
loadingRows={8}
height={640}
onRowClick={openDetail}
rowToneAccessor={(row) => {
const r = reviewLabel(row.status);
if (r === "approved") return "success";
if (r === "rejected") return "danger";
return undefined;
}}
empty={
<EmptyState
icon={ListChecks}
title="조건에 맞는 클레임이 없습니다"
description="필터를 조정하거나 클레임을 추가하세요."
primaryAction={
chips.length > 0 ? (
<Button variant="outline" onClick={clearAll}>
Clear filters
</Button>
) : undefined
}
/>
}
/>
</div> </div>
{claims.isError && ( {/* Detail Drawer */}
<Card className="mb-6 border-destructive"> <Drawer
<CardContent className="flex items-center gap-2 py-4 text-sm text-destructive"> open={drawerOpen && !!selectedClaim}
<AlertCircle className="h-4 w-4" /> onOpenChange={(open) => {
{(claims.error as Error).message} setDrawerOpen(open);
</CardContent> if (!open) setSelectedId(null);
</Card> }}
)} size="lg"
title={
<div className="mb-6 grid gap-4 md:grid-cols-4"> selectedClaim
<Metric label="Total" value={counts.total} /> ? `Claim #${selectedClaim.id} · ${selectedClaim.predicate}`
<Metric label="Candidate" value={counts.candidate} /> : ""
<Metric label="Approved" value={counts.approved} /> }
<Metric label="Rejected" value={counts.rejected} /> description="근거, 출처, 검증 결과, 히스토리"
</div> headerExtra={
selectedClaim ? (
<div className="grid gap-6 xl:grid-cols-[minmax(0,1fr)_420px]"> <Badge variant={statusVariant(selectedClaim.status)}>
<Card className="min-w-0"> {reviewLabel(selectedClaim.status)}
<CardHeader> </Badge>
<div className="flex flex-wrap items-end justify-between gap-3"> ) : null
<div className="min-w-0"> }
<CardTitle>Review Queue</CardTitle> footer={
<CardDescription> selectedClaim ? (
, , , . <>
</CardDescription> <Button
variant="outline"
onClick={() => applyStatus(selectedClaim, "rejected")}
disabled={updateStatus.isPending}
>
<X className="h-4 w-4" />
Reject
</Button>
<Button
onClick={() => applyStatus(selectedClaim, "validated_claim")}
disabled={updateStatus.isPending}
>
<Check className="h-4 w-4" />
Approve
</Button>
</>
) : null
}
>
{selectedClaim && (
<div className="space-y-5">
{/* Triple summary */}
<div className="rounded-md border border-border bg-background-subtle p-3">
<div className="flex flex-wrap items-center gap-2 text-sm">
<span className="font-medium text-foreground">
{humanizeValue(selectedClaim.subject)}
</span>
{selectedClaim.subject_type && (
<span className="pill-muted">
{selectedClaim.subject_type}
</span>
)}
<ArrowRight className="h-3.5 w-3.5 text-muted-foreground" />
<Badge variant="secondary" className="font-mono">
{selectedClaim.predicate}
</Badge>
<ArrowRight className="h-3.5 w-3.5 text-muted-foreground" />
<span className="font-medium text-foreground">
{claimObjectText(selectedClaim)}
</span>
</div> </div>
<div className="flex flex-wrap gap-2"> <div className="mt-2 flex flex-wrap gap-3 text-2xs text-muted-foreground">
<div className="relative"> <span>
<Search className="pointer-events-none absolute left-3 top-2.5 h-4 w-4 text-muted-foreground" /> Confidence{" "}
<Input <span className="font-medium text-foreground tabular-nums">
value={search} {formatPercent(selectedClaim.confidence)}
onChange={(event) => setSearch(event.target.value)} </span>
className="w-48 pl-9 sm:w-56" </span>
placeholder="Search claims" <span>
/> Bucket{" "}
</div> <span className="font-medium text-foreground">
<div className="relative"> {confidenceBucket(selectedClaim)}
<Filter className="pointer-events-none absolute left-3 top-2.5 h-4 w-4 text-muted-foreground" /> </span>
<Select </span>
value={statusFilter} {selectedClaim.review_required && (
onChange={(event) => setStatusFilter(event.target.value)} <span className="pill-warning">review required</span>
className="w-32 pl-9 sm:w-40" )}
>
<option value="all">All</option>
<option value="candidate">Candidate</option>
<option value="approved">Approved</option>
<option value="rejected">Rejected</option>
</Select>
</div>
</div> </div>
</div> </div>
</CardHeader>
<CardContent>
{claims.isLoading && (
<div className="space-y-3">
{Array.from({ length: 5 }).map((_, index) => (
<Skeleton key={index} className="h-24" />
))}
</div>
)}
{!claims.isLoading && filteredClaims.length === 0 && (
<p className="py-10 text-center text-sm text-muted-foreground">
.
</p>
)}
<div className="space-y-3">
{filteredClaims.map((claim) => {
const isSelected = selectedClaim?.id === claim.id;
return (
<article
key={claim.id}
className={cn(
"rounded-md border bg-surface transition-colors",
isSelected
? "border-brand-500 ring-1 ring-brand-500/30"
: "border-border hover:border-border-strong",
)}
>
<button
type="button"
onClick={() => setSelectedId(claim.id)}
className="block w-full px-4 pt-4 pb-3 text-left"
>
<div className="mb-2 flex flex-wrap items-center gap-1.5">
<Badge variant={statusVariant(claim.status)}>
{reviewLabel(claim.status)}
</Badge>
<Badge variant="secondary">{claim.predicate}</Badge>
<span className="text-2xs text-muted-foreground">
{confidenceBucket(claim)}
</span>
</div>
<div className="flex flex-wrap items-baseline gap-1.5 text-sm">
<span className="break-all font-medium text-foreground">
{humanizeValue(claim.subject)}
</span>
<ArrowRight className="h-3 w-3 flex-shrink-0 text-muted-foreground" />
<span className="break-all font-medium text-foreground">
{claimObject(claim)}
</span>
</div>
<div className="mt-2 flex flex-wrap gap-x-3 gap-y-1 text-2xs text-muted-foreground">
<span>
Confidence{" "}
<span className="font-medium text-foreground tabular-nums">
{formatPercent(claim.confidence)}
</span>
</span>
<span>
Source{" "}
<span className="font-medium text-foreground">
{humanizeValue(claim.source)}
</span>
</span>
<span>
Evidence{" "}
<span
className={
claim.evidence_text
? "font-medium text-success"
: "font-medium text-warning"
}
>
{claim.evidence_text ? "yes" : "missing"}
</span>
</span>
<span>
Method{" "}
<span className="font-medium text-foreground">
{humanizeValue(claim.extraction_method)}
</span>
</span>
</div>
</button>
<div className="flex flex-wrap items-center justify-end gap-1.5 border-t border-border bg-background-subtle px-3 py-2">
<Button
variant="ghost"
size="sm"
onClick={() => setSelectedId(claim.id)}
>
<Eye className="h-3.5 w-3.5" />
Detail
</Button>
<Button
variant="outline"
size="sm"
onClick={() => applyStatus(claim, "validated_claim")}
disabled={updateStatus.isPending}
>
<Check className="h-3.5 w-3.5" />
Approve
</Button>
<Button
variant="outline"
size="sm"
onClick={() => applyStatus(claim, "rejected")}
disabled={updateStatus.isPending}
>
<X className="h-3.5 w-3.5" />
Reject
</Button>
</div>
</article>
);
})}
</div>
</CardContent>
</Card>
<aside className="xl:sticky xl:top-4 xl:self-start"> {/* Detail grid */}
<Card> <div className="grid gap-3 sm:grid-cols-2">
<CardHeader> <DetailRow label="Source" value={selectedClaim.source} />
<CardTitle>Claim Detail</CardTitle> <DetailRow
<CardDescription> label="Page Type"
, , , value={selectedClaim.page_type}
</CardDescription> />
</CardHeader> <DetailRow
<CardContent> label="Extraction Method"
{!selectedClaim && ( value={selectedClaim.extraction_method}
<p className="py-8 text-center text-sm text-muted-foreground"> />
. <DetailRow
</p> label="Validation"
)} value={
{selectedClaim && ( selectedClaim.validation_status ??
<div className="space-y-4"> selectedClaim.graph_merge_status
<div className="flex flex-wrap gap-2"> }
<Badge variant={statusVariant(selectedClaim.status)}> />
{reviewLabel(selectedClaim.status)} {selectedClaim.page_url && (
</Badge> <DetailRow
<Badge variant="outline"> label="Source URL"
{formatPercent(selectedClaim.confidence)} value={
</Badge> <a
{selectedClaim.review_required && ( href={selectedClaim.page_url}
<Badge variant="warning">review required</Badge> target="_blank"
)} rel="noopener noreferrer"
</div> className="inline-flex items-center gap-1 text-brand-600 hover:underline dark:text-brand-300"
<DetailRow label="Subject" value={selectedClaim.subject} />
<DetailRow label="Subject Type" value={selectedClaim.subject_type} />
<DetailRow label="Predicate" value={selectedClaim.predicate} />
<DetailRow
label="Object"
value={selectedClaim.object ?? selectedClaim.object_value}
/>
<DetailRow label="Source" value={selectedClaim.source} />
<DetailRow label="Source URL" value={selectedClaim.page_url} />
<DetailRow
label="Page Type"
value={selectedClaim.page_type}
/>
<DetailRow
label="Created By"
value={selectedClaim.extraction_method}
/>
<DetailRow
label="Validation"
value={
selectedClaim.validation_status ??
selectedClaim.graph_merge_status
}
/>
{selectedClaim.graph_merge_reason && (
<DetailRow
label="Graph Reason"
value={selectedClaim.graph_merge_reason}
/>
)}
{selectedClaim.evidence_text && (
<section>
<h3 className="mb-2 text-sm font-medium">Evidence Text</h3>
<div className="rounded-md bg-yellow-50 px-3 py-2 text-sm leading-relaxed text-yellow-950">
{selectedClaim.evidence_text}
</div>
</section>
)}
{selectedClaim.confidence_breakdown && (
<section>
<h3 className="mb-2 text-sm font-medium">
Confidence Breakdown
</h3>
<pre className="max-h-44 overflow-auto rounded-md bg-secondary/30 p-3 text-xs">
{JSON.stringify(
selectedClaim.confidence_breakdown,
null,
2,
)}
</pre>
</section>
)}
{selectedClaim.source_history &&
selectedClaim.source_history.length > 0 && (
<section>
<h3 className="mb-2 text-sm font-medium">
Source History
</h3>
<pre className="max-h-44 overflow-auto rounded-md bg-secondary/30 p-3 text-xs">
{JSON.stringify(selectedClaim.source_history, null, 2)}
</pre>
</section>
)}
<div className="grid grid-cols-2 gap-2 pt-2">
<Button
onClick={() =>
applyStatus(selectedClaim, "validated_claim")
}
disabled={updateStatus.isPending}
> >
<Check className="h-4 w-4" /> <ExternalLink className="h-3 w-3" />
Approve <span className="max-w-[240px] truncate">
</Button> {selectedClaim.page_url}
<Button </span>
variant="outline" </a>
onClick={() => applyStatus(selectedClaim, "rejected")} }
disabled={updateStatus.isPending} full
> />
<X className="h-4 w-4" />
Reject
</Button>
</div>
<div className="text-xs text-muted-foreground">
Last seen {formatDateTime(selectedClaim.last_seen_at)}
</div>
</div>
)} )}
</CardContent> {selectedClaim.graph_merge_reason && (
</Card> <DetailRow
</aside> label="Graph Reason"
</div> value={selectedClaim.graph_merge_reason}
</div> full
/>
)}
</div>
{/* Evidence */}
{selectedClaim.evidence_text && (
<section>
<h3 className="mb-1.5 text-2xs font-semibold uppercase tracking-wider text-muted-foreground">
Evidence Text
</h3>
<blockquote className="rounded-md border-l-4 border-warning bg-warning-subtle px-3 py-2 text-sm leading-relaxed text-foreground">
{selectedClaim.evidence_text}
</blockquote>
</section>
)}
{/* Confidence breakdown */}
{selectedClaim.confidence_breakdown && (
<section>
<h3 className="mb-1.5 text-2xs font-semibold uppercase tracking-wider text-muted-foreground">
Confidence Breakdown
</h3>
<pre className="max-h-44 overflow-auto rounded-md border border-border bg-background-subtle p-3 text-2xs">
{JSON.stringify(
selectedClaim.confidence_breakdown,
null,
2,
)}
</pre>
</section>
)}
{/* Source history */}
{selectedClaim.source_history &&
selectedClaim.source_history.length > 0 && (
<section>
<h3 className="mb-1.5 text-2xs font-semibold uppercase tracking-wider text-muted-foreground">
Source History ({selectedClaim.source_history.length})
</h3>
<pre className="max-h-44 overflow-auto rounded-md border border-border bg-background-subtle p-3 text-2xs">
{JSON.stringify(selectedClaim.source_history, null, 2)}
</pre>
</section>
)}
<div className="border-t border-border pt-3 text-2xs text-muted-foreground">
Last seen {formatDateTime(selectedClaim.last_seen_at)}
</div>
</div>
)}
</Drawer>
</>
); );
} }
function Metric({ label, value }: { label: string; value: number }) { /* ============================================================
* Sub-components
* ========================================================== */
const METRIC_TONE: Record<string, string> = {
muted: "border-border",
warning: "border-warning-border bg-warning-subtle/30",
success: "border-success-border bg-success-subtle/30",
danger: "border-danger-border bg-danger-subtle/30",
};
function Metric({
label,
value,
tone = "muted",
}: {
label: string;
value: number;
tone?: keyof typeof METRIC_TONE;
}) {
return ( return (
<Card> <Card className={`border ${METRIC_TONE[tone]}`}>
<CardContent className="py-4"> <CardContent className="py-3">
<div className="text-xs text-muted-foreground">{label}</div> <div className="text-2xs uppercase tracking-wider text-muted-foreground">
<div className="mt-1 text-2xl font-semibold"> {label}
</div>
<div className="mt-1 text-2xl font-semibold tabular-nums text-foreground">
{value.toLocaleString()} {value.toLocaleString()}
</div> </div>
</CardContent> </CardContent>
@@ -458,11 +626,48 @@ function Metric({ label, value }: { label: string; value: number }) {
); );
} }
function DetailRow({ label, value }: { label: string; value: unknown }) { function ConfidenceBar({ value }: { value: number | undefined }) {
if (typeof value !== "number") {
return <span className="text-xs text-muted-foreground/60"></span>;
}
const pct = Math.round(value * 100);
const tone =
value >= 0.9
? "bg-success"
: value >= 0.7
? "bg-info"
: value >= 0.5
? "bg-warning"
: "bg-danger";
return ( return (
<div className="rounded-md border bg-background px-3 py-2 text-sm"> <div className="flex items-center justify-end gap-2">
<div className="text-xs text-muted-foreground">{label}</div> <div className="h-1.5 w-14 overflow-hidden rounded-full bg-muted">
<div className="mt-1 break-words font-medium">{humanizeValue(value)}</div> <div className={`h-full ${tone}`} style={{ width: `${pct}%` }} />
</div>
<span className="text-xs tabular-nums text-foreground">{pct}%</span>
</div>
);
}
function DetailRow({
label,
value,
full = false,
}: {
label: string;
value: unknown;
full?: boolean;
}) {
return (
<div
className={`rounded-md border border-border bg-background-subtle px-3 py-2 ${full ? "sm:col-span-2" : ""}`}
>
<div className="text-2xs uppercase tracking-wider text-muted-foreground">
{label}
</div>
<div className="mt-1 break-words text-sm font-medium text-foreground">
{React.isValidElement(value) ? value : humanizeValue(value)}
</div>
</div> </div>
); );
} }