diff --git a/ontology_platform/crawler_platform/app/api/routes.py b/ontology_platform/crawler_platform/app/api/routes.py index 8b09b30..56e9e64 100644 --- a/ontology_platform/crawler_platform/app/api/routes.py +++ b/ontology_platform/crawler_platform/app/api/routes.py @@ -3,6 +3,7 @@ from __future__ import annotations import csv import io import json +import re from dataclasses import asdict from typing import Any @@ -28,7 +29,8 @@ from crawler_platform.app.core.database.repository import ( from crawler_platform.app.core.database.session import session_scope from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models from crawler_platform.app.core.extractor.factory import extractor_for_domain -from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, ontology_for_domain +from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, Ontology, ontology_for_domain +from crawler_platform.app.core.ontology.domain_discovery import DomainDiscoveryService from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector from crawler_platform.app.core.ontology.mapper import ontology_to_dict from crawler_platform.app.core.ontology.registry import OntologyRegistry @@ -38,6 +40,9 @@ from crawler_platform.app.core.research.graph_research_loop import GraphResearch from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload +DOMAIN_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{1,79}$") + + class CrawlRequest(BaseModel): config_path: str source_name: str @@ -149,6 +154,41 @@ class CreateProjectInlineRequest(BaseModel): ) +class DomainDefinitionRequest(BaseModel): + domain: str + description: str | None = None + entity_types: list[str] = Field(default_factory=list) + predicates: list[str] = Field(default_factory=list) + attributes: list[str] = Field(default_factory=list) + aliases: dict[str, str] = Field(default_factory=dict) + + +class DomainDefinitionUpdateRequest(BaseModel): + description: str | None = None + entity_types: list[str] = Field(default_factory=list) + predicates: list[str] = Field(default_factory=list) + attributes: list[str] = Field(default_factory=list) + aliases: dict[str, str] = Field(default_factory=dict) + + +class DomainDiscoveryRequest(BaseModel): + seed_urls: list[str] = Field(default_factory=list) + max_pages: int = 20 + max_depth: int = 1 + same_domain_only: bool = True + fetcher: str = "requests" + respect_robots_txt: bool = False + force_recrawl: bool = False + + +class DomainCandidateApplyRequest(BaseModel): + candidate_ids: list[int] = Field(default_factory=list) + + +class DomainCandidateStatusRequest(BaseModel): + status: str + + class ResetProjectRequest(BaseModel): config_path: str project_name: str | None = None @@ -165,6 +205,250 @@ def source_model_to_config(source: models.Source) -> SourceConfig: ) +def normalize_domain_name(domain: str) -> str: + normalized = domain.strip().lower() + if not DOMAIN_NAME_PATTERN.match(normalized): + raise HTTPException( + status_code=400, + detail="Domain must be 2-80 chars using lowercase letters, numbers, underscore, or hyphen.", + ) + return normalized + + +def clean_string_list(values: list[str]) -> list[str]: + cleaned: list[str] = [] + seen: set[str] = set() + for value in values: + item = value.strip() + if not item or item in seen: + continue + cleaned.append(item) + seen.add(item) + return cleaned + + +def clean_aliases(values: dict[str, str]) -> dict[str, str]: + return { + key.strip(): value.strip() + for key, value in values.items() + if key.strip() and value.strip() + } + + +def domain_row_to_ontology(row: models.DomainDefinition): + return Ontology( + domain=row.domain, + entity_types=clean_string_list(list(row.entity_types or [])), + predicates=clean_string_list(list(row.predicates or [])), + attributes=clean_string_list(list(row.attributes or [])), + aliases=clean_aliases(dict(row.aliases or {})), + ) + + +def domain_summary_payload(ont, *, is_builtin: bool, is_custom: bool, description: str | None = None): + return { + "domain": ont.domain, + "description": description, + "entity_types": list(ont.entity_types), + "predicates": list(ont.predicates), + "attribute_count": len(ont.attributes), + "attributes": list(ont.attributes), + "aliases": dict(ont.aliases), + "is_builtin": is_builtin, + "is_custom": is_custom, + } + + +def upsert_domain_definition( + session, + domain: str, + payload: DomainDefinitionUpdateRequest, +) -> models.DomainDefinition: + normalized_domain = normalize_domain_name(domain) + row = session.scalar( + select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain) + ) + if row is None: + row = models.DomainDefinition(domain=normalized_domain) + session.add(row) + session.flush() + row.description = payload.description.strip() if payload.description else None + row.entity_types = clean_string_list(payload.entity_types) + row.predicates = clean_string_list(payload.predicates) + row.attributes = clean_string_list(payload.attributes) + row.aliases = clean_aliases(payload.aliases) + row.is_builtin_override = normalized_domain in DOMAIN_ONTOLOGIES + row.updated_at = models.utcnow() + return row + + +def ensure_domain_definition(session, domain: str) -> models.DomainDefinition: + normalized_domain = normalize_domain_name(domain) + row = session.scalar( + select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain) + ) + if row is not None: + return row + + base = DOMAIN_ONTOLOGIES.get(normalized_domain) + row = models.DomainDefinition( + domain=normalized_domain, + entity_types=list(base.entity_types) if base else [], + predicates=list(base.predicates) if base else [], + attributes=list(base.attributes) if base else [], + aliases=dict(base.aliases) if base else {}, + is_builtin_override=normalized_domain in DOMAIN_ONTOLOGIES, + ) + session.add(row) + session.flush() + return row + + +def ontology_for_domain_from_db(session, domain: str): + normalized_domain = normalize_domain_name(domain) + row = session.scalar( + select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain) + ) + if row is not None: + return domain_row_to_ontology(row) + return ontology_for_domain(normalized_domain) + + +def clean_seed_urls(values: list[str]) -> list[str]: + cleaned: list[str] = [] + seen: set[str] = set() + for value in values: + url = value.strip() + if not url or url in seen: + continue + if not (url.startswith("http://") or url.startswith("https://") or url.startswith("file://")): + raise HTTPException(status_code=400, detail=f"Unsupported URL: {url}") + cleaned.append(url) + seen.add(url) + if not cleaned: + raise HTTPException(status_code=400, detail="At least one reference URL is required.") + return cleaned + + +def domain_discovery_job_payload(job: models.DomainDiscoveryJob) -> dict[str, Any]: + return { + "job_id": job.id, + "domain": job.domain, + "status": job.status, + "seed_urls": list(job.seed_urls or []), + "max_pages": job.max_pages, + "max_depth": job.max_depth, + "same_domain_only": job.same_domain_only, + "fetcher": job.fetcher, + "respect_robots_txt": job.respect_robots_txt, + "error": job.error, + "progress": job.progress or {}, + "result_summary": job.result_summary or {}, + "created_at": job.created_at.isoformat() if job.created_at else None, + "started_at": job.started_at.isoformat() if job.started_at else None, + "finished_at": job.finished_at.isoformat() if job.finished_at else None, + } + + +def domain_reference_source_payload(row: models.DomainReferenceSource) -> dict[str, Any]: + return { + "id": row.id, + "domain": row.domain, + "url": row.url, + "label": row.label, + "status": row.status, + "last_crawled_at": row.last_crawled_at.isoformat() if row.last_crawled_at else None, + "metadata": row.metadata_json or {}, + "created_at": row.created_at.isoformat() if row.created_at else None, + "updated_at": row.updated_at.isoformat() if row.updated_at else None, + } + + +def already_crawled_reference_urls(session, domain: str, seed_urls: list[str]) -> set[str]: + if not seed_urls: + return set() + rows = session.scalars( + select(models.DomainReferenceSource).where( + models.DomainReferenceSource.domain == domain, + models.DomainReferenceSource.url.in_(seed_urls), + models.DomainReferenceSource.status == "active", + models.DomainReferenceSource.last_crawled_at.is_not(None), + ) + ).all() + return {row.url for row in rows} + + +def domain_candidate_payload(row: models.DomainSchemaCandidate, evidence_rows: list[models.DomainCandidateEvidence]) -> dict[str, Any]: + return { + "id": row.id, + "domain": row.domain, + "job_id": row.job_id, + "candidate_type": row.candidate_type, + "name": row.name, + "description": row.description, + "confidence": row.confidence, + "occurrence_count": row.occurrence_count, + "status": row.status, + "metadata": row.metadata_json or {}, + "created_at": row.created_at.isoformat() if row.created_at else None, + "updated_at": row.updated_at.isoformat() if row.updated_at else None, + "evidence": [ + { + "id": evidence.id, + "job_id": evidence.job_id, + "url": evidence.url, + "title": evidence.title, + "snippet": evidence.snippet, + "metadata": evidence.metadata_json or {}, + "created_at": evidence.created_at.isoformat() if evidence.created_at else None, + } + for evidence in evidence_rows + ], + } + + +def run_domain_discovery_job(database_url: str, job_id: int) -> None: + try: + with session_scope(database_url) as session: + DomainDiscoveryService(session).run_job(job_id) + except Exception as exc: + with session_scope(database_url) as session: + job = session.get(models.DomainDiscoveryJob, job_id) + if job is not None: + job.status = "failed" + job.error = str(exc) + job.finished_at = models.utcnow() + progress = dict(job.progress or {}) + progress["errors"] = [*progress.get("errors", []), str(exc)] + job.progress = progress + + +def recover_interrupted_domain_discovery_jobs(database_url: str) -> None: + """Mark domain discovery jobs that cannot survive a server restart.""" + with session_scope(database_url) as session: + rows = session.scalars( + select(models.DomainDiscoveryJob).where( + models.DomainDiscoveryJob.status.in_( + ["pending", "running", "cancel_requested"] + ) + ) + ).all() + for job in rows: + progress = dict(job.progress or {}) + errors = list(progress.get("errors") or []) + if job.status == "cancel_requested": + job.status = "canceled" + progress["completion_reason"] = "canceled" + else: + job.status = "failed" + job.error = "Server restarted before this background analysis finished." + progress["completion_reason"] = "interrupted" + errors.append(job.error) + progress["errors"] = errors[-10:] + job.progress = progress + job.finished_at = models.utcnow() + + def project_config_from_project_row(session, project: models.Project) -> ProjectConfig: config_dict = dict(project.config or {}) if not config_dict: @@ -195,6 +479,32 @@ def project_config_to_dict(config: ProjectConfig) -> dict[str, Any]: } +def delete_project_data(session, project_id: int, include_project: bool = False) -> dict[str, int]: + repo = KnowledgeRepository(session) + deleted = repo.reset_project_runtime_data(project_id) + for table_model in [ + models.OntologyProposal, + models.OntologyTriple, + models.OntologyRelationType, + models.OntologyEntityType, + models.Source, + ]: + count = ( + session.query(table_model) + .filter(table_model.project_id == project_id) + .delete(synchronize_session=False) + ) + deleted[table_model.__tablename__] = deleted.get(table_model.__tablename__, 0) + int(count or 0) + if include_project: + project_count = ( + session.query(models.Project) + .filter(models.Project.id == project_id) + .delete(synchronize_session=False) + ) + deleted[models.Project.__tablename__] = int(project_count or 0) + return deleted + + class CreateEntityRequest(BaseModel): entity_type: str name: str @@ -485,6 +795,8 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A def register_routes(app, database_url: str) -> None: + recover_interrupted_domain_discovery_jobs(database_url) + @app.get("/health") def health(): return {"ok": True} @@ -520,16 +832,298 @@ def register_routes(app, database_url: str) -> None: @app.get("/domains") def list_domains(): - """Available pre-defined ontology domains for project creation.""" - return [ - { - "domain": ont.domain, - "entity_types": list(ont.entity_types), - "predicates": list(ont.predicates), - "attribute_count": len(ont.attributes), + """Available ontology domains for project creation.""" + with session_scope(database_url) as session: + rows = session.scalars(select(models.DomainDefinition)).all() + custom_by_domain = {row.domain: row for row in rows} + payloads = [] + for domain, built_in in DOMAIN_ONTOLOGIES.items(): + row = custom_by_domain.pop(domain, None) + if row is not None: + payloads.append( + domain_summary_payload( + domain_row_to_ontology(row), + is_builtin=True, + is_custom=True, + description=row.description, + ) + ) + else: + payloads.append( + domain_summary_payload( + built_in, + is_builtin=True, + is_custom=False, + ) + ) + for row in sorted(custom_by_domain.values(), key=lambda item: item.domain): + payloads.append( + domain_summary_payload( + domain_row_to_ontology(row), + is_builtin=False, + is_custom=True, + description=row.description, + ) + ) + return payloads + + @app.post("/domains") + def create_domain(request: DomainDefinitionRequest): + domain = normalize_domain_name(request.domain) + with session_scope(database_url) as session: + existing = session.scalar( + select(models.DomainDefinition).where(models.DomainDefinition.domain == domain) + ) + if existing is not None or domain in DOMAIN_ONTOLOGIES: + raise HTTPException(status_code=409, detail=f"Domain '{domain}' already exists.") + row = upsert_domain_definition( + session, + domain, + DomainDefinitionUpdateRequest( + description=request.description, + entity_types=request.entity_types, + predicates=request.predicates, + attributes=request.attributes, + aliases=request.aliases, + ), + ) + return domain_summary_payload( + domain_row_to_ontology(row), + is_builtin=domain in DOMAIN_ONTOLOGIES, + is_custom=True, + description=row.description, + ) + + @app.put("/domains/{domain}") + def update_domain(domain: str, request: DomainDefinitionUpdateRequest): + with session_scope(database_url) as session: + row = upsert_domain_definition(session, domain, request) + return domain_summary_payload( + domain_row_to_ontology(row), + is_builtin=row.domain in DOMAIN_ONTOLOGIES, + is_custom=True, + description=row.description, + ) + + @app.get("/domains/{domain}/discovery/jobs") + def list_domain_discovery_jobs(domain: str, limit: int = 20): + normalized_domain = normalize_domain_name(domain) + with session_scope(database_url) as session: + rows = session.scalars( + select(models.DomainDiscoveryJob) + .where(models.DomainDiscoveryJob.domain == normalized_domain) + .order_by(models.DomainDiscoveryJob.created_at.desc()) + .limit(max(min(limit, 100), 1)) + ).all() + return [domain_discovery_job_payload(row) for row in rows] + + @app.get("/domains/{domain}/reference-sources") + def list_domain_reference_sources(domain: str): + normalized_domain = normalize_domain_name(domain) + with session_scope(database_url) as session: + rows = session.scalars( + select(models.DomainReferenceSource) + .where(models.DomainReferenceSource.domain == normalized_domain) + .order_by(models.DomainReferenceSource.updated_at.desc()) + ).all() + return [domain_reference_source_payload(row) for row in rows] + + @app.post("/domains/{domain}/discovery/jobs") + def create_domain_discovery_job( + domain: str, + request: DomainDiscoveryRequest, + background_tasks: BackgroundTasks, + ): + normalized_domain = normalize_domain_name(domain) + seed_urls = clean_seed_urls(request.seed_urls) + with session_scope(database_url) as session: + ensure_domain_definition(session, normalized_domain) + skipped_seed_urls: list[str] = [] + if not request.force_recrawl: + skipped = already_crawled_reference_urls(session, normalized_domain, seed_urls) + skipped_seed_urls = [url for url in seed_urls if url in skipped] + seed_urls = [url for url in seed_urls if url not in skipped] + + if not seed_urls: + job = models.DomainDiscoveryJob( + domain=normalized_domain, + status="completed", + seed_urls=[], + max_pages=max(min(request.max_pages, 200), 1), + max_depth=max(min(request.max_depth, 5), 0), + same_domain_only=request.same_domain_only, + fetcher=request.fetcher if request.fetcher in {"requests", "playwright", "browser"} else "requests", + respect_robots_txt=request.respect_robots_txt, + progress={ + "visited_count": 0, + "queued_count": 0, + "candidate_count": 0, + "observation_count": 0, + "max_pages": max(min(request.max_pages, 200), 1), + "skipped_seed_urls": skipped_seed_urls, + "errors": [], + "pages": [], + }, + result_summary={ + "visited_count": 0, + "candidate_count": 0, + "skipped_seed_urls": skipped_seed_urls, + }, + finished_at=models.utcnow(), + ) + session.add(job) + session.flush() + return domain_discovery_job_payload(job) + + job = models.DomainDiscoveryJob( + domain=normalized_domain, + status="pending", + seed_urls=seed_urls, + max_pages=max(min(request.max_pages, 200), 1), + max_depth=max(min(request.max_depth, 5), 0), + same_domain_only=request.same_domain_only, + fetcher=request.fetcher if request.fetcher in {"requests", "playwright", "browser"} else "requests", + respect_robots_txt=request.respect_robots_txt, + progress={ + "visited_count": 0, + "queued_count": len(seed_urls), + "candidate_count": 0, + "observation_count": 0, + "max_pages": max(min(request.max_pages, 200), 1), + "skipped_seed_urls": skipped_seed_urls, + "errors": [], + "pages": [], + }, + ) + session.add(job) + session.flush() + response = domain_discovery_job_payload(job) + background_tasks.add_task(run_domain_discovery_job, database_url, response["job_id"]) + return response + + @app.get("/domains/{domain}/discovery/jobs/{job_id}") + def domain_discovery_job(domain: str, job_id: int): + normalized_domain = normalize_domain_name(domain) + with session_scope(database_url) as session: + job = session.get(models.DomainDiscoveryJob, job_id) + if job is None or job.domain != normalized_domain: + raise HTTPException(status_code=404, detail="domain discovery job not found") + return domain_discovery_job_payload(job) + + @app.post("/domains/{domain}/discovery/jobs/{job_id}/cancel") + def cancel_domain_discovery_job(domain: str, job_id: int): + normalized_domain = normalize_domain_name(domain) + with session_scope(database_url) as session: + job = session.get(models.DomainDiscoveryJob, job_id) + if job is None or job.domain != normalized_domain: + raise HTTPException(status_code=404, detail="domain discovery job not found") + if job.status in {"completed", "failed", "canceled"}: + return domain_discovery_job_payload(job) + job.status = "cancel_requested" + job.error = "cancel requested by user" + return domain_discovery_job_payload(job) + + @app.get("/domains/{domain}/candidates") + def list_domain_candidates(domain: str, status: str | None = None, limit: int = 200): + normalized_domain = normalize_domain_name(domain) + with session_scope(database_url) as session: + query = select(models.DomainSchemaCandidate).where( + models.DomainSchemaCandidate.domain == normalized_domain + ) + if status: + query = query.where(models.DomainSchemaCandidate.status == status) + rows = session.scalars( + query.order_by( + models.DomainSchemaCandidate.status, + models.DomainSchemaCandidate.candidate_type, + models.DomainSchemaCandidate.confidence.desc(), + models.DomainSchemaCandidate.name, + ).limit(max(min(limit, 500), 1)) + ).all() + payload = [] + for row in rows: + evidence = session.scalars( + select(models.DomainCandidateEvidence) + .where(models.DomainCandidateEvidence.candidate_id == row.id) + .order_by(models.DomainCandidateEvidence.created_at.desc()) + .limit(3) + ).all() + payload.append(domain_candidate_payload(row, evidence)) + return payload + + @app.post("/domains/{domain}/candidates/apply") + def apply_domain_candidates(domain: str, request: DomainCandidateApplyRequest): + normalized_domain = normalize_domain_name(domain) + if not request.candidate_ids: + raise HTTPException(status_code=400, detail="No candidates selected.") + with session_scope(database_url) as session: + definition = ensure_domain_definition(session, normalized_domain) + rows = session.scalars( + select(models.DomainSchemaCandidate).where( + models.DomainSchemaCandidate.domain == normalized_domain, + models.DomainSchemaCandidate.id.in_(request.candidate_ids), + ) + ).all() + entity_types = set(definition.entity_types or []) + predicates = set(definition.predicates or []) + attributes = set(definition.attributes or []) + aliases = dict(definition.aliases or {}) + applied = {"entity_type": 0, "predicate": 0, "attribute": 0, "alias": 0} + for row in rows: + if row.candidate_type == "entity_type": + entity_types.add(row.name) + applied["entity_type"] += 1 + elif row.candidate_type == "predicate": + predicates.add(row.name) + applied["predicate"] += 1 + elif row.candidate_type == "attribute": + attributes.add(row.name) + applied["attribute"] += 1 + elif row.candidate_type == "alias": + metadata = dict(row.metadata_json or {}) + targets = metadata.get("targets") or [] + target = metadata.get("target") or (targets[0] if targets else None) + if target: + aliases[row.name] = str(target) + applied["alias"] += 1 + row.status = "approved" + row.updated_at = models.utcnow() + definition.entity_types = sorted(entity_types) + definition.predicates = sorted(predicates) + definition.attributes = sorted(attributes) + definition.aliases = aliases + definition.is_builtin_override = normalized_domain in DOMAIN_ONTOLOGIES + definition.updated_at = models.utcnow() + return { + "ok": True, + "domain": normalized_domain, + "applied": applied, + "definition": domain_summary_payload( + domain_row_to_ontology(definition), + is_builtin=normalized_domain in DOMAIN_ONTOLOGIES, + is_custom=True, + description=definition.description, + ), } - for ont in DOMAIN_ONTOLOGIES.values() - ] + + @app.post("/domains/{domain}/candidates/{candidate_id}/status") + def update_domain_candidate_status(domain: str, candidate_id: int, request: DomainCandidateStatusRequest): + normalized_domain = normalize_domain_name(domain) + if request.status not in {"pending_review", "approved", "rejected"}: + raise HTTPException(status_code=400, detail="Unsupported candidate status.") + with session_scope(database_url) as session: + row = session.get(models.DomainSchemaCandidate, candidate_id) + if row is None or row.domain != normalized_domain: + raise HTTPException(status_code=404, detail="candidate not found") + row.status = request.status + row.updated_at = models.utcnow() + evidence = session.scalars( + select(models.DomainCandidateEvidence) + .where(models.DomainCandidateEvidence.candidate_id == row.id) + .order_by(models.DomainCandidateEvidence.created_at.desc()) + .limit(3) + ).all() + return domain_candidate_payload(row, evidence) @app.post("/projects/reset") def reset_project(request: ResetProjectRequest): @@ -568,6 +1162,35 @@ def register_routes(app, database_url: str) -> None: "deleted": deleted, } + @app.post("/projects/{project_name}/reset") + def reset_project_by_name(project_name: str): + with session_scope(database_url) as session: + repo = KnowledgeRepository(session) + project = repo.get_project(project_name) + config = project_config_from_project_row(session, project) + deleted = repo.reset_project_runtime_data(project.id) + project = repo.upsert_project(config) + return { + "ok": True, + "name": project.name, + "domain": project.domain, + "reset": True, + "deleted": deleted, + } + + @app.delete("/projects/{project_name}") + def delete_project(project_name: str): + with session_scope(database_url) as session: + project = KnowledgeRepository(session).get_project(project_name) + domain = project.domain + deleted = delete_project_data(session, project.id, include_project=True) + return { + "ok": True, + "name": project_name, + "domain": domain, + "deleted": deleted, + } + @app.get("/projects/{project_name}") def project_detail(project_name: str): with session_scope(database_url) as session: @@ -630,7 +1253,8 @@ def register_routes(app, database_url: str) -> None: @app.get("/ontology/{domain}") def ontology(domain: str): - return ontology_to_dict(ontology_for_domain(domain)) + with session_scope(database_url) as session: + return ontology_to_dict(ontology_for_domain_from_db(session, domain)) @app.get("/projects/{project_name}/ontology/registry") def ontology_registry(project_name: str): diff --git a/ontology_platform/crawler_platform/app/core/database/models.py b/ontology_platform/crawler_platform/app/core/database/models.py index 5d35f96..dbd314b 100644 --- a/ontology_platform/crawler_platform/app/core/database/models.py +++ b/ontology_platform/crawler_platform/app/core/database/models.py @@ -56,6 +56,91 @@ class Source(Base): project = relationship("Project", back_populates="sources") +class DomainDefinition(Base): + __tablename__ = "domain_definitions" + + id = Column(Integer, primary_key=True) + domain = Column(String(80), unique=True, nullable=False, index=True) + description = Column(Text) + entity_types = Column(JSON, nullable=False, default=list) + predicates = Column(JSON, nullable=False, default=list) + attributes = Column(JSON, nullable=False, default=list) + aliases = Column(JSON, nullable=False, default=dict) + is_builtin_override = Column(Boolean, nullable=False, default=False) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + +class DomainReferenceSource(Base): + __tablename__ = "domain_reference_sources" + __table_args__ = ( + UniqueConstraint("domain", "url", name="uq_domain_reference_source"), + ) + + id = Column(Integer, primary_key=True) + domain = Column(String(80), nullable=False, index=True) + url = Column(Text, nullable=False) + label = Column(String(160)) + status = Column(String(40), nullable=False, default="active", index=True) + last_crawled_at = Column(DateTime(timezone=True), nullable=True) + metadata_json = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + +class DomainDiscoveryJob(Base): + __tablename__ = "domain_discovery_jobs" + + id = Column(Integer, primary_key=True) + domain = Column(String(80), nullable=False, index=True) + status = Column(String(40), nullable=False, default="pending", index=True) + seed_urls = Column(JSON, nullable=False, default=list) + max_pages = Column(Integer, nullable=False, default=20) + max_depth = Column(Integer, nullable=False, default=1) + same_domain_only = Column(Boolean, nullable=False, default=True) + fetcher = Column(String(80), nullable=False, default="requests") + respect_robots_txt = Column(Boolean, nullable=False, default=False) + error = Column(Text) + progress = Column(JSON, nullable=False, default=dict) + result_summary = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + + +class DomainSchemaCandidate(Base): + __tablename__ = "domain_schema_candidates" + __table_args__ = ( + UniqueConstraint("domain", "candidate_type", "name", name="uq_domain_schema_candidate"), + ) + + id = Column(Integer, primary_key=True) + domain = Column(String(80), nullable=False, index=True) + job_id = Column(Integer, ForeignKey("domain_discovery_jobs.id"), nullable=True, index=True) + candidate_type = Column(String(40), nullable=False, index=True) + name = Column(String(160), nullable=False, index=True) + description = Column(Text) + confidence = Column(Float, nullable=False, default=0.5) + occurrence_count = Column(Integer, nullable=False, default=1) + status = Column(String(40), nullable=False, default="pending_review", index=True) + metadata_json = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + +class DomainCandidateEvidence(Base): + __tablename__ = "domain_candidate_evidence" + + id = Column(Integer, primary_key=True) + candidate_id = Column(Integer, ForeignKey("domain_schema_candidates.id"), nullable=False, index=True) + job_id = Column(Integer, ForeignKey("domain_discovery_jobs.id"), nullable=True, index=True) + url = Column(Text, nullable=False) + title = Column(Text) + snippet = Column(Text, nullable=False) + metadata_json = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + class Page(Base): __tablename__ = "pages" __table_args__ = (UniqueConstraint("project_id", "url", name="uq_page_project_url"),) diff --git a/ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py b/ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py new file mode 100644 index 0000000..dfd96a8 --- /dev/null +++ b/ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py @@ -0,0 +1,643 @@ +from __future__ import annotations + +from collections import Counter, deque +from dataclasses import dataclass, field +import json +import re +from typing import Any +from urllib.parse import urlparse + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from crawler_platform.app.core.crawler.discovery import discover_links +from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher +from crawler_platform.app.core.crawler.page_classifier import classify_page +from crawler_platform.app.core.crawler.plugins import default_parser_registry +from crawler_platform.app.core.database import models + + +SCHEMA_TYPES = {"entity_type", "predicate", "attribute", "alias"} +RELATION_HINTS = { + "brand": ("Brand", "hasBrand"), + "브랜드": ("Brand", "hasBrand"), + "category": ("Category", "hasCategory"), + "카테고리": ("Category", "hasCategory"), + "ingredient": ("Ingredient", "hasIngredient"), + "ingredients": ("Ingredient", "hasIngredient"), + "성분": ("Ingredient", "hasIngredient"), + "effect": ("Effect", "hasEffect"), + "benefit": ("Effect", "hasEffect"), + "효과": ("Effect", "hasEffect"), + "효능": ("Effect", "hasEffect"), + "warning": ("Warning", "hasWarning"), + "주의": ("Warning", "hasWarning"), + "review": ("Review", "hasReview"), + "rating": ("Rating", "hasRating"), + "image": ("Image", "hasImage"), + "price": ("Price", "hasPrice"), + "가격": ("Price", "hasPrice"), + "offer": ("Offer", "hasOffer"), +} + + +@dataclass(slots=True) +class CandidateEvidence: + url: str + title: str | None + snippet: str + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True) +class CandidateObservation: + candidate_type: str + name: str + description: str + confidence: float + evidence: CandidateEvidence + metadata: dict[str, Any] = field(default_factory=dict) + + +class DomainDiscoveryService: + def __init__(self, session: Session): + self.session = session + self.parser = default_parser_registry().get("generic") + + def run_job(self, job_id: int) -> None: + job = self.session.get(models.DomainDiscoveryJob, job_id) + if job is None: + return + + job.status = "running" + job.started_at = models.utcnow() + job.progress = { + "visited_count": 0, + "queued_count": len(job.seed_urls or []), + "candidate_count": 0, + "errors": [], + "pages": [], + } + self.session.commit() + + visited: set[str] = set() + queued: set[str] = set() + queue: deque[tuple[str, int, str]] = deque() + for seed_url in job.seed_urls or []: + normalized = normalize_url(str(seed_url)) + if normalized: + queue.append((normalized, 0, seed_host(normalized))) + queued.add(normalized) + self._upsert_reference_source(job.domain, normalized) + + fetcher = make_fetcher(job.fetcher, rate_limit_per_minute=30) + robots = RobotsPolicy() + observation_count = 0 + errors: list[str] = [] + page_payloads: list[dict[str, Any]] = [] + + while queue and len(visited) < max(min(job.max_pages, 200), 1): + if self._cancel_requested(job): + job.status = "canceled" + job.finished_at = models.utcnow() + self.session.commit() + return + + url, depth, root_host = queue.popleft() + if url in visited: + continue + visited.add(url) + + if depth > max(job.max_depth, 0): + continue + if job.same_domain_only and seed_host(url) != root_host: + continue + + decision = robots.check(url, job.respect_robots_txt) + if not decision.allowed: + errors.append(f"{decision.reason}: {url}") + self._save_progress(job, visited, queue, observation_count, errors, page_payloads) + continue + + try: + fetched = fetcher.fetch(url) + parsed = self.parser.parse(fetched.analysis_html, fetched.final_url or url) + page_type = classify_page( + fetched.final_url or url, + parsed.title or fetched.title, + parsed.text or parsed.raw_text, + html=fetched.analysis_html, + source_zones=parsed.source_zones or [], + ) + page_observations = mine_schema_candidates( + url=fetched.final_url or url, + title=parsed.title or fetched.title, + text=parsed.text or parsed.raw_text, + html=fetched.analysis_html, + page_type=page_type, + ) + observation_count += len(page_observations) + if page_observations: + self._save_observations(job, page_observations) + page_payloads.append( + { + "url": fetched.final_url or url, + "title": parsed.title or fetched.title, + "page_type": page_type, + "candidate_count": len(page_observations), + "text_length": len(parsed.text or ""), + "warnings": [*fetched.warnings, *(parsed.extraction_warnings or [])], + } + ) + self._enqueue_links( + html=fetched.analysis_html, + base_url=fetched.final_url or url, + root_host=root_host, + depth=depth, + max_depth=max(job.max_depth, 0), + same_domain_only=job.same_domain_only, + queue=queue, + queued=queued, + visited=visited, + ) + except Exception as exc: + errors.append(f"{url}: {exc}") + + self._save_progress(job, visited, queue, observation_count, errors, page_payloads) + + reached_page_limit = len(visited) >= max(min(job.max_pages, 200), 1) + completion_reason = "max_pages_reached" if reached_page_limit else "no_more_links" + saved = self._candidate_count(job.domain) + job.status = "completed" + job.finished_at = models.utcnow() + job.result_summary = { + "visited_count": len(visited), + "candidate_count": saved, + "completion_reason": completion_reason, + "errors": errors[-20:], + "pages": page_payloads[-50:], + } + self._save_progress( + job, + visited, + queue, + observation_count, + errors, + page_payloads, + completion_reason=completion_reason, + ) + self.session.commit() + + def _save_progress( + self, + job: models.DomainDiscoveryJob, + visited: set[str], + queue: deque[tuple[str, int, str]], + observation_count: int, + errors: list[str], + page_payloads: list[dict[str, Any]], + completion_reason: str | None = None, + ) -> None: + job.progress = { + "visited_count": len(visited), + "queued_count": len(queue), + "candidate_count": self._candidate_count(job.domain), + "observation_count": observation_count, + "max_pages": job.max_pages, + "errors": errors[-10:], + "pages": page_payloads[-10:], + } + if completion_reason: + job.progress["completion_reason"] = completion_reason + self.session.commit() + + def _cancel_requested(self, job: models.DomainDiscoveryJob) -> bool: + self.session.expire(job) + return job.status == "cancel_requested" + + def _candidate_count(self, domain: str) -> int: + return int( + self.session.query(models.DomainSchemaCandidate) + .filter(models.DomainSchemaCandidate.domain == domain) + .count() + ) + + def _save_observations(self, job: models.DomainDiscoveryJob, observations: list[CandidateObservation]) -> int: + if not observations: + return 0 + + by_key: dict[tuple[str, str], list[CandidateObservation]] = {} + for observation in observations: + if observation.candidate_type not in SCHEMA_TYPES: + continue + key = (observation.candidate_type, observation.name) + by_key.setdefault(key, []).append(observation) + + saved_count = 0 + for (candidate_type, name), rows in sorted(by_key.items()): + row = self.session.scalar( + select(models.DomainSchemaCandidate).where( + models.DomainSchemaCandidate.domain == job.domain, + models.DomainSchemaCandidate.candidate_type == candidate_type, + models.DomainSchemaCandidate.name == name, + ) + ) + confidence = min(max(max(item.confidence for item in rows) + min(len(rows), 10) * 0.02, 0.0), 0.98) + metadata = merge_metadata(item.metadata for item in rows) + if row is None: + row = models.DomainSchemaCandidate( + domain=job.domain, + job_id=job.id, + candidate_type=candidate_type, + name=name, + description=rows[0].description, + confidence=round(confidence, 4), + occurrence_count=len(rows), + metadata_json=metadata, + ) + self.session.add(row) + self.session.flush() + else: + row.job_id = job.id + row.description = row.description or rows[0].description + row.confidence = max(row.confidence, round(confidence, 4)) + row.occurrence_count += len(rows) + row.metadata_json = {**(row.metadata_json or {}), **metadata} + row.updated_at = models.utcnow() + for evidence in rows[:8]: + self.session.add( + models.DomainCandidateEvidence( + candidate_id=row.id, + job_id=job.id, + url=evidence.evidence.url, + title=evidence.evidence.title, + snippet=evidence.evidence.snippet[:1000], + metadata_json=evidence.evidence.metadata, + ) + ) + saved_count += 1 + return saved_count + + def _upsert_reference_source(self, domain: str, url: str) -> models.DomainReferenceSource: + row = self.session.scalar( + select(models.DomainReferenceSource).where( + models.DomainReferenceSource.domain == domain, + models.DomainReferenceSource.url == url, + ) + ) + if row is None: + row = models.DomainReferenceSource(domain=domain, url=url) + self.session.add(row) + self.session.flush() + row.last_crawled_at = models.utcnow() + row.updated_at = models.utcnow() + return row + + def _enqueue_links( + self, + *, + html: str, + base_url: str, + root_host: str, + depth: int, + max_depth: int, + same_domain_only: bool, + queue: deque[tuple[str, int, str]], + queued: set[str], + visited: set[str], + ) -> None: + if depth >= max_depth: + return + for link in discover_links(html, base_url, limit=100): + next_url = normalize_url(link.url) + if not next_url or next_url in queued or next_url in visited: + continue + if same_domain_only and seed_host(next_url) != root_host: + continue + queue.append((next_url, depth + 1, root_host)) + queued.add(next_url) + + +def mine_schema_candidates( + *, + url: str, + title: str | None, + text: str, + html: str, + page_type: str, +) -> list[CandidateObservation]: + observations: list[CandidateObservation] = [] + evidence_base = {"page_type": page_type} + page_type_entity = entity_type_from_page_type(page_type) + if page_type_entity: + observations.append( + observation( + "entity_type", + page_type_entity, + f"Detected from page type {page_type}.", + 0.72, + url, + title, + title or first_sentence(text), + evidence_base, + ) + ) + + soup = soup_from_html(html) + labels = collect_structural_labels(soup, text) + json_ld_types, json_ld_keys = collect_json_ld_schema(soup) + + for schema_type, count in json_ld_types.items(): + observations.append( + observation( + "entity_type", + pascal_case(schema_type), + "Detected from JSON-LD schema type.", + min(0.86 + count * 0.02, 0.95), + url, + title, + f"JSON-LD @type: {schema_type}", + {**evidence_base, "source": "json_ld"}, + ) + ) + for key, count in json_ld_keys.items(): + attr = attribute_name(key) + if attr: + observations.append( + observation( + "attribute", + attr, + "Detected from structured data property.", + min(0.78 + count * 0.02, 0.92), + url, + title, + f"Structured data property: {key}", + {**evidence_base, "source": "json_ld", "raw_label": key}, + ) + ) + + for label, count in labels.items(): + attr = attribute_name(label) + if not attr: + continue + observations.append( + observation( + "attribute", + attr, + "Detected from page labels, tables, metadata, or key-value text.", + min(0.62 + count * 0.04, 0.88), + url, + title, + label, + {**evidence_base, "source": "page_label", "raw_label": label}, + ) + ) + hint = relation_hint(label) + if hint: + entity_type, predicate = hint + observations.append( + observation( + "entity_type", + entity_type, + f"Detected from repeated label '{label}'.", + min(0.68 + count * 0.03, 0.9), + url, + title, + label, + {**evidence_base, "source": "relation_hint", "raw_label": label}, + ) + ) + observations.append( + observation( + "predicate", + predicate, + f"Suggested relation for label '{label}'.", + min(0.7 + count * 0.03, 0.92), + url, + title, + label, + { + **evidence_base, + "source": "relation_hint", + "raw_label": label, + "object_type": entity_type, + }, + ) + ) + observations.append( + observation( + "alias", + attr, + f"Map source field '{attr}' to relation '{predicate}'.", + 0.66, + url, + title, + label, + {**evidence_base, "source": "relation_hint", "target": predicate}, + ) + ) + + return observations + + +def observation( + candidate_type: str, + name: str, + description: str, + confidence: float, + url: str, + title: str | None, + snippet: str, + metadata: dict[str, Any], +) -> CandidateObservation: + return CandidateObservation( + candidate_type=candidate_type, + name=name[:160], + description=description, + confidence=confidence, + evidence=CandidateEvidence(url=url, title=title, snippet=clean_snippet(snippet), metadata=metadata), + metadata=metadata, + ) + + +def soup_from_html(html: str): + from bs4 import BeautifulSoup + + return BeautifulSoup(html or "", "html.parser") + + +def collect_json_ld_schema(soup) -> tuple[Counter[str], Counter[str]]: + types: Counter[str] = Counter() + keys: Counter[str] = Counter() + for tag in soup.find_all("script"): + script_type = " ".join(str(item) for item in tag.get("type", "").split()).lower() + if "ld+json" not in script_type: + continue + raw = tag.string or tag.get_text(" ", strip=True) + try: + payload = json.loads(raw) + except Exception: + continue + visit_json_ld(payload, types, keys) + return types, keys + + +def visit_json_ld(value: Any, types: Counter[str], keys: Counter[str]) -> None: + if isinstance(value, list): + for item in value: + visit_json_ld(item, types, keys) + return + if not isinstance(value, dict): + return + for key, nested in value.items(): + if key == "@type": + for item in ensure_list(nested): + if isinstance(item, str) and is_schema_name(item): + types[item] += 1 + elif not key.startswith("@"): + keys[key] += 1 + visit_json_ld(nested, types, keys) + + +def collect_structural_labels(soup, text: str) -> Counter[str]: + labels: Counter[str] = Counter() + for selector in ["th", "dt", "label", "[itemprop]", "[property]", "[name]"]: + for node in soup.select(selector): + label = "" + if selector == "[itemprop]": + label = node.get("itemprop", "") + elif selector == "[property]": + label = str(node.get("property", "")).split(":")[-1] + elif selector == "[name]": + label = str(node.get("name", "")).split(":")[-1] + else: + label = node.get_text(" ", strip=True) + add_label(labels, label) + + for line in text.splitlines()[:500]: + match = re.match(r"^\s*([^::|]{2,40})\s*[::|]\s*(.{1,200})$", line) + if match: + add_label(labels, match.group(1)) + return labels + + +def add_label(labels: Counter[str], label: str) -> None: + cleaned = clean_label(label) + if cleaned and not is_noisy_label(cleaned): + labels[cleaned] += 1 + + +def relation_hint(label: str) -> tuple[str, str] | None: + lowered = label.lower() + for token, hint in RELATION_HINTS.items(): + if token in lowered: + return hint + return None + + +def entity_type_from_page_type(page_type: str) -> str | None: + mapping = { + "ProductPage": "Product", + "CategoryPage": "Category", + "BrandStoryPage": "Brand", + "NoticePage": "Notice", + "PromotionPage": "Promotion", + "ReviewPage": "Review", + "BoardPage": "Article", + } + return mapping.get(page_type) + + +def attribute_name(label: str) -> str: + cleaned = clean_label(label) + if not cleaned or is_noisy_label(cleaned): + return "" + if re.search(r"[가-힣]", cleaned): + return re.sub(r"[\s/()-]+", "_", cleaned).strip("_")[:80] + words = re.findall(r"[A-Za-z0-9]+", cleaned) + if not words: + return "" + return "_".join(word.lower() for word in words)[:80] + + +def pascal_case(value: str) -> str: + cleaned = clean_label(value) + parts = re.findall(r"[A-Za-z0-9가-힣]+", cleaned) + if not parts: + return "" + return "".join(part[:1].upper() + part[1:] for part in parts)[:80] + + +def clean_label(value: str) -> str: + return re.sub(r"\s+", " ", str(value or "").replace("\xa0", " ")).strip(" -_*:/|") + + +def is_schema_name(value: str) -> bool: + return 1 <= len(value) <= 80 and not value.startswith("http") + + +def is_noisy_label(value: str) -> bool: + lowered = value.lower().strip() + if len(lowered) < 2 or len(lowered) > 80: + return True + noisy = { + "class", + "style", + "viewport", + "charset", + "csrf-token", + "description", + "keywords", + "robots", + "generator", + "theme-color", + } + return lowered in noisy or lowered.startswith("twitter:") + + +def clean_snippet(value: str) -> str: + return re.sub(r"\s+", " ", str(value or "")).strip()[:1000] + + +def first_sentence(text: str) -> str: + cleaned = clean_snippet(text) + for delimiter in [". ", "\n", "。", "다."]: + if delimiter in cleaned: + return cleaned.split(delimiter, 1)[0][:240] + return cleaned[:240] + + +def normalize_url(url: str) -> str: + return str(url or "").strip().rstrip("/") + + +def seed_host(url: str) -> str: + return urlparse(url).netloc.lower() + + +def ensure_list(value: Any) -> list[Any]: + if isinstance(value, list): + return value + return [value] + + +def merge_metadata(items) -> dict[str, Any]: + merged: dict[str, Any] = {} + sources: set[str] = set() + targets: set[str] = set() + object_types: set[str] = set() + for item in items: + merged.update(item or {}) + if item.get("source"): + sources.add(str(item["source"])) + if item.get("target"): + targets.add(str(item["target"])) + if item.get("object_type"): + object_types.add(str(item["object_type"])) + if sources: + merged["sources"] = sorted(sources) + if targets: + merged["targets"] = sorted(targets) + if object_types: + merged["object_types"] = sorted(object_types) + return merged diff --git a/ontology_platform/web/frontend/src/App.tsx b/ontology_platform/web/frontend/src/App.tsx index 51ff222..36c1b51 100644 --- a/ontology_platform/web/frontend/src/App.tsx +++ b/ontology_platform/web/frontend/src/App.tsx @@ -1,6 +1,7 @@ import { Routes, Route } from "react-router-dom"; import AppShell from "@/components/layout/AppShell"; import OnboardingPage from "@/pages/OnboardingPage"; +import DomainBuilderPage from "@/pages/DomainBuilderPage"; import ConfigureSourcesPage from "@/pages/ConfigureSourcesPage"; import CrawlPage from "@/pages/CrawlPage"; import ResearchPage from "@/pages/ResearchPage"; @@ -20,6 +21,7 @@ function App() { }> } /> } /> + } /> } /> } /> } /> diff --git a/ontology_platform/web/frontend/src/components/layout/navigation.ts b/ontology_platform/web/frontend/src/components/layout/navigation.ts index f1e6ada..a2c4182 100644 --- a/ontology_platform/web/frontend/src/components/layout/navigation.ts +++ b/ontology_platform/web/frontend/src/components/layout/navigation.ts @@ -5,6 +5,7 @@ import { ClipboardCheck, Code2, GitBranch, + Globe2, Layers3, LayoutDashboard, ListChecks, @@ -66,6 +67,15 @@ export const NAV_GROUPS: NavGroup[] = [ keywords: ["create", "start", "wizard"], shortcut: "g n", }, + { + id: "domains", + to: "/domains", + labelKey: "nav.domains", + label: "Domain Builder", + icon: Globe2, + keywords: ["domain", "schema", "reference", "crawler"], + shortcut: "g m", + }, ], }, { diff --git a/ontology_platform/web/frontend/src/components/ui/data-table.tsx b/ontology_platform/web/frontend/src/components/ui/data-table.tsx index 524a6ea..2d03c0f 100644 --- a/ontology_platform/web/frontend/src/components/ui/data-table.tsx +++ b/ontology_platform/web/frontend/src/components/ui/data-table.tsx @@ -808,6 +808,7 @@ function TableShell({ scope="col" style={{ width: header.getSize(), + maxWidth: header.getSize(), ...getPinStyle(col, "header"), }} aria-sort={ @@ -921,16 +922,19 @@ function TableShell({ key={cell.id} style={{ width: col.getSize(), + maxWidth: col.getSize(), ...getPinStyle(col, "cell"), }} className={cn( cellPadClass, alignClass, - "text-foreground", + "overflow-hidden text-foreground align-middle", pinned && "bg-surface", )} > - {flexRender(col.columnDef.cell, cell.getContext())} +
+ {flexRender(col.columnDef.cell, cell.getContext())} +
); })} diff --git a/ontology_platform/web/frontend/src/components/ui/dialog.tsx b/ontology_platform/web/frontend/src/components/ui/dialog.tsx index 7400fef..7031cba 100644 --- a/ontology_platform/web/frontend/src/components/ui/dialog.tsx +++ b/ontology_platform/web/frontend/src/components/ui/dialog.tsx @@ -92,7 +92,7 @@ export const DialogPanel = React.forwardRef( "fixed left-1/2 top-1/2 z-50 -translate-x-1/2 -translate-y-1/2", "flex max-h-[90vh] w-full max-w-lg flex-col overflow-hidden", "rounded-xl border border-border bg-surface-raised shadow-xl outline-none", - "data-[state=open]:animate-fade-in-up", + "data-[state=open]:animate-fade-in", className, )} {...props} diff --git a/ontology_platform/web/frontend/src/hooks/useDomains.ts b/ontology_platform/web/frontend/src/hooks/useDomains.ts index 861f5d3..4bca78e 100644 --- a/ontology_platform/web/frontend/src/hooks/useDomains.ts +++ b/ontology_platform/web/frontend/src/hooks/useDomains.ts @@ -1,5 +1,14 @@ -import { useQuery } from "@tanstack/react-query"; -import { domainsApi, DomainSummary } from "@/lib/api/domains"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { + domainsApi, + DomainDefinitionRequest, + DomainDefinitionUpdateRequest, + DomainDiscoveryJob, + DomainDiscoveryRequest, + DomainReferenceSource, + DomainCandidate, + DomainSummary, +} from "@/lib/api/domains"; import { ontologyApi, OntologyDetail } from "@/lib/api/ontology"; import { queryKeys } from "./queryKeys"; @@ -11,6 +20,151 @@ export function useDomains() { }); } +export function useCreateDomain() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (body: DomainDefinitionRequest) => domainsApi.create(body), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: queryKeys.domains.all }); + queryClient.invalidateQueries({ queryKey: queryKeys.ontology.all }); + }, + }); +} + +export function useUpdateDomain() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: ({ + domain, + body, + }: { + domain: string; + body: DomainDefinitionUpdateRequest; + }) => domainsApi.update(domain, body), + onSuccess: (_data, variables) => { + queryClient.invalidateQueries({ queryKey: queryKeys.domains.all }); + queryClient.invalidateQueries({ + queryKey: queryKeys.ontology.byDomain(variables.domain), + }); + }, + }); +} + +export function useDomainDiscoveryJobs(domain: string | undefined) { + return useQuery({ + queryKey: [...queryKeys.domains.all, "discoveryJobs", domain ?? ""], + queryFn: () => domainsApi.discoveryJobs(domain!), + enabled: Boolean(domain), + refetchInterval: (query) => { + const jobs = query.state.data; + return jobs?.some((job) => ["pending", "running", "cancel_requested"].includes(job.status)) + ? 2000 + : false; + }, + }); +} + +export function useDomainReferenceSources(domain: string | undefined) { + return useQuery({ + queryKey: [...queryKeys.domains.all, "referenceSources", domain ?? ""], + queryFn: () => domainsApi.referenceSources(domain!), + enabled: Boolean(domain), + }); +} + +export function useDomainCandidates(domain: string | undefined) { + return useQuery({ + queryKey: [...queryKeys.domains.all, "candidates", domain ?? ""], + queryFn: () => domainsApi.candidates(domain!), + enabled: Boolean(domain), + refetchInterval: 2000, + }); +} + +export function useStartDomainDiscovery() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: ({ + domain, + body, + }: { + domain: string; + body: DomainDiscoveryRequest; + }) => domainsApi.startDiscovery(domain, body), + onSuccess: (_data, variables) => { + queryClient.invalidateQueries({ + queryKey: [...queryKeys.domains.all, "discoveryJobs", variables.domain], + }); + queryClient.invalidateQueries({ + queryKey: [...queryKeys.domains.all, "referenceSources", variables.domain], + }); + queryClient.invalidateQueries({ + queryKey: [...queryKeys.domains.all, "candidates", variables.domain], + }); + }, + }); +} + +export function useCancelDomainDiscovery() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: ({ + domain, + jobId, + }: { + domain: string; + jobId: string; + }) => domainsApi.cancelDiscovery(domain, jobId), + onSuccess: (_data, variables) => { + queryClient.invalidateQueries({ + queryKey: [...queryKeys.domains.all, "discoveryJobs", variables.domain], + }); + }, + }); +} + +export function useApplyDomainCandidates() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: ({ + domain, + candidateIds, + }: { + domain: string; + candidateIds: string[]; + }) => domainsApi.applyCandidates(domain, candidateIds), + onSuccess: (_data, variables) => { + queryClient.invalidateQueries({ queryKey: queryKeys.domains.all }); + queryClient.invalidateQueries({ + queryKey: queryKeys.ontology.byDomain(variables.domain), + }); + queryClient.invalidateQueries({ + queryKey: [...queryKeys.domains.all, "candidates", variables.domain], + }); + }, + }); +} + +export function useUpdateDomainCandidateStatus() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: ({ + domain, + candidateId, + status, + }: { + domain: string; + candidateId: string; + status: "pending_review" | "approved" | "rejected"; + }) => domainsApi.updateCandidateStatus(domain, candidateId, status), + onSuccess: (_data, variables) => { + queryClient.invalidateQueries({ + queryKey: [...queryKeys.domains.all, "candidates", variables.domain], + }); + }, + }); +} + export function useOntology(domain: string | undefined) { return useQuery({ queryKey: queryKeys.ontology.byDomain(domain ?? ""), diff --git a/ontology_platform/web/frontend/src/hooks/useProjects.ts b/ontology_platform/web/frontend/src/hooks/useProjects.ts index 6c2cb63..434d5e2 100644 --- a/ontology_platform/web/frontend/src/hooks/useProjects.ts +++ b/ontology_platform/web/frontend/src/hooks/useProjects.ts @@ -43,3 +43,31 @@ export function useCreateProjectInline() { }, }); } + +export function useResetProject() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (projectName: string) => projectsApi.reset(projectName), + onSuccess: (_data, projectName) => { + queryClient.invalidateQueries({ queryKey: queryKeys.projects.all }); + queryClient.invalidateQueries({ + queryKey: queryKeys.projects.detail(projectName), + }); + queryClient.invalidateQueries({ queryKey: queryKeys.platform.all }); + }, + }); +} + +export function useDeleteProject() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (projectName: string) => projectsApi.delete(projectName), + onSuccess: (_data, projectName) => { + queryClient.invalidateQueries({ queryKey: queryKeys.projects.all }); + queryClient.removeQueries({ + queryKey: queryKeys.projects.detail(projectName), + }); + queryClient.invalidateQueries({ queryKey: queryKeys.platform.all }); + }, + }); +} diff --git a/ontology_platform/web/frontend/src/lib/api/domains.ts b/ontology_platform/web/frontend/src/lib/api/domains.ts index cb42a0d..2af051e 100644 --- a/ontology_platform/web/frontend/src/lib/api/domains.ts +++ b/ontology_platform/web/frontend/src/lib/api/domains.ts @@ -3,15 +3,177 @@ import { apiClient } from "./client"; export const domainSummarySchema = z.object({ domain: z.string(), + description: z.string().nullable().optional(), entity_types: z.array(z.string()), predicates: z.array(z.string()), attribute_count: z.number().int(), + attributes: z.array(z.string()).default([]), + aliases: z.record(z.string(), z.string()).default({}), + is_builtin: z.boolean().default(false), + is_custom: z.boolean().default(false), }); export const domainListSchema = z.array(domainSummarySchema); export type DomainSummary = z.infer; +export const domainDiscoveryJobSchema = z.object({ + job_id: z.union([z.string(), z.number()]).transform(String), + domain: z.string(), + status: z.string(), + seed_urls: z.array(z.string()).default([]), + max_pages: z.number(), + max_depth: z.number(), + same_domain_only: z.boolean(), + fetcher: z.string(), + respect_robots_txt: z.boolean(), + error: z.string().nullable().optional(), + progress: z.record(z.unknown()).default({}), + result_summary: z.record(z.unknown()).default({}), + created_at: z.string().nullable().optional(), + started_at: z.string().nullable().optional(), + finished_at: z.string().nullable().optional(), +}); + +export const domainReferenceSourceSchema = z.object({ + id: z.union([z.string(), z.number()]).transform(String), + domain: z.string(), + url: z.string(), + label: z.string().nullable().optional(), + status: z.string(), + last_crawled_at: z.string().nullable().optional(), + metadata: z.record(z.unknown()).default({}), + created_at: z.string().nullable().optional(), + updated_at: z.string().nullable().optional(), +}); + +export const domainCandidateEvidenceSchema = z.object({ + id: z.union([z.string(), z.number()]).transform(String), + job_id: z.union([z.string(), z.number()]).nullable().optional().transform((v) => + v == null ? null : String(v), + ), + url: z.string(), + title: z.string().nullable().optional(), + snippet: z.string(), + metadata: z.record(z.unknown()).default({}), + created_at: z.string().nullable().optional(), +}); + +export const domainCandidateSchema = z.object({ + id: z.union([z.string(), z.number()]).transform(String), + domain: z.string(), + job_id: z.union([z.string(), z.number()]).nullable().optional().transform((v) => + v == null ? null : String(v), + ), + candidate_type: z.string(), + name: z.string(), + description: z.string().nullable().optional(), + confidence: z.number(), + occurrence_count: z.number(), + status: z.string(), + metadata: z.record(z.unknown()).default({}), + evidence: z.array(domainCandidateEvidenceSchema).default([]), + created_at: z.string().nullable().optional(), + updated_at: z.string().nullable().optional(), +}); + +export const domainDiscoveryJobListSchema = z.array(domainDiscoveryJobSchema); +export const domainReferenceSourceListSchema = z.array(domainReferenceSourceSchema); +export const domainCandidateListSchema = z.array(domainCandidateSchema); +export const domainCandidateApplyResponseSchema = z.object({ + ok: z.boolean(), + domain: z.string(), + applied: z.record(z.number()), + definition: domainSummarySchema, +}); + +export type DomainDiscoveryJob = z.infer; +export type DomainReferenceSource = z.infer; +export type DomainCandidate = z.infer; +export type DomainCandidateApplyResponse = z.infer< + typeof domainCandidateApplyResponseSchema +>; + +export interface DomainDefinitionRequest { + domain: string; + description?: string | null; + entity_types: string[]; + predicates: string[]; + attributes: string[]; + aliases?: Record; +} + +export type DomainDefinitionUpdateRequest = Omit< + DomainDefinitionRequest, + "domain" +>; + +export interface DomainDiscoveryRequest { + seed_urls: string[]; + max_pages?: number; + max_depth?: number; + same_domain_only?: boolean; + fetcher?: string; + respect_robots_txt?: boolean; + force_recrawl?: boolean; +} + export const domainsApi = { list: () => apiClient.get("/domains", domainListSchema), + create: (body: DomainDefinitionRequest) => + apiClient.post("/domains", domainSummarySchema, body), + update: (domain: string, body: DomainDefinitionUpdateRequest) => + apiClient.put( + `/domains/${encodeURIComponent(domain)}`, + domainSummarySchema, + body, + ), + discoveryJobs: (domain: string) => + apiClient.get( + `/domains/${encodeURIComponent(domain)}/discovery/jobs`, + domainDiscoveryJobListSchema, + ), + referenceSources: (domain: string) => + apiClient.get( + `/domains/${encodeURIComponent(domain)}/reference-sources`, + domainReferenceSourceListSchema, + ), + startDiscovery: (domain: string, body: DomainDiscoveryRequest) => + apiClient.post( + `/domains/${encodeURIComponent(domain)}/discovery/jobs`, + domainDiscoveryJobSchema, + body, + ), + discoveryJob: (domain: string, jobId: string) => + apiClient.get( + `/domains/${encodeURIComponent(domain)}/discovery/jobs/${encodeURIComponent(jobId)}`, + domainDiscoveryJobSchema, + ), + cancelDiscovery: (domain: string, jobId: string) => + apiClient.post( + `/domains/${encodeURIComponent(domain)}/discovery/jobs/${encodeURIComponent(jobId)}/cancel`, + domainDiscoveryJobSchema, + ), + candidates: (domain: string, status?: string) => + apiClient.get( + `/domains/${encodeURIComponent(domain)}/candidates`, + domainCandidateListSchema, + { query: { status } }, + ), + applyCandidates: (domain: string, candidateIds: string[]) => + apiClient.post( + `/domains/${encodeURIComponent(domain)}/candidates/apply`, + domainCandidateApplyResponseSchema, + { candidate_ids: candidateIds.map(Number) }, + ), + updateCandidateStatus: ( + domain: string, + candidateId: string, + status: "pending_review" | "approved" | "rejected", + ) => + apiClient.post( + `/domains/${encodeURIComponent(domain)}/candidates/${encodeURIComponent(candidateId)}/status`, + domainCandidateSchema, + { status }, + ), }; diff --git a/ontology_platform/web/frontend/src/lib/api/projects.ts b/ontology_platform/web/frontend/src/lib/api/projects.ts index 12c7bfc..202d3a5 100644 --- a/ontology_platform/web/frontend/src/lib/api/projects.ts +++ b/ontology_platform/web/frontend/src/lib/api/projects.ts @@ -35,9 +35,17 @@ export const createProjectResponseSchema = z.object({ domain: z.string(), }); +export const projectActionResponseSchema = z.object({ + ok: z.boolean(), + name: z.string(), + domain: z.string().optional(), + deleted: z.record(z.number()).optional(), +}); + export type ProjectSummary = z.infer; export type ProjectDetail = z.infer; export type ProjectSource = z.infer; +export type ProjectActionResponse = z.infer; export interface CreateProjectRequest { config_path: string; @@ -77,4 +85,14 @@ export const projectsApi = { apiClient.post("/projects", createProjectResponseSchema, body), createInline: (body: CreateProjectInlineRequest) => apiClient.post("/projects/inline", createProjectResponseSchema, body), + reset: (projectName: string) => + apiClient.post( + `/projects/${encodeURIComponent(projectName)}/reset`, + projectActionResponseSchema, + ), + delete: (projectName: string) => + apiClient.delete( + `/projects/${encodeURIComponent(projectName)}`, + projectActionResponseSchema, + ), }; diff --git a/ontology_platform/web/frontend/src/pages/DashboardPage.tsx b/ontology_platform/web/frontend/src/pages/DashboardPage.tsx index 4f1666c..50e8af4 100644 --- a/ontology_platform/web/frontend/src/pages/DashboardPage.tsx +++ b/ontology_platform/web/frontend/src/pages/DashboardPage.tsx @@ -1,6 +1,15 @@ import { useNavigate } from "react-router-dom"; import { useTranslation } from "react-i18next"; -import { Plus, FolderOpen, AlertCircle, Clock } from "lucide-react"; +import { useState } from "react"; +import { toast } from "sonner"; +import { + Plus, + FolderOpen, + AlertCircle, + Clock, + DatabaseZap, + Trash2, +} from "lucide-react"; import { Card, CardContent, @@ -9,8 +18,21 @@ import { CardTitle, } from "@/components/ui/card"; import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogBody, + DialogDescription, + DialogFooter, + DialogHeader, + DialogPanel, + DialogTitle, +} from "@/components/ui/dialog"; import { Skeleton } from "@/components/ui/skeleton"; -import { useProjects } from "@/hooks/useProjects"; +import { + useDeleteProject, + useProjects, + useResetProject, +} from "@/hooks/useProjects"; import { usePipeline } from "@/hooks/usePlatform"; import { formatPercent } from "@/lib/display"; import { ProjectSummary } from "@/lib/api/projects"; @@ -128,6 +150,11 @@ function ProjectCard({ onOpen: (path: string) => void; }) { const pipeline = usePipeline(project.name); + const resetProject = useResetProject(); + const deleteProject = useDeleteProject(); + const [confirmAction, setConfirmAction] = useState<"reset" | "delete" | null>( + null, + ); const claimStage = pipeline.data?.stages.find((stage) => stage.key === "claims"); const approvedStage = pipeline.data?.stages.find( (stage) => stage.key === "validated", @@ -136,35 +163,136 @@ function ProjectCard({ const claimCount = claimStage?.count ?? 0; const approvedCount = approvedStage?.count ?? 0; const approvalRate = claimCount > 0 ? approvedCount / claimCount : 0; + const isDeleting = deleteProject.isPending; + const isResetting = resetProject.isPending; + + const handleProjectAction = async () => { + if (!confirmAction) return; + try { + if (confirmAction === "reset") { + await resetProject.mutateAsync(project.name); + toast.success(`${project.name} 데이터가 초기화되었습니다`); + } else { + await deleteProject.mutateAsync(project.name); + toast.success(`${project.name} 프로젝트가 삭제되었습니다`); + } + setConfirmAction(null); + } catch (error) { + toast.error((error as Error).message); + } + }; return ( - onOpen(`/sources/${project.name}`)} - onKeyDown={(e) => { - if (e.key === "Enter" || e.key === " ") { - onOpen(`/sources/${project.name}`); - } - }} - className="cursor-pointer transition-colors hover:bg-accent/40" - > - - {project.name} - {project.domain} - - -
- - - + <> + onOpen(`/sources/${project.name}`)} + onKeyDown={(e) => { + if (e.key === "Enter" || e.key === " ") { + onOpen(`/sources/${project.name}`); + } + }} + className="relative cursor-pointer transition-colors hover:bg-accent/40" + > +
+ +
-
- - {formatRelative(project.updated_at)} -
- -
+ + {project.name} + {project.domain} + + +
+ + + +
+
+ + {formatRelative(project.updated_at)} +
+
+ + + { + if (!open && !isDeleting && !isResetting) setConfirmAction(null); + }} + ariaLabel="프로젝트 작업 확인" + > + + setConfirmAction(null)}> + + {confirmAction === "delete" ? "프로젝트 삭제" : "데이터 초기화"} + + + {confirmAction === "delete" + ? "프로젝트와 연결된 모든 데이터가 삭제됩니다." + : "프로젝트는 유지하고 크롤링/엔티티/클레임 데이터를 초기화합니다."} + + + +

+ {project.name} + {confirmAction === "delete" + ? " 프로젝트를 삭제할까요?" + : " 데이터를 초기화할까요?"} +

+
+ + + + +
+
+ ); } diff --git a/ontology_platform/web/frontend/src/pages/DomainBuilderPage.tsx b/ontology_platform/web/frontend/src/pages/DomainBuilderPage.tsx new file mode 100644 index 0000000..1b8b991 --- /dev/null +++ b/ontology_platform/web/frontend/src/pages/DomainBuilderPage.tsx @@ -0,0 +1,1082 @@ +import { useEffect, useMemo, useState } from "react"; +import type { ReactNode } from "react"; +import { useSearchParams } from "react-router-dom"; +import { + Ban, + Check, + Globe2, + Loader2, + Plus, + RefreshCw, + SearchCheck, + X, +} from "lucide-react"; +import { toast } from "sonner"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Textarea } from "@/components/ui/textarea"; +import { + useApplyDomainCandidates, + useCancelDomainDiscovery, + useCreateDomain, + useDomainCandidates, + useDomainDiscoveryJobs, + useDomainReferenceSources, + useDomains, + useStartDomainDiscovery, + useUpdateDomain, + useUpdateDomainCandidateStatus, +} from "@/hooks/useDomains"; +import type { + DomainCandidate, + DomainDiscoveryJob, + DomainReferenceSource, + DomainSummary, +} from "@/lib/api/domains"; +import { cn } from "@/lib/utils"; + +type SchemaKind = "entity_type" | "predicate" | "attribute" | "alias"; +type SchemaPayload = Pick< + DomainSummary, + "entity_types" | "predicates" | "attributes" | "aliases" +>; + +const CANDIDATE_LABELS: Record = { + entity_type: "엔티티", + predicate: "관계", + attribute: "속성", + alias: "별칭", +}; + +const JOB_STATUS_LABELS: Record = { + pending: "대기 중", + running: "분석 중", + cancel_requested: "취소 중", + canceled: "취소됨", + completed: "완료", + failed: "실패", +}; + +const COMPLETION_REASON_LABELS: Record = { + no_more_links: "더 탐색할 링크가 없어 완료되었습니다.", + max_pages_reached: "설정한 최대 페이지 수에 도달해 완료되었습니다.", + interrupted: "서버 재시작으로 실행 중이던 분석이 중단되었습니다. 다시 분석을 시작하세요.", + canceled: "사용자 요청으로 분석이 취소되었습니다.", +}; + +function parseLines(value: string): string[] { + const seen = new Set(); + return value + .split(/\n|,/) + .map((item) => item.trim()) + .filter((item) => { + if (!item || seen.has(item)) return false; + seen.add(item); + return true; + }); +} + +function uniqueSorted(values: string[]): string[] { + return Array.from(new Set(values.map((value) => value.trim()).filter(Boolean))).sort(); +} + +function numberValue(value: unknown, fallback = 0): number { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim()) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : fallback; + } + return fallback; +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) ? value.map(String) : []; +} + +function stringValue(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function pendingIds(candidates: DomainCandidate[] | undefined): string[] { + return (candidates ?? []) + .filter((candidate) => candidate.status === "pending_review") + .map((candidate) => candidate.id); +} + +function groupedCandidates(candidates: DomainCandidate[] | undefined) { + const groups: Record = { + entity_type: [], + predicate: [], + attribute: [], + alias: [], + }; + for (const candidate of candidates ?? []) { + if (!groups[candidate.candidate_type]) groups[candidate.candidate_type] = []; + groups[candidate.candidate_type].push(candidate); + } + return groups; +} + +export default function DomainBuilderPage() { + const [searchParams, setSearchParams] = useSearchParams(); + const { data: domains, isLoading: domainsLoading } = useDomains(); + const createDomain = useCreateDomain(); + const updateDomain = useUpdateDomain(); + const startDiscovery = useStartDomainDiscovery(); + const cancelDiscovery = useCancelDomainDiscovery(); + const applyCandidates = useApplyDomainCandidates(); + const updateCandidateStatus = useUpdateDomainCandidateStatus(); + + const selectedDomainName = + searchParams.get("domain") || domains?.[0]?.domain || ""; + const selectedDomain = domains?.find( + (domain) => domain.domain === selectedDomainName, + ); + const { data: referenceSources, refetch: refetchReferenceSources } = + useDomainReferenceSources(selectedDomainName || undefined); + const { data: jobs, refetch: refetchJobs } = + useDomainDiscoveryJobs(selectedDomainName || undefined); + const { data: candidates, refetch: refetchCandidates } = + useDomainCandidates(selectedDomainName || undefined); + + const [newDomain, setNewDomain] = useState(""); + const [newDescription, setNewDescription] = useState(""); + const [seedUrls, setSeedUrls] = useState(""); + const [maxPages, setMaxPages] = useState(20); + const [maxDepth, setMaxDepth] = useState(1); + const [sameDomainOnly, setSameDomainOnly] = useState(true); + const [forceRecrawl, setForceRecrawl] = useState(false); + const [selectedCandidateIds, setSelectedCandidateIds] = useState([]); + const [schemaDrafts, setSchemaDrafts] = useState>({ + entity_type: "", + predicate: "", + attribute: "", + alias: "", + }); + + useEffect(() => { + setSelectedCandidateIds([]); + }, [selectedDomainName]); + + const latestJob = jobs?.[0]; + const activeJob = jobs?.find((job) => + ["pending", "running", "cancel_requested"].includes(job.status), + ); + const groups = useMemo(() => groupedCandidates(candidates), [candidates]); + const pendingCandidateIds = useMemo(() => pendingIds(candidates), [candidates]); + const pendingCount = pendingCandidateIds.length; + const analyzedUrls = useMemo( + () => + new Set( + (referenceSources ?? []) + .filter((source) => source.last_crawled_at) + .map((source) => source.url), + ), + [referenceSources], + ); + const typedUrls = parseLines(seedUrls); + const duplicateTypedUrls = typedUrls.filter((url) => analyzedUrls.has(url)); + + const selectDomain = (domain: string) => { + setSearchParams({ domain }); + }; + + const refreshAll = () => { + void refetchJobs(); + void refetchCandidates(); + void refetchReferenceSources(); + }; + + const createNewDomain = async () => { + const domain = newDomain.trim().toLowerCase(); + if (!domain) { + toast.error("도메인 ID를 입력하세요."); + return; + } + try { + const created = await createDomain.mutateAsync({ + domain, + description: newDescription.trim() || null, + entity_types: [], + predicates: [], + attributes: [], + aliases: {}, + }); + toast.success("도메인을 만들고 선택했습니다."); + setNewDomain(""); + setNewDescription(""); + selectDomain(created.domain); + } catch (error) { + toast.error((error as Error).message); + } + }; + + const updateDomainSchema = async (next: SchemaPayload, message: string) => { + if (!selectedDomain) return; + try { + await updateDomain.mutateAsync({ + domain: selectedDomain.domain, + body: { + description: selectedDomain.description ?? null, + entity_types: next.entity_types, + predicates: next.predicates, + attributes: next.attributes, + aliases: next.aliases, + }, + }); + toast.success(message); + refreshAll(); + } catch (error) { + toast.error((error as Error).message); + } + }; + + const addSchemaItem = async (kind: SchemaKind) => { + if (!selectedDomain) { + toast.error("먼저 도메인을 선택하세요."); + return; + } + const rawValue = schemaDrafts[kind].trim(); + if (!rawValue) { + toast.error("추가할 값을 입력하세요."); + return; + } + + const next: SchemaPayload = { + entity_types: [...selectedDomain.entity_types], + predicates: [...selectedDomain.predicates], + attributes: [...selectedDomain.attributes], + aliases: { ...selectedDomain.aliases }, + }; + + if (kind === "entity_type") { + next.entity_types = uniqueSorted([...next.entity_types, rawValue]); + } else if (kind === "predicate") { + next.predicates = uniqueSorted([...next.predicates, rawValue]); + } else if (kind === "attribute") { + next.attributes = uniqueSorted([...next.attributes, rawValue]); + } else { + const [rawKey, ...rawTarget] = rawValue.split("="); + const key = rawKey.trim(); + const target = rawTarget.join("=").trim(); + if (!key || !target) { + toast.error("별칭은 source=target 형식으로 입력하세요."); + return; + } + next.aliases[key] = target; + } + + await updateDomainSchema(next, "도메인 스키마에 추가했습니다."); + setSchemaDrafts((current) => ({ ...current, [kind]: "" })); + }; + + const removeSchemaItem = async (kind: SchemaKind, value: string) => { + if (!selectedDomain) return; + const next: SchemaPayload = { + entity_types: [...selectedDomain.entity_types], + predicates: [...selectedDomain.predicates], + attributes: [...selectedDomain.attributes], + aliases: { ...selectedDomain.aliases }, + }; + + if (kind === "entity_type") { + next.entity_types = next.entity_types.filter((item) => item !== value); + } else if (kind === "predicate") { + next.predicates = next.predicates.filter((item) => item !== value); + for (const [alias, target] of Object.entries(next.aliases)) { + if (target === value) delete next.aliases[alias]; + } + } else if (kind === "attribute") { + next.attributes = next.attributes.filter((item) => item !== value); + } else { + delete next.aliases[value]; + } + + await updateDomainSchema(next, "도메인 스키마에서 삭제했습니다."); + }; + + const runDiscovery = async () => { + if (!selectedDomainName) { + toast.error("먼저 도메인을 선택하거나 생성하세요."); + return; + } + const urls = parseLines(seedUrls); + if (urls.length === 0) { + toast.error("참고 사이트 URL을 1개 이상 입력하세요."); + return; + } + try { + const job = await startDiscovery.mutateAsync({ + domain: selectedDomainName, + body: { + seed_urls: urls, + max_pages: maxPages, + max_depth: maxDepth, + same_domain_only: sameDomainOnly, + fetcher: "requests", + respect_robots_txt: false, + force_recrawl: forceRecrawl, + }, + }); + const skipped = stringArray(job.progress.skipped_seed_urls); + if (skipped.length > 0 && job.status === "completed") { + toast.success("이미 분석한 참고 사이트라 재크롤링을 건너뛰었습니다."); + } else if (skipped.length > 0) { + toast.success(`${skipped.length}개 중복 URL을 제외하고 분석을 시작했습니다.`); + } else { + toast.success("참고 사이트 분석을 시작했습니다."); + } + setSeedUrls(""); + refreshAll(); + } catch (error) { + toast.error((error as Error).message); + } + }; + + const cancelActiveJob = async () => { + if (!selectedDomainName || !activeJob) return; + try { + await cancelDiscovery.mutateAsync({ + domain: selectedDomainName, + jobId: activeJob.job_id, + }); + toast.success("분석 취소를 요청했습니다."); + void refetchJobs(); + } catch (error) { + toast.error((error as Error).message); + } + }; + + const toggleCandidate = (candidateId: string) => { + setSelectedCandidateIds((current) => + current.includes(candidateId) + ? current.filter((id) => id !== candidateId) + : [...current, candidateId], + ); + }; + + const selectAllPending = () => { + setSelectedCandidateIds(pendingCandidateIds); + }; + + const clearSelected = () => { + setSelectedCandidateIds([]); + }; + + const toggleGroup = (rows: DomainCandidate[]) => { + const ids = rows + .filter((candidate) => candidate.status === "pending_review") + .map((candidate) => candidate.id); + const allSelected = ids.every((id) => selectedCandidateIds.includes(id)); + setSelectedCandidateIds((current) => + allSelected + ? current.filter((id) => !ids.includes(id)) + : Array.from(new Set([...current, ...ids])), + ); + }; + + const applySelected = async () => { + if (!selectedDomainName || selectedCandidateIds.length === 0) return; + try { + const result = await applyCandidates.mutateAsync({ + domain: selectedDomainName, + candidateIds: selectedCandidateIds, + }); + const total = Object.values(result.applied).reduce((sum, value) => sum + value, 0); + toast.success(`${total}개 후보를 도메인에 반영했습니다.`); + setSelectedCandidateIds([]); + refreshAll(); + } catch (error) { + toast.error((error as Error).message); + } + }; + + const rejectCandidate = async (candidateId: string) => { + if (!selectedDomainName) return; + try { + await updateCandidateStatus.mutateAsync({ + domain: selectedDomainName, + candidateId, + status: "rejected", + }); + toast.success("후보를 제외했습니다."); + } catch (error) { + toast.error((error as Error).message); + } + }; + + return ( +
+
+
+

도메인 관리

+

+ 도메인을 선택하거나 생성한 뒤 현재 스키마를 직접 편집하고, 참고 사이트 분석에서 발견한 후보를 검토해 반영합니다. +

+
+ +
+ +
+ + +
+ + setSchemaDrafts((current) => ({ ...current, [kind]: value })) + } + onAdd={addSchemaItem} + onRemove={removeSchemaItem} + /> + + + + + + 2. 참고 사이트 분석 + + + + +
+
+ +