domain builder
This commit is contained in:
LASTA_DEV01\lasta
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
|
||||
@@ -28,7 +29,8 @@ from crawler_platform.app.core.database.repository import (
|
||||
from crawler_platform.app.core.database.session import session_scope
|
||||
from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
|
||||
from crawler_platform.app.core.extractor.factory import extractor_for_domain
|
||||
from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, ontology_for_domain
|
||||
from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, Ontology, ontology_for_domain
|
||||
from crawler_platform.app.core.ontology.domain_discovery import DomainDiscoveryService
|
||||
from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
|
||||
from crawler_platform.app.core.ontology.mapper import ontology_to_dict
|
||||
from crawler_platform.app.core.ontology.registry import OntologyRegistry
|
||||
@@ -38,6 +40,9 @@ from crawler_platform.app.core.research.graph_research_loop import GraphResearch
|
||||
from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload
|
||||
|
||||
|
||||
DOMAIN_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{1,79}$")
|
||||
|
||||
|
||||
class CrawlRequest(BaseModel):
|
||||
config_path: str
|
||||
source_name: str
|
||||
@@ -149,6 +154,41 @@ class CreateProjectInlineRequest(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
class DomainDefinitionRequest(BaseModel):
|
||||
domain: str
|
||||
description: str | None = None
|
||||
entity_types: list[str] = Field(default_factory=list)
|
||||
predicates: list[str] = Field(default_factory=list)
|
||||
attributes: list[str] = Field(default_factory=list)
|
||||
aliases: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class DomainDefinitionUpdateRequest(BaseModel):
|
||||
description: str | None = None
|
||||
entity_types: list[str] = Field(default_factory=list)
|
||||
predicates: list[str] = Field(default_factory=list)
|
||||
attributes: list[str] = Field(default_factory=list)
|
||||
aliases: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class DomainDiscoveryRequest(BaseModel):
|
||||
seed_urls: list[str] = Field(default_factory=list)
|
||||
max_pages: int = 20
|
||||
max_depth: int = 1
|
||||
same_domain_only: bool = True
|
||||
fetcher: str = "requests"
|
||||
respect_robots_txt: bool = False
|
||||
force_recrawl: bool = False
|
||||
|
||||
|
||||
class DomainCandidateApplyRequest(BaseModel):
|
||||
candidate_ids: list[int] = Field(default_factory=list)
|
||||
|
||||
|
||||
class DomainCandidateStatusRequest(BaseModel):
|
||||
status: str
|
||||
|
||||
|
||||
class ResetProjectRequest(BaseModel):
|
||||
config_path: str
|
||||
project_name: str | None = None
|
||||
@@ -165,6 +205,250 @@ def source_model_to_config(source: models.Source) -> SourceConfig:
|
||||
)
|
||||
|
||||
|
||||
def normalize_domain_name(domain: str) -> str:
|
||||
normalized = domain.strip().lower()
|
||||
if not DOMAIN_NAME_PATTERN.match(normalized):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Domain must be 2-80 chars using lowercase letters, numbers, underscore, or hyphen.",
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def clean_string_list(values: list[str]) -> list[str]:
|
||||
cleaned: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for value in values:
|
||||
item = value.strip()
|
||||
if not item or item in seen:
|
||||
continue
|
||||
cleaned.append(item)
|
||||
seen.add(item)
|
||||
return cleaned
|
||||
|
||||
|
||||
def clean_aliases(values: dict[str, str]) -> dict[str, str]:
|
||||
return {
|
||||
key.strip(): value.strip()
|
||||
for key, value in values.items()
|
||||
if key.strip() and value.strip()
|
||||
}
|
||||
|
||||
|
||||
def domain_row_to_ontology(row: models.DomainDefinition):
|
||||
return Ontology(
|
||||
domain=row.domain,
|
||||
entity_types=clean_string_list(list(row.entity_types or [])),
|
||||
predicates=clean_string_list(list(row.predicates or [])),
|
||||
attributes=clean_string_list(list(row.attributes or [])),
|
||||
aliases=clean_aliases(dict(row.aliases or {})),
|
||||
)
|
||||
|
||||
|
||||
def domain_summary_payload(ont, *, is_builtin: bool, is_custom: bool, description: str | None = None):
|
||||
return {
|
||||
"domain": ont.domain,
|
||||
"description": description,
|
||||
"entity_types": list(ont.entity_types),
|
||||
"predicates": list(ont.predicates),
|
||||
"attribute_count": len(ont.attributes),
|
||||
"attributes": list(ont.attributes),
|
||||
"aliases": dict(ont.aliases),
|
||||
"is_builtin": is_builtin,
|
||||
"is_custom": is_custom,
|
||||
}
|
||||
|
||||
|
||||
def upsert_domain_definition(
|
||||
session,
|
||||
domain: str,
|
||||
payload: DomainDefinitionUpdateRequest,
|
||||
) -> models.DomainDefinition:
|
||||
normalized_domain = normalize_domain_name(domain)
|
||||
row = session.scalar(
|
||||
select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
|
||||
)
|
||||
if row is None:
|
||||
row = models.DomainDefinition(domain=normalized_domain)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
row.description = payload.description.strip() if payload.description else None
|
||||
row.entity_types = clean_string_list(payload.entity_types)
|
||||
row.predicates = clean_string_list(payload.predicates)
|
||||
row.attributes = clean_string_list(payload.attributes)
|
||||
row.aliases = clean_aliases(payload.aliases)
|
||||
row.is_builtin_override = normalized_domain in DOMAIN_ONTOLOGIES
|
||||
row.updated_at = models.utcnow()
|
||||
return row
|
||||
|
||||
|
||||
def ensure_domain_definition(session, domain: str) -> models.DomainDefinition:
|
||||
normalized_domain = normalize_domain_name(domain)
|
||||
row = session.scalar(
|
||||
select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
|
||||
)
|
||||
if row is not None:
|
||||
return row
|
||||
|
||||
base = DOMAIN_ONTOLOGIES.get(normalized_domain)
|
||||
row = models.DomainDefinition(
|
||||
domain=normalized_domain,
|
||||
entity_types=list(base.entity_types) if base else [],
|
||||
predicates=list(base.predicates) if base else [],
|
||||
attributes=list(base.attributes) if base else [],
|
||||
aliases=dict(base.aliases) if base else {},
|
||||
is_builtin_override=normalized_domain in DOMAIN_ONTOLOGIES,
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return row
|
||||
|
||||
|
||||
def ontology_for_domain_from_db(session, domain: str):
|
||||
normalized_domain = normalize_domain_name(domain)
|
||||
row = session.scalar(
|
||||
select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
|
||||
)
|
||||
if row is not None:
|
||||
return domain_row_to_ontology(row)
|
||||
return ontology_for_domain(normalized_domain)
|
||||
|
||||
|
||||
def clean_seed_urls(values: list[str]) -> list[str]:
|
||||
cleaned: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for value in values:
|
||||
url = value.strip()
|
||||
if not url or url in seen:
|
||||
continue
|
||||
if not (url.startswith("http://") or url.startswith("https://") or url.startswith("file://")):
|
||||
raise HTTPException(status_code=400, detail=f"Unsupported URL: {url}")
|
||||
cleaned.append(url)
|
||||
seen.add(url)
|
||||
if not cleaned:
|
||||
raise HTTPException(status_code=400, detail="At least one reference URL is required.")
|
||||
return cleaned
|
||||
|
||||
|
||||
def domain_discovery_job_payload(job: models.DomainDiscoveryJob) -> dict[str, Any]:
|
||||
return {
|
||||
"job_id": job.id,
|
||||
"domain": job.domain,
|
||||
"status": job.status,
|
||||
"seed_urls": list(job.seed_urls or []),
|
||||
"max_pages": job.max_pages,
|
||||
"max_depth": job.max_depth,
|
||||
"same_domain_only": job.same_domain_only,
|
||||
"fetcher": job.fetcher,
|
||||
"respect_robots_txt": job.respect_robots_txt,
|
||||
"error": job.error,
|
||||
"progress": job.progress or {},
|
||||
"result_summary": job.result_summary or {},
|
||||
"created_at": job.created_at.isoformat() if job.created_at else None,
|
||||
"started_at": job.started_at.isoformat() if job.started_at else None,
|
||||
"finished_at": job.finished_at.isoformat() if job.finished_at else None,
|
||||
}
|
||||
|
||||
|
||||
def domain_reference_source_payload(row: models.DomainReferenceSource) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"domain": row.domain,
|
||||
"url": row.url,
|
||||
"label": row.label,
|
||||
"status": row.status,
|
||||
"last_crawled_at": row.last_crawled_at.isoformat() if row.last_crawled_at else None,
|
||||
"metadata": row.metadata_json or {},
|
||||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||||
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
def already_crawled_reference_urls(session, domain: str, seed_urls: list[str]) -> set[str]:
|
||||
if not seed_urls:
|
||||
return set()
|
||||
rows = session.scalars(
|
||||
select(models.DomainReferenceSource).where(
|
||||
models.DomainReferenceSource.domain == domain,
|
||||
models.DomainReferenceSource.url.in_(seed_urls),
|
||||
models.DomainReferenceSource.status == "active",
|
||||
models.DomainReferenceSource.last_crawled_at.is_not(None),
|
||||
)
|
||||
).all()
|
||||
return {row.url for row in rows}
|
||||
|
||||
|
||||
def domain_candidate_payload(row: models.DomainSchemaCandidate, evidence_rows: list[models.DomainCandidateEvidence]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"domain": row.domain,
|
||||
"job_id": row.job_id,
|
||||
"candidate_type": row.candidate_type,
|
||||
"name": row.name,
|
||||
"description": row.description,
|
||||
"confidence": row.confidence,
|
||||
"occurrence_count": row.occurrence_count,
|
||||
"status": row.status,
|
||||
"metadata": row.metadata_json or {},
|
||||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||||
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
||||
"evidence": [
|
||||
{
|
||||
"id": evidence.id,
|
||||
"job_id": evidence.job_id,
|
||||
"url": evidence.url,
|
||||
"title": evidence.title,
|
||||
"snippet": evidence.snippet,
|
||||
"metadata": evidence.metadata_json or {},
|
||||
"created_at": evidence.created_at.isoformat() if evidence.created_at else None,
|
||||
}
|
||||
for evidence in evidence_rows
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def run_domain_discovery_job(database_url: str, job_id: int) -> None:
|
||||
try:
|
||||
with session_scope(database_url) as session:
|
||||
DomainDiscoveryService(session).run_job(job_id)
|
||||
except Exception as exc:
|
||||
with session_scope(database_url) as session:
|
||||
job = session.get(models.DomainDiscoveryJob, job_id)
|
||||
if job is not None:
|
||||
job.status = "failed"
|
||||
job.error = str(exc)
|
||||
job.finished_at = models.utcnow()
|
||||
progress = dict(job.progress or {})
|
||||
progress["errors"] = [*progress.get("errors", []), str(exc)]
|
||||
job.progress = progress
|
||||
|
||||
|
||||
def recover_interrupted_domain_discovery_jobs(database_url: str) -> None:
|
||||
"""Mark domain discovery jobs that cannot survive a server restart."""
|
||||
with session_scope(database_url) as session:
|
||||
rows = session.scalars(
|
||||
select(models.DomainDiscoveryJob).where(
|
||||
models.DomainDiscoveryJob.status.in_(
|
||||
["pending", "running", "cancel_requested"]
|
||||
)
|
||||
)
|
||||
).all()
|
||||
for job in rows:
|
||||
progress = dict(job.progress or {})
|
||||
errors = list(progress.get("errors") or [])
|
||||
if job.status == "cancel_requested":
|
||||
job.status = "canceled"
|
||||
progress["completion_reason"] = "canceled"
|
||||
else:
|
||||
job.status = "failed"
|
||||
job.error = "Server restarted before this background analysis finished."
|
||||
progress["completion_reason"] = "interrupted"
|
||||
errors.append(job.error)
|
||||
progress["errors"] = errors[-10:]
|
||||
job.progress = progress
|
||||
job.finished_at = models.utcnow()
|
||||
|
||||
|
||||
def project_config_from_project_row(session, project: models.Project) -> ProjectConfig:
|
||||
config_dict = dict(project.config or {})
|
||||
if not config_dict:
|
||||
@@ -195,6 +479,32 @@ def project_config_to_dict(config: ProjectConfig) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def delete_project_data(session, project_id: int, include_project: bool = False) -> dict[str, int]:
|
||||
repo = KnowledgeRepository(session)
|
||||
deleted = repo.reset_project_runtime_data(project_id)
|
||||
for table_model in [
|
||||
models.OntologyProposal,
|
||||
models.OntologyTriple,
|
||||
models.OntologyRelationType,
|
||||
models.OntologyEntityType,
|
||||
models.Source,
|
||||
]:
|
||||
count = (
|
||||
session.query(table_model)
|
||||
.filter(table_model.project_id == project_id)
|
||||
.delete(synchronize_session=False)
|
||||
)
|
||||
deleted[table_model.__tablename__] = deleted.get(table_model.__tablename__, 0) + int(count or 0)
|
||||
if include_project:
|
||||
project_count = (
|
||||
session.query(models.Project)
|
||||
.filter(models.Project.id == project_id)
|
||||
.delete(synchronize_session=False)
|
||||
)
|
||||
deleted[models.Project.__tablename__] = int(project_count or 0)
|
||||
return deleted
|
||||
|
||||
|
||||
class CreateEntityRequest(BaseModel):
|
||||
entity_type: str
|
||||
name: str
|
||||
@@ -485,6 +795,8 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A
|
||||
|
||||
|
||||
def register_routes(app, database_url: str) -> None:
|
||||
recover_interrupted_domain_discovery_jobs(database_url)
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"ok": True}
|
||||
@@ -520,16 +832,298 @@ def register_routes(app, database_url: str) -> None:
|
||||
|
||||
@app.get("/domains")
|
||||
def list_domains():
|
||||
"""Available pre-defined ontology domains for project creation."""
|
||||
return [
|
||||
{
|
||||
"domain": ont.domain,
|
||||
"entity_types": list(ont.entity_types),
|
||||
"predicates": list(ont.predicates),
|
||||
"attribute_count": len(ont.attributes),
|
||||
"""Available ontology domains for project creation."""
|
||||
with session_scope(database_url) as session:
|
||||
rows = session.scalars(select(models.DomainDefinition)).all()
|
||||
custom_by_domain = {row.domain: row for row in rows}
|
||||
payloads = []
|
||||
for domain, built_in in DOMAIN_ONTOLOGIES.items():
|
||||
row = custom_by_domain.pop(domain, None)
|
||||
if row is not None:
|
||||
payloads.append(
|
||||
domain_summary_payload(
|
||||
domain_row_to_ontology(row),
|
||||
is_builtin=True,
|
||||
is_custom=True,
|
||||
description=row.description,
|
||||
)
|
||||
)
|
||||
else:
|
||||
payloads.append(
|
||||
domain_summary_payload(
|
||||
built_in,
|
||||
is_builtin=True,
|
||||
is_custom=False,
|
||||
)
|
||||
)
|
||||
for row in sorted(custom_by_domain.values(), key=lambda item: item.domain):
|
||||
payloads.append(
|
||||
domain_summary_payload(
|
||||
domain_row_to_ontology(row),
|
||||
is_builtin=False,
|
||||
is_custom=True,
|
||||
description=row.description,
|
||||
)
|
||||
)
|
||||
return payloads
|
||||
|
||||
@app.post("/domains")
|
||||
def create_domain(request: DomainDefinitionRequest):
|
||||
domain = normalize_domain_name(request.domain)
|
||||
with session_scope(database_url) as session:
|
||||
existing = session.scalar(
|
||||
select(models.DomainDefinition).where(models.DomainDefinition.domain == domain)
|
||||
)
|
||||
if existing is not None or domain in DOMAIN_ONTOLOGIES:
|
||||
raise HTTPException(status_code=409, detail=f"Domain '{domain}' already exists.")
|
||||
row = upsert_domain_definition(
|
||||
session,
|
||||
domain,
|
||||
DomainDefinitionUpdateRequest(
|
||||
description=request.description,
|
||||
entity_types=request.entity_types,
|
||||
predicates=request.predicates,
|
||||
attributes=request.attributes,
|
||||
aliases=request.aliases,
|
||||
),
|
||||
)
|
||||
return domain_summary_payload(
|
||||
domain_row_to_ontology(row),
|
||||
is_builtin=domain in DOMAIN_ONTOLOGIES,
|
||||
is_custom=True,
|
||||
description=row.description,
|
||||
)
|
||||
|
||||
@app.put("/domains/{domain}")
|
||||
def update_domain(domain: str, request: DomainDefinitionUpdateRequest):
|
||||
with session_scope(database_url) as session:
|
||||
row = upsert_domain_definition(session, domain, request)
|
||||
return domain_summary_payload(
|
||||
domain_row_to_ontology(row),
|
||||
is_builtin=row.domain in DOMAIN_ONTOLOGIES,
|
||||
is_custom=True,
|
||||
description=row.description,
|
||||
)
|
||||
|
||||
@app.get("/domains/{domain}/discovery/jobs")
|
||||
def list_domain_discovery_jobs(domain: str, limit: int = 20):
|
||||
normalized_domain = normalize_domain_name(domain)
|
||||
with session_scope(database_url) as session:
|
||||
rows = session.scalars(
|
||||
select(models.DomainDiscoveryJob)
|
||||
.where(models.DomainDiscoveryJob.domain == normalized_domain)
|
||||
.order_by(models.DomainDiscoveryJob.created_at.desc())
|
||||
.limit(max(min(limit, 100), 1))
|
||||
).all()
|
||||
return [domain_discovery_job_payload(row) for row in rows]
|
||||
|
||||
@app.get("/domains/{domain}/reference-sources")
|
||||
def list_domain_reference_sources(domain: str):
|
||||
normalized_domain = normalize_domain_name(domain)
|
||||
with session_scope(database_url) as session:
|
||||
rows = session.scalars(
|
||||
select(models.DomainReferenceSource)
|
||||
.where(models.DomainReferenceSource.domain == normalized_domain)
|
||||
.order_by(models.DomainReferenceSource.updated_at.desc())
|
||||
).all()
|
||||
return [domain_reference_source_payload(row) for row in rows]
|
||||
|
||||
@app.post("/domains/{domain}/discovery/jobs")
|
||||
def create_domain_discovery_job(
|
||||
domain: str,
|
||||
request: DomainDiscoveryRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
):
|
||||
normalized_domain = normalize_domain_name(domain)
|
||||
seed_urls = clean_seed_urls(request.seed_urls)
|
||||
with session_scope(database_url) as session:
|
||||
ensure_domain_definition(session, normalized_domain)
|
||||
skipped_seed_urls: list[str] = []
|
||||
if not request.force_recrawl:
|
||||
skipped = already_crawled_reference_urls(session, normalized_domain, seed_urls)
|
||||
skipped_seed_urls = [url for url in seed_urls if url in skipped]
|
||||
seed_urls = [url for url in seed_urls if url not in skipped]
|
||||
|
||||
if not seed_urls:
|
||||
job = models.DomainDiscoveryJob(
|
||||
domain=normalized_domain,
|
||||
status="completed",
|
||||
seed_urls=[],
|
||||
max_pages=max(min(request.max_pages, 200), 1),
|
||||
max_depth=max(min(request.max_depth, 5), 0),
|
||||
same_domain_only=request.same_domain_only,
|
||||
fetcher=request.fetcher if request.fetcher in {"requests", "playwright", "browser"} else "requests",
|
||||
respect_robots_txt=request.respect_robots_txt,
|
||||
progress={
|
||||
"visited_count": 0,
|
||||
"queued_count": 0,
|
||||
"candidate_count": 0,
|
||||
"observation_count": 0,
|
||||
"max_pages": max(min(request.max_pages, 200), 1),
|
||||
"skipped_seed_urls": skipped_seed_urls,
|
||||
"errors": [],
|
||||
"pages": [],
|
||||
},
|
||||
result_summary={
|
||||
"visited_count": 0,
|
||||
"candidate_count": 0,
|
||||
"skipped_seed_urls": skipped_seed_urls,
|
||||
},
|
||||
finished_at=models.utcnow(),
|
||||
)
|
||||
session.add(job)
|
||||
session.flush()
|
||||
return domain_discovery_job_payload(job)
|
||||
|
||||
job = models.DomainDiscoveryJob(
|
||||
domain=normalized_domain,
|
||||
status="pending",
|
||||
seed_urls=seed_urls,
|
||||
max_pages=max(min(request.max_pages, 200), 1),
|
||||
max_depth=max(min(request.max_depth, 5), 0),
|
||||
same_domain_only=request.same_domain_only,
|
||||
fetcher=request.fetcher if request.fetcher in {"requests", "playwright", "browser"} else "requests",
|
||||
respect_robots_txt=request.respect_robots_txt,
|
||||
progress={
|
||||
"visited_count": 0,
|
||||
"queued_count": len(seed_urls),
|
||||
"candidate_count": 0,
|
||||
"observation_count": 0,
|
||||
"max_pages": max(min(request.max_pages, 200), 1),
|
||||
"skipped_seed_urls": skipped_seed_urls,
|
||||
"errors": [],
|
||||
"pages": [],
|
||||
},
|
||||
)
|
||||
session.add(job)
|
||||
session.flush()
|
||||
response = domain_discovery_job_payload(job)
|
||||
background_tasks.add_task(run_domain_discovery_job, database_url, response["job_id"])
|
||||
return response
|
||||
|
||||
@app.get("/domains/{domain}/discovery/jobs/{job_id}")
|
||||
def domain_discovery_job(domain: str, job_id: int):
|
||||
normalized_domain = normalize_domain_name(domain)
|
||||
with session_scope(database_url) as session:
|
||||
job = session.get(models.DomainDiscoveryJob, job_id)
|
||||
if job is None or job.domain != normalized_domain:
|
||||
raise HTTPException(status_code=404, detail="domain discovery job not found")
|
||||
return domain_discovery_job_payload(job)
|
||||
|
||||
@app.post("/domains/{domain}/discovery/jobs/{job_id}/cancel")
|
||||
def cancel_domain_discovery_job(domain: str, job_id: int):
|
||||
normalized_domain = normalize_domain_name(domain)
|
||||
with session_scope(database_url) as session:
|
||||
job = session.get(models.DomainDiscoveryJob, job_id)
|
||||
if job is None or job.domain != normalized_domain:
|
||||
raise HTTPException(status_code=404, detail="domain discovery job not found")
|
||||
if job.status in {"completed", "failed", "canceled"}:
|
||||
return domain_discovery_job_payload(job)
|
||||
job.status = "cancel_requested"
|
||||
job.error = "cancel requested by user"
|
||||
return domain_discovery_job_payload(job)
|
||||
|
||||
@app.get("/domains/{domain}/candidates")
|
||||
def list_domain_candidates(domain: str, status: str | None = None, limit: int = 200):
|
||||
normalized_domain = normalize_domain_name(domain)
|
||||
with session_scope(database_url) as session:
|
||||
query = select(models.DomainSchemaCandidate).where(
|
||||
models.DomainSchemaCandidate.domain == normalized_domain
|
||||
)
|
||||
if status:
|
||||
query = query.where(models.DomainSchemaCandidate.status == status)
|
||||
rows = session.scalars(
|
||||
query.order_by(
|
||||
models.DomainSchemaCandidate.status,
|
||||
models.DomainSchemaCandidate.candidate_type,
|
||||
models.DomainSchemaCandidate.confidence.desc(),
|
||||
models.DomainSchemaCandidate.name,
|
||||
).limit(max(min(limit, 500), 1))
|
||||
).all()
|
||||
payload = []
|
||||
for row in rows:
|
||||
evidence = session.scalars(
|
||||
select(models.DomainCandidateEvidence)
|
||||
.where(models.DomainCandidateEvidence.candidate_id == row.id)
|
||||
.order_by(models.DomainCandidateEvidence.created_at.desc())
|
||||
.limit(3)
|
||||
).all()
|
||||
payload.append(domain_candidate_payload(row, evidence))
|
||||
return payload
|
||||
|
||||
@app.post("/domains/{domain}/candidates/apply")
|
||||
def apply_domain_candidates(domain: str, request: DomainCandidateApplyRequest):
|
||||
normalized_domain = normalize_domain_name(domain)
|
||||
if not request.candidate_ids:
|
||||
raise HTTPException(status_code=400, detail="No candidates selected.")
|
||||
with session_scope(database_url) as session:
|
||||
definition = ensure_domain_definition(session, normalized_domain)
|
||||
rows = session.scalars(
|
||||
select(models.DomainSchemaCandidate).where(
|
||||
models.DomainSchemaCandidate.domain == normalized_domain,
|
||||
models.DomainSchemaCandidate.id.in_(request.candidate_ids),
|
||||
)
|
||||
).all()
|
||||
entity_types = set(definition.entity_types or [])
|
||||
predicates = set(definition.predicates or [])
|
||||
attributes = set(definition.attributes or [])
|
||||
aliases = dict(definition.aliases or {})
|
||||
applied = {"entity_type": 0, "predicate": 0, "attribute": 0, "alias": 0}
|
||||
for row in rows:
|
||||
if row.candidate_type == "entity_type":
|
||||
entity_types.add(row.name)
|
||||
applied["entity_type"] += 1
|
||||
elif row.candidate_type == "predicate":
|
||||
predicates.add(row.name)
|
||||
applied["predicate"] += 1
|
||||
elif row.candidate_type == "attribute":
|
||||
attributes.add(row.name)
|
||||
applied["attribute"] += 1
|
||||
elif row.candidate_type == "alias":
|
||||
metadata = dict(row.metadata_json or {})
|
||||
targets = metadata.get("targets") or []
|
||||
target = metadata.get("target") or (targets[0] if targets else None)
|
||||
if target:
|
||||
aliases[row.name] = str(target)
|
||||
applied["alias"] += 1
|
||||
row.status = "approved"
|
||||
row.updated_at = models.utcnow()
|
||||
definition.entity_types = sorted(entity_types)
|
||||
definition.predicates = sorted(predicates)
|
||||
definition.attributes = sorted(attributes)
|
||||
definition.aliases = aliases
|
||||
definition.is_builtin_override = normalized_domain in DOMAIN_ONTOLOGIES
|
||||
definition.updated_at = models.utcnow()
|
||||
return {
|
||||
"ok": True,
|
||||
"domain": normalized_domain,
|
||||
"applied": applied,
|
||||
"definition": domain_summary_payload(
|
||||
domain_row_to_ontology(definition),
|
||||
is_builtin=normalized_domain in DOMAIN_ONTOLOGIES,
|
||||
is_custom=True,
|
||||
description=definition.description,
|
||||
),
|
||||
}
|
||||
for ont in DOMAIN_ONTOLOGIES.values()
|
||||
]
|
||||
|
||||
@app.post("/domains/{domain}/candidates/{candidate_id}/status")
|
||||
def update_domain_candidate_status(domain: str, candidate_id: int, request: DomainCandidateStatusRequest):
|
||||
normalized_domain = normalize_domain_name(domain)
|
||||
if request.status not in {"pending_review", "approved", "rejected"}:
|
||||
raise HTTPException(status_code=400, detail="Unsupported candidate status.")
|
||||
with session_scope(database_url) as session:
|
||||
row = session.get(models.DomainSchemaCandidate, candidate_id)
|
||||
if row is None or row.domain != normalized_domain:
|
||||
raise HTTPException(status_code=404, detail="candidate not found")
|
||||
row.status = request.status
|
||||
row.updated_at = models.utcnow()
|
||||
evidence = session.scalars(
|
||||
select(models.DomainCandidateEvidence)
|
||||
.where(models.DomainCandidateEvidence.candidate_id == row.id)
|
||||
.order_by(models.DomainCandidateEvidence.created_at.desc())
|
||||
.limit(3)
|
||||
).all()
|
||||
return domain_candidate_payload(row, evidence)
|
||||
|
||||
@app.post("/projects/reset")
|
||||
def reset_project(request: ResetProjectRequest):
|
||||
@@ -568,6 +1162,35 @@ def register_routes(app, database_url: str) -> None:
|
||||
"deleted": deleted,
|
||||
}
|
||||
|
||||
@app.post("/projects/{project_name}/reset")
|
||||
def reset_project_by_name(project_name: str):
|
||||
with session_scope(database_url) as session:
|
||||
repo = KnowledgeRepository(session)
|
||||
project = repo.get_project(project_name)
|
||||
config = project_config_from_project_row(session, project)
|
||||
deleted = repo.reset_project_runtime_data(project.id)
|
||||
project = repo.upsert_project(config)
|
||||
return {
|
||||
"ok": True,
|
||||
"name": project.name,
|
||||
"domain": project.domain,
|
||||
"reset": True,
|
||||
"deleted": deleted,
|
||||
}
|
||||
|
||||
@app.delete("/projects/{project_name}")
|
||||
def delete_project(project_name: str):
|
||||
with session_scope(database_url) as session:
|
||||
project = KnowledgeRepository(session).get_project(project_name)
|
||||
domain = project.domain
|
||||
deleted = delete_project_data(session, project.id, include_project=True)
|
||||
return {
|
||||
"ok": True,
|
||||
"name": project_name,
|
||||
"domain": domain,
|
||||
"deleted": deleted,
|
||||
}
|
||||
|
||||
@app.get("/projects/{project_name}")
|
||||
def project_detail(project_name: str):
|
||||
with session_scope(database_url) as session:
|
||||
@@ -630,7 +1253,8 @@ def register_routes(app, database_url: str) -> None:
|
||||
|
||||
@app.get("/ontology/{domain}")
|
||||
def ontology(domain: str):
|
||||
return ontology_to_dict(ontology_for_domain(domain))
|
||||
with session_scope(database_url) as session:
|
||||
return ontology_to_dict(ontology_for_domain_from_db(session, domain))
|
||||
|
||||
@app.get("/projects/{project_name}/ontology/registry")
|
||||
def ontology_registry(project_name: str):
|
||||
|
||||
@@ -56,6 +56,91 @@ class Source(Base):
|
||||
project = relationship("Project", back_populates="sources")
|
||||
|
||||
|
||||
class DomainDefinition(Base):
|
||||
__tablename__ = "domain_definitions"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
domain = Column(String(80), unique=True, nullable=False, index=True)
|
||||
description = Column(Text)
|
||||
entity_types = Column(JSON, nullable=False, default=list)
|
||||
predicates = Column(JSON, nullable=False, default=list)
|
||||
attributes = Column(JSON, nullable=False, default=list)
|
||||
aliases = Column(JSON, nullable=False, default=dict)
|
||||
is_builtin_override = Column(Boolean, nullable=False, default=False)
|
||||
created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
|
||||
updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
|
||||
|
||||
|
||||
class DomainReferenceSource(Base):
|
||||
__tablename__ = "domain_reference_sources"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("domain", "url", name="uq_domain_reference_source"),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
domain = Column(String(80), nullable=False, index=True)
|
||||
url = Column(Text, nullable=False)
|
||||
label = Column(String(160))
|
||||
status = Column(String(40), nullable=False, default="active", index=True)
|
||||
last_crawled_at = Column(DateTime(timezone=True), nullable=True)
|
||||
metadata_json = Column(JSON, nullable=False, default=dict)
|
||||
created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
|
||||
updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
|
||||
|
||||
|
||||
class DomainDiscoveryJob(Base):
|
||||
__tablename__ = "domain_discovery_jobs"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
domain = Column(String(80), nullable=False, index=True)
|
||||
status = Column(String(40), nullable=False, default="pending", index=True)
|
||||
seed_urls = Column(JSON, nullable=False, default=list)
|
||||
max_pages = Column(Integer, nullable=False, default=20)
|
||||
max_depth = Column(Integer, nullable=False, default=1)
|
||||
same_domain_only = Column(Boolean, nullable=False, default=True)
|
||||
fetcher = Column(String(80), nullable=False, default="requests")
|
||||
respect_robots_txt = Column(Boolean, nullable=False, default=False)
|
||||
error = Column(Text)
|
||||
progress = Column(JSON, nullable=False, default=dict)
|
||||
result_summary = Column(JSON, nullable=False, default=dict)
|
||||
created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
|
||||
started_at = Column(DateTime(timezone=True), nullable=True)
|
||||
finished_at = Column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class DomainSchemaCandidate(Base):
|
||||
__tablename__ = "domain_schema_candidates"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("domain", "candidate_type", "name", name="uq_domain_schema_candidate"),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
domain = Column(String(80), nullable=False, index=True)
|
||||
job_id = Column(Integer, ForeignKey("domain_discovery_jobs.id"), nullable=True, index=True)
|
||||
candidate_type = Column(String(40), nullable=False, index=True)
|
||||
name = Column(String(160), nullable=False, index=True)
|
||||
description = Column(Text)
|
||||
confidence = Column(Float, nullable=False, default=0.5)
|
||||
occurrence_count = Column(Integer, nullable=False, default=1)
|
||||
status = Column(String(40), nullable=False, default="pending_review", index=True)
|
||||
metadata_json = Column(JSON, nullable=False, default=dict)
|
||||
created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
|
||||
updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
|
||||
|
||||
|
||||
class DomainCandidateEvidence(Base):
|
||||
__tablename__ = "domain_candidate_evidence"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
candidate_id = Column(Integer, ForeignKey("domain_schema_candidates.id"), nullable=False, index=True)
|
||||
job_id = Column(Integer, ForeignKey("domain_discovery_jobs.id"), nullable=True, index=True)
|
||||
url = Column(Text, nullable=False)
|
||||
title = Column(Text)
|
||||
snippet = Column(Text, nullable=False)
|
||||
metadata_json = Column(JSON, nullable=False, default=dict)
|
||||
created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
|
||||
|
||||
|
||||
class Page(Base):
|
||||
__tablename__ = "pages"
|
||||
__table_args__ = (UniqueConstraint("project_id", "url", name="uq_page_project_url"),)
|
||||
|
||||
@@ -0,0 +1,643 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter, deque
|
||||
from dataclasses import dataclass, field
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from crawler_platform.app.core.crawler.discovery import discover_links
|
||||
from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
|
||||
from crawler_platform.app.core.crawler.page_classifier import classify_page
|
||||
from crawler_platform.app.core.crawler.plugins import default_parser_registry
|
||||
from crawler_platform.app.core.database import models
|
||||
|
||||
|
||||
SCHEMA_TYPES = {"entity_type", "predicate", "attribute", "alias"}
|
||||
RELATION_HINTS = {
|
||||
"brand": ("Brand", "hasBrand"),
|
||||
"브랜드": ("Brand", "hasBrand"),
|
||||
"category": ("Category", "hasCategory"),
|
||||
"카테고리": ("Category", "hasCategory"),
|
||||
"ingredient": ("Ingredient", "hasIngredient"),
|
||||
"ingredients": ("Ingredient", "hasIngredient"),
|
||||
"성분": ("Ingredient", "hasIngredient"),
|
||||
"effect": ("Effect", "hasEffect"),
|
||||
"benefit": ("Effect", "hasEffect"),
|
||||
"효과": ("Effect", "hasEffect"),
|
||||
"효능": ("Effect", "hasEffect"),
|
||||
"warning": ("Warning", "hasWarning"),
|
||||
"주의": ("Warning", "hasWarning"),
|
||||
"review": ("Review", "hasReview"),
|
||||
"rating": ("Rating", "hasRating"),
|
||||
"image": ("Image", "hasImage"),
|
||||
"price": ("Price", "hasPrice"),
|
||||
"가격": ("Price", "hasPrice"),
|
||||
"offer": ("Offer", "hasOffer"),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CandidateEvidence:
|
||||
url: str
|
||||
title: str | None
|
||||
snippet: str
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CandidateObservation:
|
||||
candidate_type: str
|
||||
name: str
|
||||
description: str
|
||||
confidence: float
|
||||
evidence: CandidateEvidence
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class DomainDiscoveryService:
|
||||
def __init__(self, session: Session):
|
||||
self.session = session
|
||||
self.parser = default_parser_registry().get("generic")
|
||||
|
||||
def run_job(self, job_id: int) -> None:
|
||||
job = self.session.get(models.DomainDiscoveryJob, job_id)
|
||||
if job is None:
|
||||
return
|
||||
|
||||
job.status = "running"
|
||||
job.started_at = models.utcnow()
|
||||
job.progress = {
|
||||
"visited_count": 0,
|
||||
"queued_count": len(job.seed_urls or []),
|
||||
"candidate_count": 0,
|
||||
"errors": [],
|
||||
"pages": [],
|
||||
}
|
||||
self.session.commit()
|
||||
|
||||
visited: set[str] = set()
|
||||
queued: set[str] = set()
|
||||
queue: deque[tuple[str, int, str]] = deque()
|
||||
for seed_url in job.seed_urls or []:
|
||||
normalized = normalize_url(str(seed_url))
|
||||
if normalized:
|
||||
queue.append((normalized, 0, seed_host(normalized)))
|
||||
queued.add(normalized)
|
||||
self._upsert_reference_source(job.domain, normalized)
|
||||
|
||||
fetcher = make_fetcher(job.fetcher, rate_limit_per_minute=30)
|
||||
robots = RobotsPolicy()
|
||||
observation_count = 0
|
||||
errors: list[str] = []
|
||||
page_payloads: list[dict[str, Any]] = []
|
||||
|
||||
while queue and len(visited) < max(min(job.max_pages, 200), 1):
|
||||
if self._cancel_requested(job):
|
||||
job.status = "canceled"
|
||||
job.finished_at = models.utcnow()
|
||||
self.session.commit()
|
||||
return
|
||||
|
||||
url, depth, root_host = queue.popleft()
|
||||
if url in visited:
|
||||
continue
|
||||
visited.add(url)
|
||||
|
||||
if depth > max(job.max_depth, 0):
|
||||
continue
|
||||
if job.same_domain_only and seed_host(url) != root_host:
|
||||
continue
|
||||
|
||||
decision = robots.check(url, job.respect_robots_txt)
|
||||
if not decision.allowed:
|
||||
errors.append(f"{decision.reason}: {url}")
|
||||
self._save_progress(job, visited, queue, observation_count, errors, page_payloads)
|
||||
continue
|
||||
|
||||
try:
|
||||
fetched = fetcher.fetch(url)
|
||||
parsed = self.parser.parse(fetched.analysis_html, fetched.final_url or url)
|
||||
page_type = classify_page(
|
||||
fetched.final_url or url,
|
||||
parsed.title or fetched.title,
|
||||
parsed.text or parsed.raw_text,
|
||||
html=fetched.analysis_html,
|
||||
source_zones=parsed.source_zones or [],
|
||||
)
|
||||
page_observations = mine_schema_candidates(
|
||||
url=fetched.final_url or url,
|
||||
title=parsed.title or fetched.title,
|
||||
text=parsed.text or parsed.raw_text,
|
||||
html=fetched.analysis_html,
|
||||
page_type=page_type,
|
||||
)
|
||||
observation_count += len(page_observations)
|
||||
if page_observations:
|
||||
self._save_observations(job, page_observations)
|
||||
page_payloads.append(
|
||||
{
|
||||
"url": fetched.final_url or url,
|
||||
"title": parsed.title or fetched.title,
|
||||
"page_type": page_type,
|
||||
"candidate_count": len(page_observations),
|
||||
"text_length": len(parsed.text or ""),
|
||||
"warnings": [*fetched.warnings, *(parsed.extraction_warnings or [])],
|
||||
}
|
||||
)
|
||||
self._enqueue_links(
|
||||
html=fetched.analysis_html,
|
||||
base_url=fetched.final_url or url,
|
||||
root_host=root_host,
|
||||
depth=depth,
|
||||
max_depth=max(job.max_depth, 0),
|
||||
same_domain_only=job.same_domain_only,
|
||||
queue=queue,
|
||||
queued=queued,
|
||||
visited=visited,
|
||||
)
|
||||
except Exception as exc:
|
||||
errors.append(f"{url}: {exc}")
|
||||
|
||||
self._save_progress(job, visited, queue, observation_count, errors, page_payloads)
|
||||
|
||||
reached_page_limit = len(visited) >= max(min(job.max_pages, 200), 1)
|
||||
completion_reason = "max_pages_reached" if reached_page_limit else "no_more_links"
|
||||
saved = self._candidate_count(job.domain)
|
||||
job.status = "completed"
|
||||
job.finished_at = models.utcnow()
|
||||
job.result_summary = {
|
||||
"visited_count": len(visited),
|
||||
"candidate_count": saved,
|
||||
"completion_reason": completion_reason,
|
||||
"errors": errors[-20:],
|
||||
"pages": page_payloads[-50:],
|
||||
}
|
||||
self._save_progress(
|
||||
job,
|
||||
visited,
|
||||
queue,
|
||||
observation_count,
|
||||
errors,
|
||||
page_payloads,
|
||||
completion_reason=completion_reason,
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
def _save_progress(
|
||||
self,
|
||||
job: models.DomainDiscoveryJob,
|
||||
visited: set[str],
|
||||
queue: deque[tuple[str, int, str]],
|
||||
observation_count: int,
|
||||
errors: list[str],
|
||||
page_payloads: list[dict[str, Any]],
|
||||
completion_reason: str | None = None,
|
||||
) -> None:
|
||||
job.progress = {
|
||||
"visited_count": len(visited),
|
||||
"queued_count": len(queue),
|
||||
"candidate_count": self._candidate_count(job.domain),
|
||||
"observation_count": observation_count,
|
||||
"max_pages": job.max_pages,
|
||||
"errors": errors[-10:],
|
||||
"pages": page_payloads[-10:],
|
||||
}
|
||||
if completion_reason:
|
||||
job.progress["completion_reason"] = completion_reason
|
||||
self.session.commit()
|
||||
|
||||
def _cancel_requested(self, job: models.DomainDiscoveryJob) -> bool:
|
||||
self.session.expire(job)
|
||||
return job.status == "cancel_requested"
|
||||
|
||||
def _candidate_count(self, domain: str) -> int:
|
||||
return int(
|
||||
self.session.query(models.DomainSchemaCandidate)
|
||||
.filter(models.DomainSchemaCandidate.domain == domain)
|
||||
.count()
|
||||
)
|
||||
|
||||
def _save_observations(self, job: models.DomainDiscoveryJob, observations: list[CandidateObservation]) -> int:
|
||||
if not observations:
|
||||
return 0
|
||||
|
||||
by_key: dict[tuple[str, str], list[CandidateObservation]] = {}
|
||||
for observation in observations:
|
||||
if observation.candidate_type not in SCHEMA_TYPES:
|
||||
continue
|
||||
key = (observation.candidate_type, observation.name)
|
||||
by_key.setdefault(key, []).append(observation)
|
||||
|
||||
saved_count = 0
|
||||
for (candidate_type, name), rows in sorted(by_key.items()):
|
||||
row = self.session.scalar(
|
||||
select(models.DomainSchemaCandidate).where(
|
||||
models.DomainSchemaCandidate.domain == job.domain,
|
||||
models.DomainSchemaCandidate.candidate_type == candidate_type,
|
||||
models.DomainSchemaCandidate.name == name,
|
||||
)
|
||||
)
|
||||
confidence = min(max(max(item.confidence for item in rows) + min(len(rows), 10) * 0.02, 0.0), 0.98)
|
||||
metadata = merge_metadata(item.metadata for item in rows)
|
||||
if row is None:
|
||||
row = models.DomainSchemaCandidate(
|
||||
domain=job.domain,
|
||||
job_id=job.id,
|
||||
candidate_type=candidate_type,
|
||||
name=name,
|
||||
description=rows[0].description,
|
||||
confidence=round(confidence, 4),
|
||||
occurrence_count=len(rows),
|
||||
metadata_json=metadata,
|
||||
)
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
else:
|
||||
row.job_id = job.id
|
||||
row.description = row.description or rows[0].description
|
||||
row.confidence = max(row.confidence, round(confidence, 4))
|
||||
row.occurrence_count += len(rows)
|
||||
row.metadata_json = {**(row.metadata_json or {}), **metadata}
|
||||
row.updated_at = models.utcnow()
|
||||
for evidence in rows[:8]:
|
||||
self.session.add(
|
||||
models.DomainCandidateEvidence(
|
||||
candidate_id=row.id,
|
||||
job_id=job.id,
|
||||
url=evidence.evidence.url,
|
||||
title=evidence.evidence.title,
|
||||
snippet=evidence.evidence.snippet[:1000],
|
||||
metadata_json=evidence.evidence.metadata,
|
||||
)
|
||||
)
|
||||
saved_count += 1
|
||||
return saved_count
|
||||
|
||||
def _upsert_reference_source(self, domain: str, url: str) -> models.DomainReferenceSource:
|
||||
row = self.session.scalar(
|
||||
select(models.DomainReferenceSource).where(
|
||||
models.DomainReferenceSource.domain == domain,
|
||||
models.DomainReferenceSource.url == url,
|
||||
)
|
||||
)
|
||||
if row is None:
|
||||
row = models.DomainReferenceSource(domain=domain, url=url)
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
row.last_crawled_at = models.utcnow()
|
||||
row.updated_at = models.utcnow()
|
||||
return row
|
||||
|
||||
def _enqueue_links(
|
||||
self,
|
||||
*,
|
||||
html: str,
|
||||
base_url: str,
|
||||
root_host: str,
|
||||
depth: int,
|
||||
max_depth: int,
|
||||
same_domain_only: bool,
|
||||
queue: deque[tuple[str, int, str]],
|
||||
queued: set[str],
|
||||
visited: set[str],
|
||||
) -> None:
|
||||
if depth >= max_depth:
|
||||
return
|
||||
for link in discover_links(html, base_url, limit=100):
|
||||
next_url = normalize_url(link.url)
|
||||
if not next_url or next_url in queued or next_url in visited:
|
||||
continue
|
||||
if same_domain_only and seed_host(next_url) != root_host:
|
||||
continue
|
||||
queue.append((next_url, depth + 1, root_host))
|
||||
queued.add(next_url)
|
||||
|
||||
|
||||
def mine_schema_candidates(
|
||||
*,
|
||||
url: str,
|
||||
title: str | None,
|
||||
text: str,
|
||||
html: str,
|
||||
page_type: str,
|
||||
) -> list[CandidateObservation]:
|
||||
observations: list[CandidateObservation] = []
|
||||
evidence_base = {"page_type": page_type}
|
||||
page_type_entity = entity_type_from_page_type(page_type)
|
||||
if page_type_entity:
|
||||
observations.append(
|
||||
observation(
|
||||
"entity_type",
|
||||
page_type_entity,
|
||||
f"Detected from page type {page_type}.",
|
||||
0.72,
|
||||
url,
|
||||
title,
|
||||
title or first_sentence(text),
|
||||
evidence_base,
|
||||
)
|
||||
)
|
||||
|
||||
soup = soup_from_html(html)
|
||||
labels = collect_structural_labels(soup, text)
|
||||
json_ld_types, json_ld_keys = collect_json_ld_schema(soup)
|
||||
|
||||
for schema_type, count in json_ld_types.items():
|
||||
observations.append(
|
||||
observation(
|
||||
"entity_type",
|
||||
pascal_case(schema_type),
|
||||
"Detected from JSON-LD schema type.",
|
||||
min(0.86 + count * 0.02, 0.95),
|
||||
url,
|
||||
title,
|
||||
f"JSON-LD @type: {schema_type}",
|
||||
{**evidence_base, "source": "json_ld"},
|
||||
)
|
||||
)
|
||||
for key, count in json_ld_keys.items():
|
||||
attr = attribute_name(key)
|
||||
if attr:
|
||||
observations.append(
|
||||
observation(
|
||||
"attribute",
|
||||
attr,
|
||||
"Detected from structured data property.",
|
||||
min(0.78 + count * 0.02, 0.92),
|
||||
url,
|
||||
title,
|
||||
f"Structured data property: {key}",
|
||||
{**evidence_base, "source": "json_ld", "raw_label": key},
|
||||
)
|
||||
)
|
||||
|
||||
for label, count in labels.items():
|
||||
attr = attribute_name(label)
|
||||
if not attr:
|
||||
continue
|
||||
observations.append(
|
||||
observation(
|
||||
"attribute",
|
||||
attr,
|
||||
"Detected from page labels, tables, metadata, or key-value text.",
|
||||
min(0.62 + count * 0.04, 0.88),
|
||||
url,
|
||||
title,
|
||||
label,
|
||||
{**evidence_base, "source": "page_label", "raw_label": label},
|
||||
)
|
||||
)
|
||||
hint = relation_hint(label)
|
||||
if hint:
|
||||
entity_type, predicate = hint
|
||||
observations.append(
|
||||
observation(
|
||||
"entity_type",
|
||||
entity_type,
|
||||
f"Detected from repeated label '{label}'.",
|
||||
min(0.68 + count * 0.03, 0.9),
|
||||
url,
|
||||
title,
|
||||
label,
|
||||
{**evidence_base, "source": "relation_hint", "raw_label": label},
|
||||
)
|
||||
)
|
||||
observations.append(
|
||||
observation(
|
||||
"predicate",
|
||||
predicate,
|
||||
f"Suggested relation for label '{label}'.",
|
||||
min(0.7 + count * 0.03, 0.92),
|
||||
url,
|
||||
title,
|
||||
label,
|
||||
{
|
||||
**evidence_base,
|
||||
"source": "relation_hint",
|
||||
"raw_label": label,
|
||||
"object_type": entity_type,
|
||||
},
|
||||
)
|
||||
)
|
||||
observations.append(
|
||||
observation(
|
||||
"alias",
|
||||
attr,
|
||||
f"Map source field '{attr}' to relation '{predicate}'.",
|
||||
0.66,
|
||||
url,
|
||||
title,
|
||||
label,
|
||||
{**evidence_base, "source": "relation_hint", "target": predicate},
|
||||
)
|
||||
)
|
||||
|
||||
return observations
|
||||
|
||||
|
||||
def observation(
|
||||
candidate_type: str,
|
||||
name: str,
|
||||
description: str,
|
||||
confidence: float,
|
||||
url: str,
|
||||
title: str | None,
|
||||
snippet: str,
|
||||
metadata: dict[str, Any],
|
||||
) -> CandidateObservation:
|
||||
return CandidateObservation(
|
||||
candidate_type=candidate_type,
|
||||
name=name[:160],
|
||||
description=description,
|
||||
confidence=confidence,
|
||||
evidence=CandidateEvidence(url=url, title=title, snippet=clean_snippet(snippet), metadata=metadata),
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
def soup_from_html(html: str):
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
return BeautifulSoup(html or "", "html.parser")
|
||||
|
||||
|
||||
def collect_json_ld_schema(soup) -> tuple[Counter[str], Counter[str]]:
|
||||
types: Counter[str] = Counter()
|
||||
keys: Counter[str] = Counter()
|
||||
for tag in soup.find_all("script"):
|
||||
script_type = " ".join(str(item) for item in tag.get("type", "").split()).lower()
|
||||
if "ld+json" not in script_type:
|
||||
continue
|
||||
raw = tag.string or tag.get_text(" ", strip=True)
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except Exception:
|
||||
continue
|
||||
visit_json_ld(payload, types, keys)
|
||||
return types, keys
|
||||
|
||||
|
||||
def visit_json_ld(value: Any, types: Counter[str], keys: Counter[str]) -> None:
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
visit_json_ld(item, types, keys)
|
||||
return
|
||||
if not isinstance(value, dict):
|
||||
return
|
||||
for key, nested in value.items():
|
||||
if key == "@type":
|
||||
for item in ensure_list(nested):
|
||||
if isinstance(item, str) and is_schema_name(item):
|
||||
types[item] += 1
|
||||
elif not key.startswith("@"):
|
||||
keys[key] += 1
|
||||
visit_json_ld(nested, types, keys)
|
||||
|
||||
|
||||
def collect_structural_labels(soup, text: str) -> Counter[str]:
|
||||
labels: Counter[str] = Counter()
|
||||
for selector in ["th", "dt", "label", "[itemprop]", "[property]", "[name]"]:
|
||||
for node in soup.select(selector):
|
||||
label = ""
|
||||
if selector == "[itemprop]":
|
||||
label = node.get("itemprop", "")
|
||||
elif selector == "[property]":
|
||||
label = str(node.get("property", "")).split(":")[-1]
|
||||
elif selector == "[name]":
|
||||
label = str(node.get("name", "")).split(":")[-1]
|
||||
else:
|
||||
label = node.get_text(" ", strip=True)
|
||||
add_label(labels, label)
|
||||
|
||||
for line in text.splitlines()[:500]:
|
||||
match = re.match(r"^\s*([^:：|]{2,40})\s*[:：|]\s*(.{1,200})$", line)
|
||||
if match:
|
||||
add_label(labels, match.group(1))
|
||||
return labels
|
||||
|
||||
|
||||
def add_label(labels: Counter[str], label: str) -> None:
|
||||
cleaned = clean_label(label)
|
||||
if cleaned and not is_noisy_label(cleaned):
|
||||
labels[cleaned] += 1
|
||||
|
||||
|
||||
def relation_hint(label: str) -> tuple[str, str] | None:
|
||||
lowered = label.lower()
|
||||
for token, hint in RELATION_HINTS.items():
|
||||
if token in lowered:
|
||||
return hint
|
||||
return None
|
||||
|
||||
|
||||
def entity_type_from_page_type(page_type: str) -> str | None:
|
||||
mapping = {
|
||||
"ProductPage": "Product",
|
||||
"CategoryPage": "Category",
|
||||
"BrandStoryPage": "Brand",
|
||||
"NoticePage": "Notice",
|
||||
"PromotionPage": "Promotion",
|
||||
"ReviewPage": "Review",
|
||||
"BoardPage": "Article",
|
||||
}
|
||||
return mapping.get(page_type)
|
||||
|
||||
|
||||
def attribute_name(label: str) -> str:
|
||||
cleaned = clean_label(label)
|
||||
if not cleaned or is_noisy_label(cleaned):
|
||||
return ""
|
||||
if re.search(r"[가-힣]", cleaned):
|
||||
return re.sub(r"[\s/()-]+", "_", cleaned).strip("_")[:80]
|
||||
words = re.findall(r"[A-Za-z0-9]+", cleaned)
|
||||
if not words:
|
||||
return ""
|
||||
return "_".join(word.lower() for word in words)[:80]
|
||||
|
||||
|
||||
def pascal_case(value: str) -> str:
|
||||
cleaned = clean_label(value)
|
||||
parts = re.findall(r"[A-Za-z0-9가-힣]+", cleaned)
|
||||
if not parts:
|
||||
return ""
|
||||
return "".join(part[:1].upper() + part[1:] for part in parts)[:80]
|
||||
|
||||
|
||||
def clean_label(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", str(value or "").replace("\xa0", " ")).strip(" -_*:/|")
|
||||
|
||||
|
||||
def is_schema_name(value: str) -> bool:
|
||||
return 1 <= len(value) <= 80 and not value.startswith("http")
|
||||
|
||||
|
||||
def is_noisy_label(value: str) -> bool:
|
||||
lowered = value.lower().strip()
|
||||
if len(lowered) < 2 or len(lowered) > 80:
|
||||
return True
|
||||
noisy = {
|
||||
"class",
|
||||
"style",
|
||||
"viewport",
|
||||
"charset",
|
||||
"csrf-token",
|
||||
"description",
|
||||
"keywords",
|
||||
"robots",
|
||||
"generator",
|
||||
"theme-color",
|
||||
}
|
||||
return lowered in noisy or lowered.startswith("twitter:")
|
||||
|
||||
|
||||
def clean_snippet(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", str(value or "")).strip()[:1000]
|
||||
|
||||
|
||||
def first_sentence(text: str) -> str:
|
||||
cleaned = clean_snippet(text)
|
||||
for delimiter in [". ", "\n", "。", "다."]:
|
||||
if delimiter in cleaned:
|
||||
return cleaned.split(delimiter, 1)[0][:240]
|
||||
return cleaned[:240]
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str:
|
||||
return str(url or "").strip().rstrip("/")
|
||||
|
||||
|
||||
def seed_host(url: str) -> str:
|
||||
return urlparse(url).netloc.lower()
|
||||
|
||||
|
||||
def ensure_list(value: Any) -> list[Any]:
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
return [value]
|
||||
|
||||
|
||||
def merge_metadata(items) -> dict[str, Any]:
|
||||
merged: dict[str, Any] = {}
|
||||
sources: set[str] = set()
|
||||
targets: set[str] = set()
|
||||
object_types: set[str] = set()
|
||||
for item in items:
|
||||
merged.update(item or {})
|
||||
if item.get("source"):
|
||||
sources.add(str(item["source"]))
|
||||
if item.get("target"):
|
||||
targets.add(str(item["target"]))
|
||||
if item.get("object_type"):
|
||||
object_types.add(str(item["object_type"]))
|
||||
if sources:
|
||||
merged["sources"] = sorted(sources)
|
||||
if targets:
|
||||
merged["targets"] = sorted(targets)
|
||||
if object_types:
|
||||
merged["object_types"] = sorted(object_types)
|
||||
return merged
|
||||
Reference in New Issue
Block a user