diff --git a/.gitignore b/.gitignore index 071b10b..ea31646 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ __pycache__/ *.sqlite3-* uvicorn.*.log .server-logs/ +.env +ontology_platform/.env /ontology_platform/data/ui_projects.json diff --git a/docs/PHASE_PLANNING.md b/docs/PHASE_PLANNING.md index bc2e2d4..d9e646f 100644 --- a/docs/PHASE_PLANNING.md +++ b/docs/PHASE_PLANNING.md @@ -2,12 +2,13 @@ PHASE_PLANNING.md 요청된 작업을 다음 순서로 진행한다. -0. 이전 PHASE_INDEX.md 다른이름 으로 변경. (예 : 26_05_19_PHASE_INDEX.md) -1. 요청된 작업 폴더 생성 (예 : 26_05_19_기능개선 ) -2. 해당 작업을 분석후 Phase 계획 -3. Phase별로 상세 작업 계획을 세운후 PHASE_INDEX.md 새로 생성후 작성. 작성 방법은 아래 # PHASE INDEX EXAMPLE 참조 -4. Phase별 상세작업 계획은 1에 의해 생성된 폴더아래 파일을 만들어 저장( 예 : phase_01_001_pipeline.md) -5. 생성된 파일은 PHASE_INDEX.md의 해당 PHASE의 FILE: 에 기록 +0. 이전 PHASE_INDEX.md 파일은 백업을 목적으로 다른이름 으로 변경. (예 : 26_05_19_PHASE_INDEX.md) +1. 새로운 PHASE_INDEX.md 파일 생성. 이전 내용은 완전히 삭제된 상태로 시작. +2. 새로운 작업은 반드시 새로운 폴더를 생성해서 요청된 작업 폴더 생성 (예 : 26_05_19_기능개선 ) +3. 해당 작업을 분석후 Phase 계획 +4. Phase별로 상세 작업 계획을 세운후 PHASE_INDEX.md 파일에 작성. 작성 방법은 아래 # PHASE INDEX EXAMPLE 참조. 세부 항목의 끝에는 반드시 [TODO] 표시를 해서 작업이 완료되지 않았음을 명확히 한다. (예 : 1) Source Crawl, Page Clean [TODO]) +5. Phase별 상세작업 계획은 2에 의해 생성된 폴더아래 파일을 만들어 저장( 예 : phase_01_001_pipeline.md) +6. 5번 항목에 의해 생성된 파일은 PHASE_INDEX.md의 해당 PHASE의 FILE: 에 기록해 참조 하도록 한다. (예 : FILE: ./26_05_19_기능개선/phase_01_001_pipeline.md) # PHASE INDEX EXAMPLE diff --git a/docs/PHASE_WORKFLOW.md b/docs/PHASE_WORKFLOW.md index 793e74b..154f03e 100644 --- a/docs/PHASE_WORKFLOW.md +++ b/docs/PHASE_WORKFLOW.md @@ -64,7 +64,7 @@ phase 파일은 상태 기록 용도로 사용하지 않는다. 를 의미한다. -이 경우 반드시 작업 전,후 PHASE_INDEX.md에 아래 예처럼 기록한다. +이 경우 반드시 작업 전,후 PHASE_INDEX.md파일의 해당 phase의 세부항목인 [진행중] 항목 아래에 다음 예 처럼 기록한다. PHASE 3. Claim Review 개선 FILE: ./26_05_19_기능개선/phase_03_002_metadata_display.md diff --git a/ontology_platform/crawler_platform/app/api/routes.py b/ontology_platform/crawler_platform/app/api/routes.py index 56e9e64..bfb20d3 100644 --- a/ontology_platform/crawler_platform/app/api/routes.py +++ b/ontology_platform/crawler_platform/app/api/routes.py @@ -4,12 +4,14 @@ import csv import io import json import re +import time from dataclasses import asdict from typing import Any from fastapi import BackgroundTasks, HTTPException, Response from pydantic import BaseModel, Field from sqlalchemy import select +from sqlalchemy.exc import OperationalError from crawler_platform.app.config.loader import ( ProjectConfig, @@ -41,15 +43,18 @@ from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, DOMAIN_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{1,79}$") +SITE_CRAWL_CANCEL_REQUESTS: set[int] = set() class CrawlRequest(BaseModel): config_path: str source_name: str url: str + extraction_mode: str = "hybrid" extractor_provider: str = "lm_studio" extractor_model: str | None = None extractor_base_url: str | None = "http://localhost:1234/v1" + fallback_to_rules: bool = True check_robots_txt: bool = False respect_robots_txt: bool | None = None @@ -67,9 +72,11 @@ class SiteCrawlByProjectRequest(BaseModel): project_name: str source_name: str url: str + extraction_mode: str = "hybrid" extractor_provider: str = "lm_studio" extractor_model: str | None = None extractor_base_url: str | None = "http://localhost:1234/v1" + fallback_to_rules: bool = True check_robots_txt: bool = False respect_robots_txt: bool | None = None max_depth: int = 2 @@ -85,9 +92,11 @@ class SiteCrawlByProjectRequest(BaseModel): config_path=config_path_placeholder, source_name=self.source_name, url=self.url, + extraction_mode=self.extraction_mode, extractor_provider=self.extractor_provider, extractor_model=self.extractor_model, extractor_base_url=self.extractor_base_url, + fallback_to_rules=self.fallback_to_rules, check_robots_txt=self.check_robots_txt, respect_robots_txt=self.respect_robots_txt, max_depth=self.max_depth, @@ -535,9 +544,11 @@ class ResearchRunByProjectRequest(BaseModel): url: str | None = None seed_entity_id: int | None = None goal: str = "Semantic ontology exploration" + extraction_mode: str = "hybrid" extractor_provider: str = "lm_studio" extractor_model: str | None = None extractor_base_url: str | None = "http://localhost:1234/v1" + fallback_to_rules: bool = True check_robots_txt: bool = False respect_robots_txt: bool | None = None max_depth: int = 2 @@ -622,6 +633,20 @@ def site_crawl_progress_payload(result, latest_page=None) -> dict[str, Any]: payload = asdict(result) if latest_page is not None: payload["latest_page"] = asdict(latest_page) + pages = payload.get("pages") or [] + payload["extraction_summary"] = { + "llm_skipped_count": sum(1 for page in pages if page.get("llm_skipped")), + "fallback_count": sum(1 for page in pages if page.get("fallback_used")), + "conflict_claim_count": sum(int(page.get("conflict_claim_count") or 0) for page in pages), + "agreement_claim_count": sum(int(page.get("agreement_claim_count") or 0) for page in pages), + "llm_call_count": sum( + 1 + for page in pages + if page.get("extraction_mode") in {"hybrid", "llm_only", "compare"} + and not page.get("llm_skipped") + and not page.get("fallback_used") + ), + } return payload @@ -696,7 +721,58 @@ def update_site_crawl_job_metadata(job: models.CrawlJob, **updates: Any) -> None job.metadata_json = metadata +def mark_orphan_site_crawl_jobs_canceled(database_url: str) -> None: + with session_scope(database_url) as session: + rows = session.scalars( + select(models.CrawlJob).where( + models.CrawlJob.status.in_(["running", "cancel_requested"]), + ) + ).all() + for job in rows: + if (job.metadata_json or {}).get("kind") != "site_crawl": + continue + job.status = "canceled" + job.error = job.error or "canceled after server restart/interrupted crawl" + job.finished_at = models.utcnow() + + +def request_site_crawl_cancel(database_url: str, job_id: int) -> dict[str, Any]: + SITE_CRAWL_CANCEL_REQUESTS.add(job_id) + last_error: Exception | None = None + for attempt in range(3): + try: + with session_scope(database_url) as session: + job = session.get(models.CrawlJob, job_id) + if job is None or (job.metadata_json or {}).get("kind") != "site_crawl": + raise HTTPException(status_code=404, detail="site crawl job not found") + if job.status in {"completed", "failed", "canceled"}: + SITE_CRAWL_CANCEL_REQUESTS.discard(job_id) + return crawl_job_response(job) + job.status = "cancel_requested" + job.error = "cancel requested by user" + payload = crawl_job_response(job) + return payload + except OperationalError as exc: + last_error = exc + if "database is locked" not in str(exc).lower(): + raise + time.sleep(0.25 * (attempt + 1)) + return { + "job_id": job_id, + "status": "cancel_requested", + "url": None, + "error": f"cancel requested in memory; database was locked: {last_error}", + "scheduled_at": None, + "started_at": None, + "finished_at": None, + "progress": {}, + "request": {}, + } + + def is_site_crawl_cancel_requested(session, job_id: int) -> bool: + if job_id in SITE_CRAWL_CANCEL_REQUESTS: + return True session.expire_all() job = session.get(models.CrawlJob, job_id) return job is None or job.status == "cancel_requested" @@ -744,6 +820,8 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A provider=request.extractor_provider, model=request.extractor_model, base_url=request.extractor_base_url, + extraction_mode=request.extraction_mode, + fallback_to_rules=request.fallback_to_rules, ), ) @@ -775,12 +853,17 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A finished_job = session.get(models.CrawlJob, job_id) if finished_job is not None: - finished_job.status = "canceled" if finished_job.status == "cancel_requested" else "completed" + finished_job.status = ( + "canceled" + if finished_job.status == "cancel_requested" or job_id in SITE_CRAWL_CANCEL_REQUESTS + else "completed" + ) finished_job.finished_at = models.utcnow() update_site_crawl_job_metadata( finished_job, progress=site_crawl_progress_payload(result), ) + SITE_CRAWL_CANCEL_REQUESTS.discard(job_id) except Exception as exc: with session_scope(database_url) as session: job = session.get(models.CrawlJob, job_id) @@ -792,10 +875,12 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A progress = dict(metadata.get("progress") or {}) progress["errors"] = [*progress.get("errors", []), str(exc)] update_site_crawl_job_metadata(job, progress=progress) + SITE_CRAWL_CANCEL_REQUESTS.discard(job_id) def register_routes(app, database_url: str) -> None: recover_interrupted_domain_discovery_jobs(database_url) + mark_orphan_site_crawl_jobs_canceled(database_url) @app.get("/health") def health(): @@ -1536,6 +1621,8 @@ def register_routes(app, database_url: str) -> None: provider=request.extractor_provider, model=request.extractor_model, base_url=request.extractor_base_url, + extraction_mode=request.extraction_mode, + fallback_to_rules=request.fallback_to_rules, ), ) try: @@ -1546,6 +1633,13 @@ def register_routes(app, database_url: str) -> None: "page_id": result.page_id, "claim_count": result.claim_count, "entity_count": result.entity_count, + "extraction_mode": result.extraction_mode, + "effective_extraction_mode": result.effective_extraction_mode, + "llm_skipped": result.llm_skipped, + "llm_skip_reason": result.llm_skip_reason, + "fallback_used": result.fallback_used, + "agreement_claim_count": result.agreement_claim_count, + "conflict_claim_count": result.conflict_claim_count, "crawl_status": result.crawl_status, "extraction_status": result.extraction_status, "page_type": result.page_type, @@ -1648,15 +1742,7 @@ def register_routes(app, database_url: str) -> None: @app.post("/crawl-site/jobs/{job_id}/cancel") def cancel_crawl_site_job(job_id: int): - with session_scope(database_url) as session: - job = session.get(models.CrawlJob, job_id) - if job is None or (job.metadata_json or {}).get("kind") != "site_crawl": - raise HTTPException(status_code=404, detail="site crawl job not found") - if job.status in {"completed", "failed", "canceled"}: - return crawl_job_response(job) - job.status = "cancel_requested" - job.error = "cancel requested by user" - return crawl_job_response(job) + return request_site_crawl_cancel(database_url, job_id) @app.post("/discover") def discover(request: DiscoverRequest): @@ -1705,6 +1791,8 @@ def register_routes(app, database_url: str) -> None: provider=request.extractor_provider, model=request.extractor_model, base_url=request.extractor_base_url, + extraction_mode=request.extraction_mode, + fallback_to_rules=request.fallback_to_rules, ), ) try: @@ -1749,6 +1837,8 @@ def register_routes(app, database_url: str) -> None: provider=request.extractor_provider, model=request.extractor_model, base_url=request.extractor_base_url, + extraction_mode=request.extraction_mode, + fallback_to_rules=request.fallback_to_rules, ), ) try: @@ -2080,6 +2170,11 @@ def register_routes(app, database_url: str) -> None: "graph_merge_status": (claim.metadata_json or {}).get("graph_merge_status"), "graph_merge_reason": (claim.metadata_json or {}).get("graph_merge_reason"), "confidence_breakdown": (claim.metadata_json or {}).get("confidence_breakdown"), + "agreement": (claim.metadata_json or {}).get("agreement"), + "extraction_source": (claim.metadata_json or {}).get("extraction_source"), + "claim_kind": (claim.metadata_json or {}).get("claim_kind"), + "rule_confidence": (claim.metadata_json or {}).get("rule_confidence"), + "llm_confidence": (claim.metadata_json or {}).get("llm_confidence"), "review_required": (claim.metadata_json or {}).get("review_required"), "review_reason": (claim.metadata_json or {}).get("review_reason"), "conflict_status": (claim.metadata_json or {}).get("conflict_status"), @@ -2243,9 +2338,11 @@ def register_routes(app, database_url: str) -> None: config_path="", source_name=source.name, url=url, + extraction_mode=str(latest_request.get("extraction_mode") or "hybrid"), extractor_provider=str(latest_request.get("extractor_provider") or "lm_studio"), extractor_model=latest_request.get("extractor_model"), extractor_base_url=latest_request.get("extractor_base_url") or "http://localhost:1234/v1", + fallback_to_rules=bool(latest_request.get("fallback_to_rules", True)), check_robots_txt=bool(latest_request.get("check_robots_txt") or False), respect_robots_txt=latest_request.get("respect_robots_txt"), max_depth=int(latest_request.get("max_depth") or 2), @@ -2404,6 +2501,20 @@ def register_routes(app, database_url: str) -> None: "validation": (log.raw_output or {}).get("validation"), "page_context": (log.raw_output or {}).get("page_context"), "candidate_count": len((log.raw_output or {}).get("candidate_claims") or []), + "extraction_mode": (log.raw_output or {}).get("extraction_mode"), + "effective_extraction_mode": (log.raw_output or {}).get("effective_extraction_mode"), + "comparison": (log.raw_output or {}).get("comparison"), + "rule_entity_count": (log.raw_output or {}).get("rule_entity_count"), + "rule_claim_count": (log.raw_output or {}).get("rule_claim_count"), + "llm_entity_count": (log.raw_output or {}).get("llm_entity_count"), + "llm_claim_count": (log.raw_output or {}).get("llm_claim_count"), + "agreement_claim_count": (log.raw_output or {}).get("agreement_claim_count"), + "rule_only_claim_count": (log.raw_output or {}).get("rule_only_claim_count"), + "llm_only_claim_count": (log.raw_output or {}).get("llm_only_claim_count"), + "conflict_claim_count": (log.raw_output or {}).get("conflict_claim_count"), + "llm_skipped": (log.raw_output or {}).get("llm_skipped"), + "llm_skip_reason": (log.raw_output or {}).get("llm_skip_reason"), + "fallback": (log.raw_output or {}).get("fallback"), "raw_output": log.raw_output, } for log, page in rows diff --git a/ontology_platform/crawler_platform/app/cli/main.py b/ontology_platform/crawler_platform/app/cli/main.py index 2d00ba0..d71b62b 100644 --- a/ontology_platform/crawler_platform/app/cli/main.py +++ b/ontology_platform/crawler_platform/app/cli/main.py @@ -37,6 +37,8 @@ def build_parser() -> argparse.ArgumentParser: crawl.add_argument("--extractor-provider", default="rule_based", choices=["rule_based", "openai", "ollama", "lm_studio"]) crawl.add_argument("--extractor-model") crawl.add_argument("--extractor-base-url") + crawl.add_argument("--extraction-mode", default=None, choices=["rule_only", "llm_only", "hybrid", "compare"]) + crawl.add_argument("--no-rule-fallback", action="store_true") claims = sub.add_parser("claims") claims.add_argument("--project", required=True) @@ -81,6 +83,8 @@ def main() -> None: provider=args.extractor_provider, model=args.extractor_model, base_url=args.extractor_base_url, + extraction_mode=args.extraction_mode, + fallback_to_rules=not args.no_rule_fallback, ), ) result = pipeline.crawl_url(config, args.source, args.url) diff --git a/ontology_platform/crawler_platform/app/core/crawler/pipeline.py b/ontology_platform/crawler_platform/app/core/crawler/pipeline.py index 0c98373..596d2b3 100644 --- a/ontology_platform/crawler_platform/app/core/crawler/pipeline.py +++ b/ontology_platform/crawler_platform/app/core/crawler/pipeline.py @@ -16,6 +16,13 @@ class CrawlResult: page_id: int claim_count: int entity_count: int + extraction_mode: str | None = None + effective_extraction_mode: str | None = None + llm_skipped: bool = False + llm_skip_reason: str | None = None + fallback_used: bool = False + agreement_claim_count: int = 0 + conflict_claim_count: int = 0 crawl_status: str = "success" extraction_status: str = "success" page_type: str = "UnknownPage" @@ -112,10 +119,12 @@ class CrawlPipeline: bundle = self.extractor.extract_from_context(context, project_config) bundle = attach_page_context(bundle, context) claims = self.repository.save_extraction_bundle(project.id, source, page, bundle, project_config) + extraction_summary = extraction_summary_from_raw(bundle.raw_output) return CrawlResult( page_id=page.id, claim_count=len(claims), entity_count=len(bundle.entities), + **extraction_summary, crawl_status=fetch_result.crawl_status, extraction_status=parsed.extraction_status, page_type=page_type, @@ -125,3 +134,15 @@ class CrawlPipeline: robots_status=robots_decision.status, robots_reason=robots_decision.reason, ) + + +def extraction_summary_from_raw(raw_output: dict[str, object]) -> dict[str, object]: + return { + "extraction_mode": raw_output.get("extraction_mode"), + "effective_extraction_mode": raw_output.get("effective_extraction_mode"), + "llm_skipped": bool(raw_output.get("llm_skipped")), + "llm_skip_reason": raw_output.get("llm_skip_reason"), + "fallback_used": bool(raw_output.get("fallback")), + "agreement_claim_count": int(raw_output.get("agreement_claim_count") or 0), + "conflict_claim_count": int(raw_output.get("conflict_claim_count") or 0), + } diff --git a/ontology_platform/crawler_platform/app/core/crawler/site_crawler.py b/ontology_platform/crawler_platform/app/core/crawler/site_crawler.py index f6016a9..281e2e6 100644 --- a/ontology_platform/crawler_platform/app/core/crawler/site_crawler.py +++ b/ontology_platform/crawler_platform/app/core/crawler/site_crawler.py @@ -28,6 +28,13 @@ class SiteCrawlPageResult: page_id: int | None = None claim_count: int = 0 entity_count: int = 0 + extraction_mode: str | None = None + effective_extraction_mode: str | None = None + llm_skipped: bool = False + llm_skip_reason: str | None = None + fallback_used: bool = False + agreement_claim_count: int = 0 + conflict_claim_count: int = 0 discovered_count: int = 0 crawl_status: str = "success" extraction_status: str = "unknown" @@ -331,6 +338,7 @@ class SiteCrawler: bundle = self.extractor.extract_from_context(context, project_config) bundle = attach_page_context(bundle, context) claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config) + extraction_summary = extraction_summary_from_raw(bundle.raw_output) self._finish_job(job, "completed") result.analyzed_count += 1 page_result = SiteCrawlPageResult( @@ -338,6 +346,7 @@ class SiteCrawler: status="completed", claim_count=len(claims), entity_count=len(bundle.entities), + **extraction_summary, ) else: self._finish_job(job, "discovered") @@ -441,3 +450,15 @@ def classify_page( source_zones: list[dict[str, object]] | None = None, ) -> str: return classify_page_type(url, title, text, html=html, source_zones=source_zones) + + +def extraction_summary_from_raw(raw_output: dict[str, object]) -> dict[str, object]: + return { + "extraction_mode": raw_output.get("extraction_mode"), + "effective_extraction_mode": raw_output.get("effective_extraction_mode"), + "llm_skipped": bool(raw_output.get("llm_skipped")), + "llm_skip_reason": raw_output.get("llm_skip_reason"), + "fallback_used": bool(raw_output.get("fallback")), + "agreement_claim_count": int(raw_output.get("agreement_claim_count") or 0), + "conflict_claim_count": int(raw_output.get("conflict_claim_count") or 0), + } diff --git a/ontology_platform/crawler_platform/app/core/database/repository.py b/ontology_platform/crawler_platform/app/core/database/repository.py index 00906b2..411878b 100644 --- a/ontology_platform/crawler_platform/app/core/database/repository.py +++ b/ontology_platform/crawler_platform/app/core/database/repository.py @@ -197,6 +197,7 @@ class KnowledgeRepository: claims: list[models.Claim] = [] for extracted_claim in bundle.claims: + claim_status_for_row = claim_status subject = self._entity_for_claim(project_id, extracted_claim.subject_type, extracted_claim.subject_name, entity_index) object_entity = None if extracted_claim.object_name and extracted_claim.object_type: @@ -211,6 +212,9 @@ class KnowledgeRepository: **extracted_claim.metadata, "source_trust": source.trust_level, } + metadata_status = str(claim_metadata.get("validation_status") or "") + if metadata_status in {"active", "validated_claim", "candidate_claim", "rule_candidate"}: + claim_status_for_row = metadata_status claim_metadata["source_history"] = [ { "source_id": source.id, @@ -272,7 +276,7 @@ class KnowledgeRepository: confidence=confidence, confidence_reason=extracted_claim.confidence_reason, extraction_method=bundle.extractor_name, - status=claim_status, + status=claim_status_for_row, metadata_json=claim_metadata, ) self.session.add(claim) @@ -284,8 +288,8 @@ class KnowledgeRepository: claim.last_seen_at = models.utcnow() claim.confidence = max(claim.confidence, confidence) claim.confidence_reason = extracted_claim.confidence_reason or claim.confidence_reason - if claim_status == "active" or claim.status != "active": - claim.status = claim_status + if claim_status_for_row == "active" or claim.status != "active": + claim.status = claim_status_for_row claim.metadata_json = {**existing_metadata, **claim_metadata} claim.metadata_json["source_history"] = merge_source_history( existing_history, diff --git a/ontology_platform/crawler_platform/app/core/extractor/ai_provider.py b/ontology_platform/crawler_platform/app/core/extractor/ai_provider.py index ce38d2a..bf71fe3 100644 --- a/ontology_platform/crawler_platform/app/core/extractor/ai_provider.py +++ b/ontology_platform/crawler_platform/app/core/extractor/ai_provider.py @@ -31,6 +31,8 @@ class LLMJsonExtractor(AIExtractor): model: str | None = None, base_url: str | None = None, timeout_seconds: int = 300, + fallback_to_rules: bool = True, + merge_rule_claims: bool = True, ): # Local LM Studio runs on consumer hardware; keep a shorter timeout so # we can quickly fallback instead of stalling a crawl worker for 5+ min. @@ -41,6 +43,8 @@ class LLMJsonExtractor(AIExtractor): self.model = model self.base_url = base_url self.timeout_seconds = timeout_seconds + self.fallback_to_rules = fallback_to_rules + self.merge_rule_claims = merge_rule_claims def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle: return self._extract_text(page_text, project_config, context=None) @@ -64,7 +68,11 @@ class LLMJsonExtractor(AIExtractor): try: raw = self.complete_json(page_text, project_config, compact=compact_mode, context=context) bundle = self._bundle_from_raw(raw, mode_name) - enriched = self._merge_rule_fallback_claims(bundle, page_text, project_config, mode_name) + enriched = ( + self._merge_rule_fallback_claims(bundle, page_text, project_config, mode_name) + if self.merge_rule_claims + else bundle + ) if enriched.entities and enriched.claims: return self.normalize_to_ontology(enriched, project_config.ontology) if bundle.entities and bundle.claims: @@ -72,6 +80,8 @@ class LLMJsonExtractor(AIExtractor): errors.append(f"{mode_name}: AI returned no usable entities or claims") except Exception as exc: errors.append(f"{mode_name}: {exc}") + if not self.fallback_to_rules: + raise RuntimeError(" | ".join(errors)) return self._fallback_bundle(page_text, project_config, " | ".join(errors)) def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]: diff --git a/ontology_platform/crawler_platform/app/core/extractor/factory.py b/ontology_platform/crawler_platform/app/core/extractor/factory.py index e50b62d..96525bd 100644 --- a/ontology_platform/crawler_platform/app/core/extractor/factory.py +++ b/ontology_platform/crawler_platform/app/core/extractor/factory.py @@ -2,18 +2,71 @@ from __future__ import annotations from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor from crawler_platform.app.core.extractor.base import Extractor +from crawler_platform.app.core.extractor.hybrid import HybridExtractor from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor +LLM_PROVIDERS = {"openai", "ollama", "lm_studio"} + + def extractor_for_domain( domain: str, provider: str = "rule_based", model: str | None = None, base_url: str | None = None, + extraction_mode: str | None = None, + fallback_to_rules: bool = True, ) -> Extractor: - if provider in {"openai", "ollama", "lm_studio"}: - return LLMJsonExtractor(domain=domain, provider=provider, model=model, base_url=base_url) + mode = normalize_extraction_mode(provider, extraction_mode) + llm_provider = normalize_llm_provider(provider) + if mode in {"hybrid", "compare"}: + return HybridExtractor( + domain=domain, + llm_provider=llm_provider, + model=model, + base_url=base_url, + mode=mode, + fallback_to_rules=fallback_to_rules, + ) + if mode == "llm_only": + return LLMJsonExtractor( + domain=domain, + provider=llm_provider, + model=model, + base_url=base_url, + fallback_to_rules=fallback_to_rules, + merge_rule_claims=False, + ) + return rule_extractor_for_domain(domain) + + +def normalize_extraction_mode(provider: str, extraction_mode: str | None = None) -> str: + if extraction_mode: + value = extraction_mode.strip().lower().replace("-", "_") + else: + value = provider.strip().lower().replace("-", "_") + aliases = { + "rule": "rule_only", + "rules": "rule_only", + "rule_based": "rule_only", + "llm": "llm_only", + "ai": "llm_only", + } + value = aliases.get(value, value) + if value in {"rule_only", "llm_only", "hybrid", "compare"}: + return value + if value in LLM_PROVIDERS: + return "hybrid" + return "rule_only" + + +def normalize_llm_provider(provider: str) -> str: + value = provider.strip().lower() + return value if value in LLM_PROVIDERS else "lm_studio" + + +def rule_extractor_for_domain(domain: str) -> Extractor: if domain == "perfume": return PerfumeRuleBasedExtractor() return GenericRuleBasedExtractor() diff --git a/ontology_platform/crawler_platform/app/core/extractor/hybrid.py b/ontology_platform/crawler_platform/app/core/extractor/hybrid.py new file mode 100644 index 0000000..f5cde9a --- /dev/null +++ b/ontology_platform/crawler_platform/app/core/extractor/hybrid.py @@ -0,0 +1,438 @@ +from __future__ import annotations + +import json +from dataclasses import replace +from typing import Any + +from crawler_platform.app.config.loader import ProjectConfig +from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor, dedupe_entities +from crawler_platform.app.core.extractor.base import ( + ExtractedClaim, + ExtractedEntity, + ExtractionBundle, + ExtractionPageContext, + Extractor, +) +from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor +from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor + + +LLM_PAGE_TYPES = {"ProductPage", "BrandStoryPage", "ReviewPage"} +SKIP_LLM_PAGE_TYPES = {"CategoryPage", "SearchPage", "ListingPage"} +RULE_ONLY_PAGE_TYPES = {"BoardPage", "CommunityPage", "UnknownPage"} +MIN_CLEAN_TEXT_CHARS_FOR_LLM = 300 +MAX_CLEAN_TEXT_CHARS_FOR_LLM = 60000 + + +class HybridExtractor(Extractor): + """Run rule extraction first, then LLM extraction, then merge with agreement metadata.""" + + name = "hybrid_rule_llm_extractor" + provider = "hybrid" + + def __init__( + self, + domain: str, + llm_provider: str = "lm_studio", + model: str | None = None, + base_url: str | None = None, + mode: str = "hybrid", + fallback_to_rules: bool = True, + ): + self.domain = domain + self.llm_provider = llm_provider + self.model = model + self.base_url = base_url + self.mode = normalize_mode(mode) + self.fallback_to_rules = fallback_to_rules + self.rule_extractor = rule_extractor_for_domain(domain) + self.llm_extractor = LLMJsonExtractor( + domain=domain, + provider=llm_provider, + model=model, + base_url=base_url, + fallback_to_rules=False, + merge_rule_claims=False, + ) + + def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle: + return self._extract_text(page_text, project_config, context=None) + + def extract_from_context( + self, + context: ExtractionPageContext, + project_config: ProjectConfig, + ) -> ExtractionBundle: + return self._extract_text(context.clean_text, project_config, context=context) + + def _extract_text( + self, + page_text: str, + project_config: ProjectConfig, + context: ExtractionPageContext | None, + ) -> ExtractionBundle: + rule_bundle = self._rule_bundle(page_text, project_config, context) + mark_bundle(rule_bundle, source="rule", mode=self.mode) + + if self.mode == "rule_only": + rule_bundle.extractor_name = "rule_only_extractor" + rule_bundle.raw_output = { + **rule_bundle.raw_output, + "extraction_mode": self.mode, + "effective_extraction_mode": "rule_only", + "llm_skipped": True, + "llm_skip_reason": "rule_only mode", + } + return rule_bundle + + skip_reason = llm_skip_reason(context, page_text, self.mode) + if skip_reason: + return llm_skipped_bundle(rule_bundle, self, skip_reason, context) + + try: + llm_bundle = self._llm_bundle(page_text, project_config, context) + mark_bundle(llm_bundle, source="llm", mode=self.mode) + except Exception as exc: + if not self.fallback_to_rules: + raise + return fallback_bundle(rule_bundle, self, exc) + + if self.mode == "llm_only": + llm_bundle.extractor_name = "llm_only_extractor" + llm_bundle.raw_output = { + **llm_bundle.raw_output, + "extraction_mode": self.mode, + "effective_extraction_mode": "llm_only", + "rule_entity_count": len(rule_bundle.entities), + "rule_claim_count": len(rule_bundle.claims), + } + return llm_bundle + + return merge_bundles(rule_bundle, llm_bundle, self) + + def _rule_bundle( + self, + page_text: str, + project_config: ProjectConfig, + context: ExtractionPageContext | None, + ) -> ExtractionBundle: + if context is not None: + return self.rule_extractor.extract_from_context(context, project_config) + return self.rule_extractor.extract(page_text, project_config) + + def _llm_bundle( + self, + page_text: str, + project_config: ProjectConfig, + context: ExtractionPageContext | None, + ) -> ExtractionBundle: + if context is not None: + return self.llm_extractor.extract_from_context(context, project_config) + return self.llm_extractor.extract(page_text, project_config) + + def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]: + return self.extract(page_text, project_config).entities + + def extract_attributes( + self, + entity: ExtractedEntity, + page_text: str, + project_config: ProjectConfig, + ) -> dict[str, Any]: + return entity.attributes + + def extract_relations( + self, + entities: list[ExtractedEntity], + page_text: str, + project_config: ProjectConfig, + ) -> list[ExtractedClaim]: + return [] + + def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, Any]) -> ExtractionBundle: + return bundle + + +def normalize_mode(mode: str | None) -> str: + value = (mode or "hybrid").strip().lower().replace("-", "_") + aliases = { + "rule": "rule_only", + "rules": "rule_only", + "rule_based": "rule_only", + "llm": "llm_only", + "ai": "llm_only", + } + return aliases.get(value, value) + + +def rule_extractor_for_domain(domain: str) -> Extractor: + if domain == "perfume": + return PerfumeRuleBasedExtractor() + return GenericRuleBasedExtractor() + + +def mark_bundle(bundle: ExtractionBundle, *, source: str, mode: str) -> None: + for entity in bundle.entities: + entity.metadata = { + **entity.metadata, + "extraction_source": source, + f"{source}_confidence": entity.confidence, + } + for claim in bundle.claims: + claim.metadata = { + **claim.metadata, + "extraction_source": source, + "agreement": f"{source}_only", + f"{source}_confidence": claim.confidence, + "extraction_mode": mode, + } + + +def fallback_bundle(rule_bundle: ExtractionBundle, extractor: HybridExtractor, exc: Exception) -> ExtractionBundle: + reason = str(exc) + bundle = clone_bundle(rule_bundle) + bundle.extractor_name = "hybrid_rule_fallback" + bundle.provider = extractor.llm_provider + bundle.raw_output = { + **bundle.raw_output, + "extraction_mode": "fallback", + "effective_extraction_mode": "rule_only", + "requested_extraction_mode": extractor.mode, + "ai_provider": extractor.llm_provider, + "ai_model": extractor.model, + "ai_warning": reason, + "fallback": "rule_based", + } + for entity in bundle.entities: + entity.metadata["ai_fallback_reason"] = reason + for claim in bundle.claims: + claim.metadata = { + **claim.metadata, + "ai_fallback_reason": reason, + "fallback": "rule_based", + "agreement": "rule_only", + } + claim.confidence_reason = ( + f"{claim.confidence_reason}; AI fallback: {reason}" + if claim.confidence_reason + else f"AI fallback: {reason}" + ) + return bundle + + +def llm_skipped_bundle( + rule_bundle: ExtractionBundle, + extractor: HybridExtractor, + reason: str, + context: ExtractionPageContext | None, +) -> ExtractionBundle: + bundle = clone_bundle(rule_bundle) + bundle.extractor_name = "hybrid_rule_only_routed" + bundle.provider = extractor.llm_provider + bundle.raw_output = { + **bundle.raw_output, + "extraction_mode": extractor.mode, + "effective_extraction_mode": "rule_only", + "requested_extraction_mode": extractor.mode, + "ai_provider": extractor.llm_provider, + "ai_model": extractor.model, + "llm_skipped": True, + "llm_skip_reason": reason, + "page_type": context.page_type if context is not None else None, + "rule_entity_count": len(bundle.entities), + "rule_claim_count": len(bundle.claims), + "llm_entity_count": 0, + "llm_claim_count": 0, + "agreement_claim_count": 0, + "rule_only_claim_count": len(bundle.claims), + "llm_only_claim_count": 0, + "conflict_claim_count": 0, + "comparison": { + "both_agree": 0, + "rule_only": len(bundle.claims), + "llm_only": 0, + "conflict": 0, + "rejected_by_validation": 0, + }, + } + for claim in bundle.claims: + claim.metadata = { + **claim.metadata, + "llm_skipped": True, + "llm_skip_reason": reason, + "agreement": "rule_only", + } + return bundle + + +def llm_skip_reason( + context: ExtractionPageContext | None, + page_text: str, + mode: str, +) -> str | None: + if mode != "hybrid" or context is None: + return None + page_type = str(context.page_type or "UnknownPage") + clean_length = len(context.clean_text or page_text or "") + if page_type in SKIP_LLM_PAGE_TYPES: + return f"page type {page_type} is configured to skip LLM extraction" + if page_type in RULE_ONLY_PAGE_TYPES: + return f"page type {page_type} is routed to rule-only extraction" + if page_type not in LLM_PAGE_TYPES: + return f"page type {page_type} is not in the LLM page-type allowlist" + if clean_length < MIN_CLEAN_TEXT_CHARS_FOR_LLM: + return f"clean text is too short for LLM extraction: {clean_length} chars" + if clean_length > MAX_CLEAN_TEXT_CHARS_FOR_LLM: + return f"clean text is too long for LLM extraction: {clean_length} chars" + return None + + +def merge_bundles( + rule_bundle: ExtractionBundle, + llm_bundle: ExtractionBundle, + extractor: HybridExtractor, +) -> ExtractionBundle: + merged_claims: dict[tuple[str, str, str], ExtractedClaim] = {} + rule_by_subject_predicate: dict[tuple[str, str], list[ExtractedClaim]] = {} + comparison = { + "both_agree": 0, + "rule_only": 0, + "llm_only": 0, + "conflict": 0, + "rejected_by_validation": 0, + } + + for claim in clone_claims(rule_bundle.claims): + key = claim_key(claim) + merged_claims[key] = claim + rule_by_subject_predicate.setdefault(subject_predicate_key(claim), []).append(claim) + + for llm_claim in clone_claims(llm_bundle.claims): + key = claim_key(llm_claim) + existing = merged_claims.get(key) + if existing is not None: + comparison["both_agree"] += 1 + merge_agreement(existing, llm_claim) + continue + + possible_conflicts = rule_by_subject_predicate.get(subject_predicate_key(llm_claim), []) + if possible_conflicts: + comparison["conflict"] += 1 + mark_conflict(llm_claim, possible_conflicts) + for rule_claim in possible_conflicts: + rule_claim.metadata = { + **rule_claim.metadata, + "agreement": "conflict", + "conflict_status": "rule_llm_conflict", + "review_required": True, + "review_reason": "Rule and LLM produced different objects for the same subject and predicate", + } + else: + comparison["llm_only"] += 1 + llm_claim.metadata = { + **llm_claim.metadata, + "agreement": "llm_only", + "review_reason": "LLM-only claim; evidence and ontology validation required", + } + merged_claims[key] = llm_claim + + for claim in merged_claims.values(): + if claim.metadata.get("agreement") == "rule_only": + comparison["rule_only"] += 1 + + mode = extractor.mode + return ExtractionBundle( + entities=dedupe_entities([*clone_entities(rule_bundle.entities), *clone_entities(llm_bundle.entities)]), + claims=list(merged_claims.values()), + extractor_name="hybrid_rule_llm_extractor" if mode == "hybrid" else "compare_rule_llm_extractor", + provider=extractor.llm_provider, + raw_output={ + "extraction_mode": mode, + "provider": extractor.llm_provider, + "model": extractor.model, + "rule_entity_count": len(rule_bundle.entities), + "rule_claim_count": len(rule_bundle.claims), + "llm_entity_count": len(llm_bundle.entities), + "llm_claim_count": len(llm_bundle.claims), + "agreement_claim_count": comparison["both_agree"], + "rule_only_claim_count": comparison["rule_only"], + "llm_only_claim_count": comparison["llm_only"], + "conflict_claim_count": comparison["conflict"], + "comparison": comparison, + "rule_raw_output": safe_raw_output(rule_bundle.raw_output), + "llm_raw_output": safe_raw_output(llm_bundle.raw_output), + }, + ) + + +def merge_agreement(rule_claim: ExtractedClaim, llm_claim: ExtractedClaim) -> None: + rule_confidence = rule_claim.metadata.get("rule_confidence", rule_claim.confidence) + llm_confidence = llm_claim.metadata.get("llm_confidence", llm_claim.confidence) + rule_claim.confidence = max(rule_claim.confidence, llm_claim.confidence) + rule_claim.evidence_text = llm_claim.evidence_text or rule_claim.evidence_text + rule_claim.evidence_summary = llm_claim.evidence_summary or rule_claim.evidence_summary + rule_claim.metadata = { + **rule_claim.metadata, + **llm_claim.metadata, + "agreement": "rule_and_llm", + "rule_confidence": rule_confidence, + "llm_confidence": llm_confidence, + "rule_claim_merge": True, + } + reasons = [rule_claim.confidence_reason, llm_claim.confidence_reason, "Rule and LLM agreed"] + rule_claim.confidence_reason = "; ".join(reason for reason in reasons if reason) + + +def mark_conflict(llm_claim: ExtractedClaim, rule_claims: list[ExtractedClaim]) -> None: + llm_claim.metadata = { + **llm_claim.metadata, + "agreement": "conflict", + "conflict_status": "rule_llm_conflict", + "review_required": True, + "review_reason": "Rule and LLM produced different objects for the same subject and predicate", + "conflicting_rule_objects": [claim_object_key(claim) for claim in rule_claims], + } + + +def clone_bundle(bundle: ExtractionBundle) -> ExtractionBundle: + return ExtractionBundle( + entities=clone_entities(bundle.entities), + claims=clone_claims(bundle.claims), + extractor_name=bundle.extractor_name, + provider=bundle.provider, + raw_output=dict(bundle.raw_output), + ) + + +def clone_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]: + return [replace(entity, attributes=dict(entity.attributes), metadata=dict(entity.metadata)) for entity in entities] + + +def clone_claims(claims: list[ExtractedClaim]) -> list[ExtractedClaim]: + return [replace(claim, metadata=dict(claim.metadata)) for claim in claims] + + +def subject_predicate_key(claim: ExtractedClaim) -> tuple[str, str]: + return ( + claim.subject_name.strip().lower(), + claim.predicate.strip().lower(), + ) + + +def claim_key(claim: ExtractedClaim) -> tuple[str, str, str]: + subject, predicate = subject_predicate_key(claim) + return subject, predicate, claim_object_key(claim) + + +def claim_object_key(claim: ExtractedClaim) -> str: + if claim.object_name: + return claim.object_name.strip().lower() + return json.dumps(claim.object_value, ensure_ascii=False, sort_keys=True, default=str).strip().lower() + + +def safe_raw_output(raw_output: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in raw_output.items() + if key not in {"candidate_entities", "candidate_claims"} + } diff --git a/ontology_platform/crawler_platform/app/core/extractor/validation.py b/ontology_platform/crawler_platform/app/core/extractor/validation.py index a8e5fe2..bcba0f6 100644 --- a/ontology_platform/crawler_platform/app/core/extractor/validation.py +++ b/ontology_platform/crawler_platform/app/core/extractor/validation.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import dataclass, field +import re from typing import Any from crawler_platform.app.config.loader import ProjectConfig @@ -134,6 +135,9 @@ def validate_extraction_bundle(bundle: ExtractionBundle, config: ProjectConfig) claims = [] rejected_claims = [] for claim in bundle.claims: + agreement = str(claim.metadata.get("agreement") or "") + extraction_source = str(claim.metadata.get("extraction_source") or "") + claim_status_for_row = status_for_claim(claim_status, agreement) claim.predicate = normalize_predicate(claim.predicate, config.ontology) claim.subject_type = normalize_entity_type(claim.subject_type, config.domain) claim.subject_name = normalize_entity_name(claim.subject_name, claim.subject_type, config.domain) @@ -147,36 +151,52 @@ def validate_extraction_bundle(bundle: ExtractionBundle, config: ProjectConfig) } reason = invalid_claim_reason(claim, config, allowed_types, page_context) if reason: - rejected_claims.append( - { - "subject": claim.subject_name, - "predicate": claim.predicate, - "object": claim.object_name if claim.object_name is not None else claim.object_value, - "reason": reason, - "page_type": claim.metadata.get("page_type") or page_context.get("page_type"), - "source_zone": claim.metadata.get("source_zone"), + rejected = rejected_claim_payload(claim, reason, page_context) + rejected_claims.append(rejected) + if is_reviewable_claim_reason(reason): + claim.metadata = { + **claim.metadata, + "claim_kind": claim_kind_for(claim_status_for_row, agreement), + "validation_status": "candidate_claim", + "review_required": True, + "review_reason": reason, + "schema_validated": False, } - ) + claim.metadata["confidence_breakdown"] = claim_confidence_breakdown( + claim, + agreement, + extraction_source, + ontology_compatible=True, + ) + claim.confidence = claim.metadata["confidence_breakdown"]["final_confidence"] + claims.append(claim) continue claim.metadata = { **claim.metadata, - "claim_kind": "rule_candidate" if claim_status == "rule_candidate" else "ai_claim", - "validation_status": claim_status, + "claim_kind": claim_kind_for(claim_status_for_row, agreement), + "validation_status": claim_status_for_row, } - breakdown = confidence_breakdown( - llm_confidence=claim.confidence, - evidence_found=bool(claim.metadata.get("evidence_found")), - ontology_compatible=True, - source_zone_allowed=bool(claim.metadata.get("source_zone_allowed")), - ) + if agreement == "conflict": + claim.metadata = { + **claim.metadata, + "validation_status": "candidate_claim", + "review_required": True, + "review_reason": claim.metadata.get("review_reason") + or "Rule and LLM produced conflicting claims", + } + breakdown = claim_confidence_breakdown(claim, agreement, extraction_source) claim.metadata["confidence_breakdown"] = breakdown claim.confidence = breakdown["final_confidence"] claims.append(claim) + comparison = dict(bundle.raw_output.get("comparison") or {}) + if comparison: + comparison["rejected_by_validation"] = len(rejected_claims) bundle.entities = entities bundle.claims = claims bundle.raw_output = { **bundle.raw_output, + **({"comparison": comparison} if comparison else {}), "validation": { "claim_status": claim_status, "accepted_claim_count": len(claims), @@ -196,9 +216,19 @@ def validate_extraction_bundle(bundle: ExtractionBundle, config: ProjectConfig) def claim_status_for_bundle(bundle: ExtractionBundle) -> str: mode = str(bundle.raw_output.get("extraction_mode", "")).lower() + effective_mode = str(bundle.raw_output.get("effective_extraction_mode", "")).lower() fallback = str(bundle.raw_output.get("fallback", "")).lower() - if bundle.provider == "rule_based" or "rule_fallback" in bundle.extractor_name or mode == "fallback" or fallback: + if ( + bundle.provider == "rule_based" + or "rule_fallback" in bundle.extractor_name + or mode == "fallback" + or effective_mode == "rule_only" + or bundle.raw_output.get("llm_skipped") + or fallback + ): return "rule_candidate" + if mode == "compare": + return "candidate_claim" page_context = dict(bundle.raw_output.get("page_context") or {}) page_type = str(page_context.get("page_type") or "") if page_type in {"UnknownPage", "SearchPage", "CategoryPage", "BoardPage"}: @@ -206,6 +236,77 @@ def claim_status_for_bundle(bundle: ExtractionBundle) -> str: return "validated_claim" +def status_for_claim(bundle_status: str, agreement: str) -> str: + if bundle_status in {"rule_candidate", "candidate_claim"}: + return bundle_status + if agreement == "rule_only": + return "rule_candidate" + if agreement == "conflict": + return "candidate_claim" + return bundle_status + + +def claim_kind_for(claim_status: str, agreement: str) -> str: + if claim_status == "rule_candidate" or agreement == "rule_only": + return "rule_candidate" + if agreement == "conflict": + return "conflict_candidate" + return "ai_claim" + + +def claim_confidence_breakdown( + claim: ExtractedClaim, + agreement: str, + extraction_source: str, + *, + ontology_compatible: bool = True, +) -> dict[str, float]: + rule_confidence = _optional_float(claim.metadata.get("rule_confidence")) + llm_confidence = _optional_float(claim.metadata.get("llm_confidence")) + if llm_confidence is None: + llm_confidence = 0.0 if extraction_source == "rule" else claim.confidence + return confidence_breakdown( + llm_confidence=llm_confidence, + evidence_found=bool(claim.metadata.get("evidence_found")), + ontology_compatible=ontology_compatible, + source_zone_allowed=bool(claim.metadata.get("source_zone_allowed")), + rule_confidence=rule_confidence, + rule_agreement=agreement == "rule_and_llm", + ) + + +def rejected_claim_payload( + claim: ExtractedClaim, + reason: str, + page_context: dict[str, Any], +) -> dict[str, Any]: + return { + "subject": claim.subject_name, + "predicate": claim.predicate, + "object": claim.object_name if claim.object_name is not None else claim.object_value, + "reason": reason, + "page_type": claim.metadata.get("page_type") or page_context.get("page_type"), + "source_zone": claim.metadata.get("source_zone"), + } + + +def is_reviewable_claim_reason(reason: str) -> bool: + return reason in { + "claim has no source zone", + "evidence was not found in clean source zones", + "claim has no evidence", + } or reason.startswith("source zone is not claim-allowed") + + +def _optional_float(value: Any) -> float | None: + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + def allowed_entity_types(config: ProjectConfig) -> set[str]: configured = set(config.target_entities) ontology_types = set((config.ontology or {}).get("entity_types", [])) @@ -267,6 +368,8 @@ def invalid_claim_reason( if claim.object_name is not None: if claim.object_type not in allowed_types: return "object type is outside ontology" + if claim.object_type == "Brand" and looks_like_price_text(claim.object_name): + return "brand object looks like a price" if not is_meaningful_text(claim.object_name): return "object is empty, generic, or boilerplate" if canonical_entity_key(claim.subject_name, claim.subject_type) == canonical_entity_key( @@ -316,6 +419,19 @@ def is_meaningful_value(value: Any) -> bool: return True +def looks_like_price_text(value: str | None) -> bool: + if not value: + return False + clean = " ".join(str(value).strip().split()) + return bool( + re.fullmatch( + r"(?:[$€£]\s*)?\d{1,3}(?:,\d{3})*(?:\.\d+)?\s*(?:원|KRW|USD|EUR|JPY|\$|€|£)?", + clean, + flags=re.IGNORECASE, + ) + ) + + def is_meaningful_text(value: str | None, allow_short: bool = False) -> bool: if value is None: return False diff --git a/ontology_platform/crawler_platform/app/core/ontology/relation_schema.py b/ontology_platform/crawler_platform/app/core/ontology/relation_schema.py index 102c075..ee492d4 100644 --- a/ontology_platform/crawler_platform/app/core/ontology/relation_schema.py +++ b/ontology_platform/crawler_platform/app/core/ontology/relation_schema.py @@ -110,23 +110,31 @@ def confidence_breakdown( evidence_found: bool, ontology_compatible: bool, source_zone_allowed: bool, + rule_confidence: float | None = None, + rule_agreement: bool = False, source_trust: float | None = None, ) -> dict[str, float]: schema_confidence = 1.0 evidence_confidence = 0.95 if evidence_found else 0.0 ontology_confidence = 0.95 if ontology_compatible else 0.0 zone_confidence = 0.9 if source_zone_allowed else 0.0 + rule_value = rule_confidence if rule_confidence is not None else 0.0 + agreement_confidence = 0.95 if rule_agreement else 0.0 trust = source_trust if source_trust is not None else 0.8 final = ( - llm_confidence * 0.35 - + schema_confidence * 0.15 - + evidence_confidence * 0.2 - + ontology_confidence * 0.2 + llm_confidence * 0.30 + + rule_value * 0.10 + + agreement_confidence * 0.10 + + schema_confidence * 0.10 + + evidence_confidence * 0.18 + + ontology_confidence * 0.12 + zone_confidence * 0.05 + trust * 0.05 ) return { "llm_confidence": round(llm_confidence, 4), + "rule_confidence": round(rule_value, 4), + "rule_agreement_confidence": agreement_confidence, "schema_confidence": schema_confidence, "evidence_confidence": evidence_confidence, "ontology_confidence": ontology_confidence, diff --git a/ontology_platform/crawler_platform/app/core/research/graph_research_loop.py b/ontology_platform/crawler_platform/app/core/research/graph_research_loop.py index 1abf197..10ac097 100644 --- a/ontology_platform/crawler_platform/app/core/research/graph_research_loop.py +++ b/ontology_platform/crawler_platform/app/core/research/graph_research_loop.py @@ -284,6 +284,7 @@ class GraphResearchLoop: analyzed = False claim_count = 0 entity_count = 0 + extraction_summary: dict[str, Any] = {} if ( fetch_result.crawl_status == "success" and parser_result.extraction_status != "failed" @@ -311,6 +312,7 @@ class GraphResearchLoop: analyzed = True claim_count = len(claims) entity_count = len(bundle.entities) + extraction_summary = extraction_summary_from_raw(bundle.raw_output) return { "status": "explored", @@ -320,6 +322,7 @@ class GraphResearchLoop: "analyzed": analyzed, "claim_count": claim_count, "entity_count": entity_count, + "extraction": extraction_summary, "enqueued": enqueued, "scored_links": scored_links[:20], } @@ -369,3 +372,15 @@ def normalize_url(url: str) -> str: def normalized_host(url: str) -> str: return urlparse(str(url)).netloc.lower() + + +def extraction_summary_from_raw(raw_output: dict[str, Any]) -> dict[str, Any]: + return { + "extraction_mode": raw_output.get("extraction_mode"), + "effective_extraction_mode": raw_output.get("effective_extraction_mode"), + "llm_skipped": bool(raw_output.get("llm_skipped")), + "llm_skip_reason": raw_output.get("llm_skip_reason"), + "fallback_used": bool(raw_output.get("fallback")), + "agreement_claim_count": int(raw_output.get("agreement_claim_count") or 0), + "conflict_claim_count": int(raw_output.get("conflict_claim_count") or 0), + } diff --git a/ontology_platform/crawler_platform/app/domains/perfume/extractor.py b/ontology_platform/crawler_platform/app/domains/perfume/extractor.py index 6eba95b..013cacc 100644 --- a/ontology_platform/crawler_platform/app/domains/perfume/extractor.py +++ b/ontology_platform/crawler_platform/app/domains/perfume/extractor.py @@ -167,6 +167,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): brand = next((entity for entity in entities if entity.entity_type == "Brand"), None) for card in product_cards: if brand: + evidence = brand_evidence_for_product(brand, str(card["name"]), page_text) or brand.evidence_text or brand.name claims.append( ExtractedClaim( str(card["name"]), @@ -174,7 +175,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): "hasBrand", brand.name, "Brand", - evidence_text=brand.evidence_text or brand.name, + evidence_text=evidence, confidence=0.72, confidence_reason="site brand inferred from listing page", ) @@ -199,6 +200,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): claims: list[ExtractedClaim] = [] brand = next((entity for entity in entities if entity.entity_type == "Brand"), None) if brand: + evidence = brand_evidence_for_product(brand, perfume.name, page_text) or brand.evidence_text or brand.name claims.append( ExtractedClaim( perfume.name, @@ -206,7 +208,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): "hasBrand", brand.name, "Brand", - evidence_text=brand.evidence_text or brand.name, + evidence_text=evidence, confidence=0.78, confidence_reason="brand pattern matched", ) @@ -273,14 +275,16 @@ def extract_brand(page_text: str, product_name: str) -> str | None: for pattern in patterns: match = re.search(pattern, page_text, flags=re.IGNORECASE) if match: - return cleanup_value(match.group("brand")) - inferred = infer_site_brand(page_text) + brand = valid_brand(cleanup_value(match.group("brand"))) + if brand: + return brand + inferred = infer_site_brand(page_text, product_name) if inferred: return inferred lines = [line.strip() for line in page_text.splitlines() if line.strip()] if len(lines) >= 2 and lines[1].lower() not in product_name.lower(): candidate = cleanup_value(lines[1]) - if len(candidate) <= 80 and not looks_like_navigation(candidate) and product_line_score(candidate) <= 0: + if valid_brand(candidate) and len(candidate) <= 80 and not looks_like_navigation(candidate) and product_line_score(candidate) <= 0: return candidate return None @@ -301,6 +305,19 @@ def product_line_score(value: str) -> int: return score +def looks_like_price(value: str | None) -> bool: + if not value: + return False + clean = cleanup_value(str(value)) + return bool( + re.fullmatch( + r"(?:[$€£]\s*)?\d{1,3}(?:,\d{3})*(?:\.\d+)?\s*(?:원|KRW|USD|EUR|JPY|\$|€|£)?", + clean, + flags=re.IGNORECASE, + ) + ) + + def looks_like_metric_or_price(value: str) -> bool: clean = value.replace(",", "").strip() if re.fullmatch(r"\d+(?:\.\d+)?", clean): @@ -385,7 +402,10 @@ def dedupe_product_cards(cards: list[dict[str, object]]) -> list[dict[str, objec return result -def infer_site_brand(page_text: str) -> str | None: +def infer_site_brand(page_text: str, product_name: str = "") -> str | None: + brand_context = f"{product_name}\n{page_text[:1200]}".lower() + if "forment" in brand_context or "포맨트" in brand_context: + return "FORMENT" if "912 공식 홈페이지" in page_text or "912" in page_text[:500]: return "912" return None @@ -394,9 +414,19 @@ def infer_site_brand(page_text: str) -> str | None: def valid_brand(value: str | None) -> str | None: if not value or is_template_placeholder(value): return None + if looks_like_price(value) or looks_like_metric_or_price(value): + return None return value +def brand_evidence_for_product(brand: ExtractedEntity, product_name: str, page_text: str) -> str | None: + if brand.name == "FORMENT" and "포맨트" in product_name: + return product_name + if brand.name.lower() in page_text.lower(): + return brand.name + return None + + def extract_field_values(field: str, page_text: str) -> list[tuple[str, str]]: if field in NOTE_LABELS: return extract_labeled_values(page_text, NOTE_LABELS[field]) diff --git a/ontology_platform/data/crawler_platform.db b/ontology_platform/data/crawler_platform.db index ae2a58b..b546ca1 100644 Binary files a/ontology_platform/data/crawler_platform.db and b/ontology_platform/data/crawler_platform.db differ diff --git a/ontology_platform/docs/phases/26_05_19_engine_respect_plan/phase_07_001_hybrid_rule_llm_extraction.md b/ontology_platform/docs/phases/26_05_19_engine_respect_plan/phase_07_001_hybrid_rule_llm_extraction.md new file mode 100644 index 0000000..7ef26d5 --- /dev/null +++ b/ontology_platform/docs/phases/26_05_19_engine_respect_plan/phase_07_001_hybrid_rule_llm_extraction.md @@ -0,0 +1,415 @@ +# Phase 7. Hybrid Rule + LLM Extraction + +작성일: 2026-05-21 + +범위: `ontology_platform`에 포함된 product backend, 특히 `crawler_platform/app/core/extractor`, crawl/research API, Review/Page Analysis UI. + +주의: 이 phase는 OntoCast core engine을 바꾸지 않는다. 룰 기반 product extraction과 LLM JSON extraction을 product backend 레벨에서 결합한다. `/process` OntoCast workflow는 후처리/RDF 변환 후보로만 연결한다. + +## 목표 + +룰 기반 추출의 장점인 빠름, 비용 없음, 예측 가능성, 명확한 패턴 인식을 살리고, LLM 추출의 장점인 의미 해석, 타입 추론, 관계 추출, 한국어/비정형 문장 대응력을 더한다. + +핵심 원칙: + +- 룰은 먼저 실행되는 baseline extractor이자 검증 가드레일이다. +- LLM은 의미 추출기이며, evidence/ontology/source zone 검증을 통과해야 한다. +- LLM 실패는 전체 실패가 아니라 rule fallback으로 처리한다. +- 룰과 LLM이 같은 claim을 찾으면 신뢰도를 올린다. +- 룰과 LLM이 충돌하면 자동 승인하지 않고 review로 보낸다. + +## 현재 기준선 + +현재 확인된 동작: + +- `ont_platform/api/routes/extraction.py` + - `/api/v1/extract/url`, `/api/v1/process/url`, `/process/url` + - `LightweightExtractor(use_llm=False)` 고정 + - 룰 기반 기초 후보만 생성 + +- `ont_platform/core/crawler/jobs.py` + - `/api/v1/crawl/jobs` + - `LightweightExtractor(use_llm=False)` 고정 + - 기본 수집 + 기초 후보 저장 + +- `crawler_platform/app/core/extractor/factory.py` + - `provider in {"openai", "ollama", "lm_studio"}`이면 `LLMJsonExtractor` + - 그 외에는 domain별 rule extractor + +- `crawler_platform/app/core/extractor/ai_provider.py` + - LLM primary, compact retry, JSON repair, rule fallback이 이미 일부 존재 + - LLM 결과에 rule claim을 merge하는 `_merge_rule_fallback_claims()`가 이미 존재 + +- `crawler_platform/app/core/extractor/validation.py` + - evidence zone, ontology predicate, source zone, confidence breakdown 검증이 존재 + +- `web/frontend/src/pages/CrawlPage.tsx`, `ResearchPage.tsx` + - request type에는 `extractor_provider` 필드가 있으나 화면에서 선택 UI는 없음 + +## Phase 7.1 Baseline Audit + +목표: 기존 동작을 깨지 않기 위해 현재 추출 결과와 저장 구조를 고정한다. + +작업: + +- `rule_based`, `lm_studio`, `openai`, `ollama` provider별 request/response 샘플을 만든다. +- crawl, crawl-site, research/run 경로에서 extractor가 어떻게 선택되는지 문서화한다. +- `Claim`, `Entity`, `ExtractionLog`에 저장되는 `extractor_name`, `provider`, `metadata_json`, `confidence_breakdown` 구조를 샘플로 기록한다. +- LM Studio가 꺼진 상태, 켜진 상태, 모델명 누락 상태를 각각 재현한다. + +주요 파일: + +- `crawler_platform/app/core/extractor/factory.py` +- `crawler_platform/app/core/extractor/ai_provider.py` +- `crawler_platform/app/core/extractor/rule_based.py` +- `crawler_platform/app/core/database/repository.py` +- `crawler_platform/app/api/routes.py` + +완료 기준: + +- 기존 `rule_based`와 `lm_studio` 동작이 재현 가능하다. +- 최소 1개 상품 페이지 fixture로 rule 결과와 LLM 결과 샘플이 있다. +- LM Studio 장애 시 현재 fallback 결과가 확인되어 있다. + +## Phase 7.2 Extraction Mode Contract + +목표: provider와 실행 전략을 분리한다. + +현재 문제: + +- `extractor_provider`가 provider이면서 실행 전략 역할도 한다. +- 사용자는 rule only, LLM only, hybrid, compare를 명시적으로 선택할 수 없다. + +제안 계약: + +```json +{ + "extraction_mode": "hybrid", + "extractor_provider": "lm_studio", + "extractor_model": "deepseek-r1-distill-qwen-7b", + "extractor_base_url": "http://localhost:1234/v1", + "fallback_to_rules": true +} +``` + +지원 mode: + +- `rule_only`: rule extractor만 실행 +- `llm_only`: LLM extractor만 실행, fallback 선택 가능 +- `hybrid`: rule 먼저 실행, LLM 실행, 병합/검증 +- `compare`: rule 결과와 LLM 결과를 모두 보존하고 차이를 metadata/log에 저장 + +호환성: + +- 기존 `extractor_provider: "rule_based"`는 `extraction_mode="rule_only"`로 해석한다. +- 기존 `extractor_provider: "lm_studio" | "openai" | "ollama"`는 당분간 `extraction_mode="hybrid"`로 해석한다. +- 기존 UI가 provider를 보내지 않는 경우 기본값은 `hybrid + lm_studio`로 유지한다. + +수정 파일: + +- `crawler_platform/app/api/routes.py` +- `web/frontend/src/lib/api/crawl.ts` +- `web/frontend/src/lib/api/research.ts` + +완료 기준: + +- 기존 요청이 깨지지 않는다. +- 새 요청 필드로 `rule_only`, `llm_only`, `hybrid`, `compare`가 구분된다. +- API response/log에 실제 실행 mode가 남는다. + +## Phase 7.3 Hybrid Extractor + +목표: 명시적인 `HybridExtractor`를 추가한다. + +신규 파일: + +- `crawler_platform/app/core/extractor/hybrid.py` + +실행 순서: + +1. domain에 맞는 rule extractor 실행 +2. LLM extractor 실행 +3. entity dedupe +4. claim dedupe +5. rule-LLM agreement 계산 +6. conflict 계산 +7. metadata에 extraction mode와 comparison 결과 기록 +8. validation pipeline으로 전달 + +필수 metadata: + +```json +{ + "extraction_mode": "hybrid", + "rule_entity_count": 12, + "rule_claim_count": 8, + "llm_entity_count": 15, + "llm_claim_count": 13, + "agreement_claim_count": 6, + "rule_only_claim_count": 2, + "llm_only_claim_count": 7, + "conflict_claim_count": 1 +} +``` + +병합 규칙: + +- 동일 subject/predicate/object claim은 하나로 합친다. +- rule과 LLM이 모두 찾은 claim은 `agreement="rule_and_llm"`로 표시한다. +- rule만 찾은 claim은 `agreement="rule_only"`와 `claim_kind="rule_candidate"`로 표시한다. +- LLM만 찾은 claim은 evidence 검증 전까지 `agreement="llm_only"`로 표시한다. +- 같은 subject/predicate인데 object가 다르면 `conflict_status="rule_llm_conflict"`로 표시하고 review로 보낸다. + +수정 파일: + +- `crawler_platform/app/core/extractor/factory.py` +- `crawler_platform/app/core/extractor/ai_provider.py` +- `crawler_platform/app/core/extractor/base.py` + +완료 기준: + +- `extractor_provider="hybrid"` 또는 `extraction_mode="hybrid"`로 실행 가능하다. +- LLM 실패 시 rule 결과만으로 성공 response가 나온다. +- rule/LLM agreement가 claim metadata에 남는다. + +## Phase 7.4 Confidence And Validation Upgrade + +목표: LLM hallucination을 줄이고, rule agreement를 신뢰도에 반영한다. + +현재 confidence 구성: + +- `llm_confidence` +- `schema_confidence` +- `evidence_confidence` +- `ontology_confidence` +- `source_zone_confidence` +- `source_trust` +- `final_confidence` + +추가 항목: + +```json +{ + "rule_confidence": 0.75, + "rule_agreement_confidence": 0.95, + "llm_confidence": 0.82, + "evidence_confidence": 0.95, + "ontology_confidence": 0.95, + "source_zone_confidence": 0.9, + "final_confidence": 0.89 +} +``` + +정책: + +- rule+LLM agreement가 있으면 confidence bonus를 준다. +- LLM-only claim은 evidence가 없으면 자동 승인하지 않는다. +- source zone을 찾지 못한 claim은 review로 보낸다. +- ontology에 없는 predicate는 reject한다. +- rule-only claim은 기본적으로 `rule_candidate`로 남긴다. +- 충돌 claim은 `review_required=true`로 남긴다. + +수정 파일: + +- `crawler_platform/app/core/ontology/relation_schema.py` +- `crawler_platform/app/core/extractor/validation.py` +- `crawler_platform/app/core/database/repository.py` + +완료 기준: + +- Review 화면에서 `confidence_breakdown.rule_agreement_confidence`를 볼 수 있다. +- evidence 없는 LLM-only claim이 `validated_claim`으로 자동 저장되지 않는다. +- rule+LLM 일치 claim은 더 높은 final confidence를 받는다. + +## Phase 7.5 Compare Mode + +목표: rule 결과와 LLM 결과를 나란히 비교하여 품질 튜닝에 사용한다. + +작업: + +- `compare` mode에서 rule bundle과 LLM bundle을 모두 실행한다. +- 최종 저장은 병합 결과로 하되, `ExtractionLog.raw_output`에 원본 두 결과를 보존한다. +- Page Analysis에서 diff summary를 표시한다. + +diff category: + +- `both_agree` +- `rule_only` +- `llm_only` +- `conflict` +- `rejected_by_validation` + +UI 표시: + +- Page Analysis: page별 extractor run 요약, rule/LLM candidate count, conflict count +- Review: claim detail에서 agreement badge 표시 + +수정 파일: + +- `crawler_platform/app/core/database/models.py` +- `crawler_platform/app/core/database/repository.py` +- `crawler_platform/app/api/routes.py` +- `web/frontend/src/pages/PageAnalysisPage.tsx` +- `web/frontend/src/pages/ReviewPage.tsx` +- `web/frontend/src/lib/api/platform.ts` + +완료 기준: + +- 같은 페이지에서 rule과 LLM의 차이를 확인할 수 있다. +- `llm_only`와 `conflict` claim을 review에서 필터링할 수 있다. + +## Phase 7.6 UI Controls + +목표: 사용자가 화면에서 mode/provider/model/base URL을 선택할 수 있게 한다. + +대상 화면: + +- Crawl Page +- Research Page + +추가 컨트롤: + +- Extraction mode select + - Rule only + - Hybrid + - LLM only + - Compare +- LLM provider select + - LM Studio + - OpenAI + - Ollama +- Model input +- Base URL input +- Fallback to rules toggle +- Optional model list refresh button + +동작: + +- mode가 `rule_only`이면 provider/model/base URL 입력을 숨긴다. +- provider가 `lm_studio`이면 기본 base URL은 `http://localhost:1234/v1`이다. +- provider가 `ollama`이면 기본 base URL은 `http://localhost:11434/api/chat`이다. +- 모델 목록은 `/extractors/models`를 사용한다. + +수정 파일: + +- `web/frontend/src/pages/CrawlPage.tsx` +- `web/frontend/src/pages/ResearchPage.tsx` +- `web/frontend/src/lib/api/crawl.ts` +- `web/frontend/src/lib/api/research.ts` + +완료 기준: + +- UI에서 mode/provider/model/base URL을 지정할 수 있다. +- 지정한 값이 `/crawl-site/by-project`, `/research/run/by-project` 요청에 포함된다. +- rule only 선택 시 LM Studio가 꺼져 있어도 작업이 시작된다. + +## Phase 7.7 Smart Routing Policy + +목표: 모든 페이지에 LLM을 쓰지 않고 가치 있는 페이지에 집중한다. + +정책: + +- `ProductPage`: hybrid 기본 +- `BrandStoryPage`: hybrid 기본 +- `ReviewPage`: hybrid 또는 llm_only +- `CategoryPage`: rule_only 또는 skip LLM +- `SearchPage`: skip LLM +- `BoardPage`: rule_only 또는 review 후보 +- 본문 길이가 너무 짧으면 rule_only +- rule 결과가 충분하고 deterministic field만 필요한 경우 LLM 생략 가능 +- research goal이 있으면 LLM 우선 + +추가 설정: + +```json +{ + "llm_page_types": ["ProductPage", "BrandStoryPage", "ReviewPage"], + "skip_llm_page_types": ["CategoryPage", "SearchPage"], + "min_clean_text_chars_for_llm": 300, + "max_llm_pages_per_job": 30 +} +``` + +수정 파일: + +- `crawler_platform/app/core/crawler/site_crawler.py` +- `crawler_platform/app/core/crawler/pipeline.py` +- `crawler_platform/app/core/research/graph_research_loop.py` + +완료 기준: + +- LLM 호출 수가 page type 정책에 따라 제한된다. +- 중요 페이지는 hybrid 추출을 받는다. +- crawl job metadata에 LLM skipped reason이 남는다. + +## Phase 7.8 OntoCast `/process` Handoff + +목표: product crawl/research 결과를 OntoCast RDF 변환과 연결한다. + +역할 분리: + +- product crawl/research: web acquisition, entity/claim 후보 수집 +- validation/review: claim 품질 관리 +- `/process`: 검증된 문서나 claim 묶음을 RDF/Turtle로 정리하는 후처리 + +작업: + +- validated claim 묶음을 OntoCast input JSON으로 변환한다. +- 프로젝트 단위 또는 selected document 단위로 `/process`를 호출할 수 있게 한다. +- `/process` 결과 Turtle을 export/graph view와 연결한다. + +완료 기준: + +- 사용자가 검증된 claim subset을 RDF/Turtle로 내보낼 수 있다. +- `/process`는 모든 crawl page마다 자동 실행되지 않는다. +- 비용 큰 LLM workflow는 명시적 후처리로만 실행된다. + +## Phase 7.9 Tests And Observability + +목표: rule/LLM/hybrid/compare 동작을 재현 가능하게 만든다. + +테스트: + +- rule only는 LM Studio 없이 통과 +- llm only는 mock LLM으로 deterministic response 검증 +- hybrid는 rule+LLM agreement metadata 생성 +- LLM 실패 시 fallback으로 성공 +- evidence 없는 LLM-only claim은 자동 승인되지 않음 +- conflict claim은 review_required가 true +- UI payload에 mode/provider/model/base URL 포함 + +관측성: + +- ExtractionLog에 mode/provider/model/base URL 저장 +- job metadata에 LLM call count, fallback count, skipped count 저장 +- Review/Page Analysis에서 agreement/fallback/conflict를 표시 + +완료 기준: + +- local LM Studio가 꺼져 있어도 rule/hybrid fallback 테스트가 통과한다. +- mock LLM 기반 테스트가 CI에서 안정적으로 돈다. +- crawl/research 결과에서 어떤 extractor가 어떤 이유로 사용됐는지 추적 가능하다. + +## 최종 Acceptance Gate + +- [ ] UI에서 Rule only, Hybrid, LLM only, Compare 선택 가능 +- [ ] Hybrid mode가 rule baseline과 LLM semantic extraction을 병합 +- [ ] LLM 장애 시 rule fallback으로 job이 실패하지 않음 +- [ ] evidence 없는 LLM-only claim이 자동 validated로 들어가지 않음 +- [ ] rule+LLM agreement가 confidence에 반영됨 +- [ ] conflict claim이 review_required로 표시됨 +- [ ] Page Analysis에서 rule/LLM 비교 결과 확인 가능 +- [ ] Review에서 extraction method, agreement, confidence breakdown 확인 가능 +- [ ] `/process`는 후처리 RDF 변환 경로로 분리 유지 + +## 권장 구현 순서 + +1. Phase 7.1 Baseline Audit +2. Phase 7.2 Extraction Mode Contract +3. Phase 7.3 Hybrid Extractor +4. Phase 7.4 Confidence And Validation Upgrade +5. Phase 7.6 UI Controls +6. Phase 7.5 Compare Mode +7. Phase 7.7 Smart Routing Policy +8. Phase 7.8 OntoCast `/process` Handoff +9. Phase 7.9 Tests And Observability diff --git a/ontology_platform/docs/phases/PHASE_INDEX.md b/ontology_platform/docs/phases/PHASE_INDEX.md index fd82835..e51c766 100644 --- a/ontology_platform/docs/phases/PHASE_INDEX.md +++ b/ontology_platform/docs/phases/PHASE_INDEX.md @@ -87,3 +87,14 @@ FILE: ./26_05_19_engine_respect_plan/phase_06_001_maintenance_loop_operations.md 2) Analyst/Researcher/Curator/Auditor/Fixer/Advisor 梨낆엫 ?뺤쓽 [?꾨즺] 3) `auth`, `audit`, `billing`, `realtime` 珥덉븞 紐⑤뱢???댁쁺 寃쎄퀎 ?뺣━ [?꾨즺] 4) destructive fix???щ엺 ?뱀씤 寃뚯씠?몃? 諛섎뱶???듦낵?섎룄濡??ㅺ퀎 [?꾨즺] + +--- + +PHASE 7. Hybrid Rule + LLM Extraction +FILE: ./26_05_19_engine_respect_plan/phase_07_001_hybrid_rule_llm_extraction.md + +1) rule baseline, LLM extraction, fallback, validation, Review UI 흐름을 기준선으로 고정 [신규] +2) `rule_only`, `llm_only`, `hybrid`, `compare` extraction mode 계약 정의 [신규] +3) product backend에 명시적 HybridExtractor와 rule/LLM agreement metadata 추가 [신규] +4) confidence breakdown에 rule agreement와 conflict/review 정책 반영 [신규] +5) Crawl/Research UI에서 mode/provider/model/base URL 선택 지원 [신규] diff --git a/ontology_platform/ont_platform/api/main.py b/ontology_platform/ont_platform/api/main.py index 3245ea4..4627c13 100644 --- a/ontology_platform/ont_platform/api/main.py +++ b/ontology_platform/ont_platform/api/main.py @@ -49,6 +49,7 @@ from ont_platform.api.product_backend import include_product_backend # noqa: E4 platform_config = importlib.import_module("ont_platform.config") logger = logging.getLogger(__name__) +_startup_error: str | None = None def _resolve_ontocast_version() -> str: @@ -124,8 +125,14 @@ def _include_phase_routers(app: FastAPI) -> None: @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: """FastAPI lifespan: build ToolBox + workflow once on startup.""" + global _startup_error settings = platform_config.load_settings() - await initialize_app_context(settings) + try: + await initialize_app_context(settings) + _startup_error = None + except Exception as exc: # noqa: BLE001 + _startup_error = str(exc) + logger.exception("App context initialization failed; product backend will remain available") try: yield finally: @@ -149,8 +156,23 @@ def create_app() -> FastAPI: # ─── /health ────────────────────────────────────────────────────── @app.get("/health", tags=["meta"]) - async def health(ctx: Annotated[AppContext, Depends(get_app_context)]) -> JSONResponse: - """Liveness check. 503 if the LLM isn't wired.""" + async def health() -> JSONResponse: + """Liveness check for the HTTP service and optional LLM readiness.""" + settings = platform_config.load_settings() + if _startup_error: + return JSONResponse( + status_code=503, + content={ + "status": "degraded", + "error": _startup_error, + "platform_version": PLATFORM_VERSION, + "ontocast_version": ONTOCAST_VERSION, + "phase": int(settings.phase), + "storage_backend": settings.storage_backend, + }, + ) + + ctx = get_app_context() if ctx.tools.llm is None: return JSONResponse( status_code=503, @@ -170,8 +192,16 @@ def create_app() -> FastAPI: # ─── /info ──────────────────────────────────────────────────────── @app.get("/info", tags=["meta"]) - async def info(ctx: Annotated[AppContext, Depends(get_app_context)]) -> JSONResponse: + async def info() -> JSONResponse: """Service-level capabilities (mirrors OntoCast /info semantics).""" + settings = platform_config.load_settings() + phase = int(settings.phase) + storage_backend = settings.storage_backend + if not _startup_error: + ctx = get_app_context() + phase = int(ctx.settings.phase) + storage_backend = ctx.settings.storage_backend + return JSONResponse( status_code=200, content={ @@ -185,8 +215,9 @@ def create_app() -> FastAPI: "capabilities": ["text-to-triples", "ontology-extraction"], "input_types": ["text", "json", "pdf", "markdown"], "output_types": ["turtle", "json"], - "phase": int(ctx.settings.phase), - "storage_backend": ctx.settings.storage_backend, + "phase": phase, + "storage_backend": storage_backend, + "startup_error": _startup_error, }, ) diff --git a/ontology_platform/pyproject.toml b/ontology_platform/pyproject.toml index a74f552..c6d01d8 100644 --- a/ontology_platform/pyproject.toml +++ b/ontology_platform/pyproject.toml @@ -71,7 +71,7 @@ dependencies = [ # PHASE3 Acceptance Gate 통과 후 활성화 # 버전 고정: experimental API 변경 위험 (Neo4j GraphRAG 분석 §13.1) # "neo4j-graphrag[openai,experimental]==1.16.0", - # "neo4j>=5.18.1,<7", + "neo4j>=5.18.1,<7", # ─── 운영 인프라 ─────────────────────────────────────────────────── # Job Queue diff --git a/ontology_platform/tests/unit/test_perfume_extraction_brand_price.py b/ontology_platform/tests/unit/test_perfume_extraction_brand_price.py new file mode 100644 index 0000000..170d6f6 --- /dev/null +++ b/ontology_platform/tests/unit/test_perfume_extraction_brand_price.py @@ -0,0 +1,79 @@ +from crawler_platform.app.config.loader import ProjectConfig +from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractionBundle +from crawler_platform.app.core.extractor.validation import validate_extraction_bundle +from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor + + +def perfume_config() -> ProjectConfig: + return ProjectConfig( + project_name="perfume-test", + domain="perfume", + target_entities=["Perfume"], + fields=[], + sources=[], + ontology={ + "entity_types": [ + "Perfume", + "Product", + "Brand", + "Note", + "Accord", + "Mood", + "Season", + "Occasion", + "Review", + ], + "predicates": ["hasBrand", "hasPrice"], + }, + ) + + +def test_forment_product_price_line_is_not_extracted_as_brand() -> None: + text = "\n".join( + [ + "포맨트 시그니처 퍼퓸 코튼메모리", + "62,000원", + "49,000원", + "기억의 표면을 어루만지다", + ] + ) + + bundle = PerfumeRuleBasedExtractor().extract(text, perfume_config()) + + brands = [entity.name for entity in bundle.entities if entity.entity_type == "Brand"] + assert brands == ["FORMENT"] + assert "62,000원" not in brands + assert any( + claim.predicate == "hasBrand" + and claim.object_name == "FORMENT" + and claim.evidence_text == "포맨트 시그니처 퍼퓸 코튼메모리" + for claim in bundle.claims + ) + assert any( + claim.predicate == "hasPrice" + and claim.object_value == {"amount": 62000.0, "currency": "원"} + for claim in bundle.claims + ) + + +def test_validation_rejects_price_like_brand_object() -> None: + bundle = ExtractionBundle( + claims=[ + ExtractedClaim( + "포맨트 시그니처 퍼퓸 코튼메모리", + "Perfume", + "hasBrand", + "62,000원", + "Brand", + evidence_text="포맨트 시그니처 퍼퓸 코튼메모리 62,000원", + confidence=0.9, + ) + ], + extractor_name="rule_only_extractor", + raw_output={"extraction_mode": "rule_only"}, + ) + + result = validate_extraction_bundle(bundle, perfume_config()) + + assert result.bundle.claims == [] + assert result.rejected_claims[0]["reason"] == "brand object looks like a price" diff --git a/ontology_platform/tests/unit/test_phase7_hybrid_extraction.py b/ontology_platform/tests/unit/test_phase7_hybrid_extraction.py new file mode 100644 index 0000000..3db3fb5 --- /dev/null +++ b/ontology_platform/tests/unit/test_phase7_hybrid_extraction.py @@ -0,0 +1,144 @@ +from crawler_platform.app.config.loader import ProjectConfig +from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractionBundle, ExtractionPageContext +from crawler_platform.app.core.extractor.hybrid import HybridExtractor, mark_bundle, merge_bundles +from crawler_platform.app.core.extractor.validation import validate_extraction_bundle + + +def project_config() -> ProjectConfig: + return ProjectConfig( + project_name="phase7-test", + domain="generic", + target_entities=["Thing"], + fields=[], + sources=[], + ontology={ + "entity_types": ["Thing"], + "predicates": ["hasPart", "color", "size"], + "relation_types": { + "hasPart": { + "allowed_subject_types": ["Thing"], + "allowed_object_types": ["Thing"], + "confidence_rules": {"min_confidence": 0.1}, + }, + "color": { + "allowed_subject_types": ["Thing"], + "literal_value": True, + "confidence_rules": {"min_confidence": 0.1}, + }, + "size": { + "allowed_subject_types": ["Thing"], + "literal_value": True, + "confidence_rules": {"min_confidence": 0.1}, + }, + }, + }, + ) + + +def test_hybrid_merge_marks_agreement_conflict_and_counts() -> None: + rule = ExtractionBundle( + claims=[ + ExtractedClaim("A", "Thing", "hasPart", "B", "Thing", confidence=0.7), + ExtractedClaim("A", "Thing", "color", object_value="red", confidence=0.6), + ] + ) + llm = ExtractionBundle( + claims=[ + ExtractedClaim("A", "Thing", "hasPart", "B", "Thing", confidence=0.8), + ExtractedClaim("A", "Thing", "color", object_value="blue", confidence=0.65), + ExtractedClaim("A", "Thing", "size", object_value="large", confidence=0.5), + ] + ) + mark_bundle(rule, source="rule", mode="compare") + mark_bundle(llm, source="llm", mode="compare") + + merged = merge_bundles(rule, llm, HybridExtractor("generic", mode="compare")) + + assert merged.extractor_name == "compare_rule_llm_extractor" + assert merged.raw_output["comparison"] == { + "both_agree": 1, + "rule_only": 0, + "llm_only": 1, + "conflict": 1, + "rejected_by_validation": 0, + } + by_predicate = {(claim.predicate, claim.object_name or claim.object_value): claim for claim in merged.claims} + assert by_predicate[("hasPart", "B")].metadata["agreement"] == "rule_and_llm" + assert by_predicate[("color", "red")].metadata["review_required"] is True + assert by_predicate[("color", "blue")].metadata["conflict_status"] == "rule_llm_conflict" + assert by_predicate[("size", "large")].metadata["agreement"] == "llm_only" + + +def test_validation_keeps_evidence_missing_llm_only_claim_as_candidate() -> None: + bundle = ExtractionBundle( + claims=[ + ExtractedClaim( + "A", + "Thing", + "color", + object_value="blue", + confidence=0.8, + metadata={ + "agreement": "llm_only", + "extraction_source": "llm", + "llm_confidence": 0.8, + }, + ) + ], + extractor_name="hybrid_rule_llm_extractor", + provider="lm_studio", + raw_output={"extraction_mode": "hybrid"}, + ) + + result = validate_extraction_bundle(bundle, project_config()) + + assert len(result.bundle.claims) == 1 + claim = result.bundle.claims[0] + assert claim.metadata["validation_status"] == "candidate_claim" + assert claim.metadata["review_required"] is True + assert claim.metadata["review_reason"] == "claim has no evidence" + + +def test_validation_marks_rule_only_claim_as_rule_candidate() -> None: + bundle = ExtractionBundle( + claims=[ + ExtractedClaim( + "Alpha", + "Thing", + "color", + object_value="red", + evidence_text="Alpha color red", + confidence=0.7, + metadata={ + "agreement": "rule_only", + "extraction_source": "rule", + "rule_confidence": 0.7, + }, + ) + ], + extractor_name="hybrid_rule_llm_extractor", + provider="lm_studio", + raw_output={"extraction_mode": "hybrid"}, + ) + + result = validate_extraction_bundle(bundle, project_config()) + + assert result.bundle.claims[0].metadata["validation_status"] == "rule_candidate" + assert result.bundle.claims[0].metadata["claim_kind"] == "rule_candidate" + + +def test_hybrid_smart_routing_skips_llm_for_category_page() -> None: + context = ExtractionPageContext( + url="https://example.test/category", + final_url=None, + title="Category", + page_type="CategoryPage", + clean_text="Category listing " * 40, + ) + + bundle = HybridExtractor("generic", mode="hybrid").extract_from_context(context, project_config()) + + assert bundle.extractor_name == "hybrid_rule_only_routed" + assert bundle.raw_output["llm_skipped"] is True + assert bundle.raw_output["effective_extraction_mode"] == "rule_only" + assert "CategoryPage" in bundle.raw_output["llm_skip_reason"] diff --git a/ontology_platform/web/frontend/src/lib/api/claims.ts b/ontology_platform/web/frontend/src/lib/api/claims.ts index 0f0ff88..1875f2a 100644 --- a/ontology_platform/web/frontend/src/lib/api/claims.ts +++ b/ontology_platform/web/frontend/src/lib/api/claims.ts @@ -28,6 +28,11 @@ export const claimSchema = z graph_merge_status: z.string().nullable().optional(), graph_merge_reason: z.string().nullable().optional(), confidence_breakdown: z.record(z.string(), z.unknown()).nullable().optional(), + agreement: z.string().nullable().optional(), + extraction_source: z.string().nullable().optional(), + claim_kind: z.string().nullable().optional(), + rule_confidence: z.number().nullable().optional(), + llm_confidence: z.number().nullable().optional(), review_required: z.boolean().nullable().optional(), review_reason: z.string().nullable().optional(), conflict_status: z.string().nullable().optional(), diff --git a/ontology_platform/web/frontend/src/lib/api/crawl.ts b/ontology_platform/web/frontend/src/lib/api/crawl.ts index 03da5b7..00c9ca0 100644 --- a/ontology_platform/web/frontend/src/lib/api/crawl.ts +++ b/ontology_platform/web/frontend/src/lib/api/crawl.ts @@ -9,10 +9,27 @@ export const crawlPageItemSchema = z status: z.string().optional(), page_type: z.string().optional(), title: z.string().nullable().optional(), + extraction_mode: z.string().nullable().optional(), + effective_extraction_mode: z.string().nullable().optional(), + llm_skipped: z.boolean().optional(), + llm_skip_reason: z.string().nullable().optional(), + fallback_used: z.boolean().optional(), + agreement_claim_count: z.number().optional(), + conflict_claim_count: z.number().optional(), error: z.string().nullable().optional(), }) .passthrough(); +export const crawlExtractionSummarySchema = z + .object({ + llm_skipped_count: z.number().default(0), + fallback_count: z.number().default(0), + conflict_claim_count: z.number().default(0), + agreement_claim_count: z.number().default(0), + llm_call_count: z.number().default(0), + }) + .passthrough(); + export const crawlProgressSchema = z .object({ seed_url: z.string().optional(), @@ -23,6 +40,7 @@ export const crawlProgressSchema = z errors: z.array(z.string()).optional(), pages: z.array(crawlPageItemSchema).optional(), latest_page: crawlPageItemSchema.optional(), + extraction_summary: crawlExtractionSummarySchema.optional(), }) .passthrough(); @@ -49,9 +67,11 @@ export interface StartSiteCrawlRequest { max_pages?: number; same_domain_only?: boolean; analyze_page_types?: string[]; + extraction_mode?: "rule_only" | "llm_only" | "hybrid" | "compare"; extractor_provider?: string; extractor_model?: string | null; extractor_base_url?: string | null; + fallback_to_rules?: boolean; check_robots_txt?: boolean; respect_robots_txt?: boolean | null; } diff --git a/ontology_platform/web/frontend/src/lib/api/platform.ts b/ontology_platform/web/frontend/src/lib/api/platform.ts index 3821b00..54227e3 100644 --- a/ontology_platform/web/frontend/src/lib/api/platform.ts +++ b/ontology_platform/web/frontend/src/lib/api/platform.ts @@ -65,6 +65,20 @@ export const extractionLogSchema = z validation: z.unknown().optional(), page_context: z.unknown().optional(), candidate_count: z.number().default(0), + extraction_mode: z.string().nullable().optional(), + effective_extraction_mode: z.string().nullable().optional(), + comparison: recordSchema.nullable().optional(), + rule_entity_count: z.number().nullable().optional(), + rule_claim_count: z.number().nullable().optional(), + llm_entity_count: z.number().nullable().optional(), + llm_claim_count: z.number().nullable().optional(), + agreement_claim_count: z.number().nullable().optional(), + rule_only_claim_count: z.number().nullable().optional(), + llm_only_claim_count: z.number().nullable().optional(), + conflict_claim_count: z.number().nullable().optional(), + llm_skipped: z.boolean().nullable().optional(), + llm_skip_reason: z.string().nullable().optional(), + fallback: z.string().nullable().optional(), raw_output: z.unknown().optional(), }) .passthrough(); diff --git a/ontology_platform/web/frontend/src/lib/api/research.ts b/ontology_platform/web/frontend/src/lib/api/research.ts index 913ef1d..b688232 100644 --- a/ontology_platform/web/frontend/src/lib/api/research.ts +++ b/ontology_platform/web/frontend/src/lib/api/research.ts @@ -62,9 +62,11 @@ export interface StartResearchRequest { min_relevance?: number; same_domain_only?: boolean; analyze_page_types?: string[]; + extraction_mode?: "rule_only" | "llm_only" | "hybrid" | "compare"; extractor_provider?: string; extractor_model?: string | null; extractor_base_url?: string | null; + fallback_to_rules?: boolean; check_robots_txt?: boolean; respect_robots_txt?: boolean | null; } diff --git a/ontology_platform/web/frontend/src/pages/CrawlPage.tsx b/ontology_platform/web/frontend/src/pages/CrawlPage.tsx index 1355ae2..e7b60fe 100644 --- a/ontology_platform/web/frontend/src/pages/CrawlPage.tsx +++ b/ontology_platform/web/frontend/src/pages/CrawlPage.tsx @@ -44,6 +44,11 @@ const startCrawlSchema = z.object({ max_depth: z.number().int().min(0).max(10), max_pages: z.number().int().min(1).max(500), same_domain_only: z.boolean(), + extraction_mode: z.enum(["rule_only", "llm_only", "hybrid", "compare"]), + extractor_provider: z.enum(["lm_studio", "openai", "ollama"]), + extractor_model: z.string().optional(), + extractor_base_url: z.string().optional(), + fallback_to_rules: z.boolean(), }); type StartCrawlFormValues = z.infer; @@ -96,14 +101,22 @@ export default function CrawlPage() { max_depth: 2, max_pages: 30, same_domain_only: true, + extraction_mode: "hybrid", + extractor_provider: "lm_studio", + extractor_model: "", + extractor_base_url: "http://localhost:1234/v1", + fallback_to_rules: true, }, }); const onStart = async (values: StartCrawlFormValues) => { try { + const usesLlm = values.extraction_mode !== "rule_only"; const created = await startCrawl.mutateAsync({ project_name: projectName, ...values, + extractor_model: usesLlm ? values.extractor_model || null : null, + extractor_base_url: usesLlm ? values.extractor_base_url || null : null, }); setActiveJobId(created.job_id); toast.success( @@ -136,6 +149,8 @@ export default function CrawlPage() { const sources = project?.sources ?? []; const sourceName = watch("source_name"); + const extractionMode = watch("extraction_mode"); + const usesLlm = extractionMode !== "rule_only"; const selectedSource = sources.find((s) => s.name === sourceName); const progress = job?.progress; const visited = progress?.visited_count ?? 0; @@ -340,6 +355,60 @@ export default function CrawlPage() { +
+
+
+ + +
+
+ + +
+
+ {usesLlm && ( + <> +
+ + +
+
+ + +
+ + + )} +
+ + {!collapsed && ( + +