from sqlalchemy.orm import Session from crawler_platform.app.config.loader import ProjectConfig, SourceConfig, load_project_config from crawler_platform.app.core.database import models from crawler_platform.app.core.database.repository import KnowledgeRepository from crawler_platform.app.core.database.session import make_engine from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle, ExtractionPageContext from crawler_platform.app.core.extractor.validation import attach_page_context from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector from crawler_platform.app.core.ontology.registry import OntologyRegistry from crawler_platform.app.core.research.graph_query import SemanticGraphQuery def make_repo(): config = load_project_config("configs/perfume_subscription.yaml") engine = make_engine("sqlite:///:memory:") models.Base.metadata.create_all(engine) session = Session(engine) repo = KnowledgeRepository(session) project = repo.upsert_project(config) source = repo.get_source(project.id, "official_brand_site") return config, session, repo, project, source def save_validated_note_claim(repo, project, source, config): page = repo.upsert_page( project.id, source.id, "https://example.com/product/cotton", "Cotton Example", 200, "Cotton Example\nTop notes: Bergamot", ) bundle = ExtractionBundle( entities=[ ExtractedEntity("Perfume", "Cotton Example", evidence_text="Cotton Example", confidence=0.95), ExtractedEntity("Note", "Bergamot", evidence_text="Top notes: Bergamot", confidence=0.95), ], claims=[ ExtractedClaim( "Cotton Example", "Perfume", "hasTopNote", "Bergamot", "Note", evidence_text="Top notes: Bergamot", confidence=0.96, confidence_reason="direct evidence", ) ], extractor_name="llm_json_extractor", provider="lm_studio", raw_output={"extraction_mode": "primary"}, ) context = ExtractionPageContext( url=page.url, final_url=page.url, title=page.title, page_type="ProductPage", clean_text=page.cleaned_text_summary, source_zones=[ { "zone_type": "product_description", "selector": "main", "text": "Top notes: Bergamot", "claim_allowed": True, } ], ) return repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config) def test_project_upsert_seeds_schema_registry(): _config, session, _repo, project, _source = make_repo() registry = OntologyRegistry(session).registry_payload(project.id) entity_names = {row["name"] for row in registry["entity_types"]} relation_names = {row["name"] for row in registry["relation_types"]} assert {"Entity", "Product", "Perfume"}.issubset(entity_names) assert {"relatedTo", "hasTopNote", "hasBrand"}.issubset(relation_names) session.close() def test_validated_claim_creates_separate_ontology_triple(): config, session, repo, project, source = make_repo() claims = save_validated_note_claim(repo, project, source, config) triples = session.query(models.OntologyTriple).all() assert len(claims) == 1 assert len(triples) == 1 assert triples[0].predicate == "hasTopNote" assert triples[0].status == "merged" assert claims[0].metadata_json["ontology_triple_id"] == triples[0].id session.close() def test_schema_proposal_and_knowledge_gap_are_governance_records(): _config, session, _repo, project, _source = make_repo() registry = OntologyRegistry(session) proposal = registry.propose_schema_change( project.id, proposal_type="relation_type", name="influencedBy", reason="New cross-domain relation observed in evidence.", evidence="A research paper influenced a product strategy.", confidence=0.73, ) gaps = KnowledgeGapDetector(session).list_open(project.id) assert proposal.status == "pending_review" assert any(gap["gap_type"] == "schema_governance" and gap["target_name"] == "influencedBy" for gap in gaps) session.close() def test_generic_graph_query_uses_ontology_triples(): config, session, repo, project, source = make_repo() save_validated_note_claim(repo, project, source, config) rows = SemanticGraphQuery(session).relation_summary(project.id) assert rows[0]["predicate"] == "hasTopNote" assert rows[0]["support_count"] == 1 session.close() def test_config_driven_academic_relation_without_perfume_adapter(): config = ProjectConfig( project_name="academic_demo", domain="academic", target_entities=["ResearchPaper", "Person"], fields=["title", "author"], sources=[SourceConfig(name="research_site", type="paper_index", trust_level=0.9)], ontology={ "entity_types": ["ResearchPaper", "Person"], "predicates": ["authoredBy"], "relation_types": { "authoredBy": { "allowed_subject_types": ["ResearchPaper"], "allowed_object_types": ["Person"], "allowed_page_types": ["ArticlePage"], "allowed_source_zones": ["article_body"], "confidence_rules": {"min_confidence": 0.82}, } }, }, ) engine = make_engine("sqlite:///:memory:") models.Base.metadata.create_all(engine) session = Session(engine) repo = KnowledgeRepository(session) project = repo.upsert_project(config) source = repo.get_source(project.id, "research_site") page = repo.upsert_page( project.id, source.id, "https://example.org/papers/semantic-systems", "Semantic Systems", 200, "Semantic Systems was authored by Ada Kim.", ) bundle = ExtractionBundle( entities=[ ExtractedEntity("ResearchPaper", "Semantic Systems", evidence_text="Semantic Systems", confidence=0.95), ExtractedEntity("Person", "Ada Kim", evidence_text="authored by Ada Kim", confidence=0.95), ], claims=[ ExtractedClaim( "Semantic Systems", "ResearchPaper", "authoredBy", "Ada Kim", "Person", evidence_text="Semantic Systems was authored by Ada Kim.", confidence=0.97, confidence_reason="directly stated", ) ], extractor_name="llm_json_extractor", provider="lm_studio", raw_output={"extraction_mode": "primary"}, ) context = ExtractionPageContext( url=page.url, final_url=page.url, title=page.title, page_type="ArticlePage", clean_text=page.cleaned_text_summary, source_zones=[ { "zone_type": "article_body", "selector": "article", "text": "Semantic Systems was authored by Ada Kim.", "claim_allowed": True, } ], ) claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config) triples = session.query(models.OntologyTriple).all() assert len(claims) == 1 assert triples[0].predicate == "authoredBy" assert triples[0].status == "merged" assert session.query(models.OntologyRelationType).filter_by(name="authoredBy").one().domain == "academic" session.close()