AI/tests/test_generic_ontology_core.py

from sqlalchemy.orm import Session

from crawler_platform.app.config.loader import ProjectConfig, SourceConfig, load_project_config
from crawler_platform.app.core.database import models
from crawler_platform.app.core.database.repository import KnowledgeRepository
from crawler_platform.app.core.database.session import make_engine
from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle, ExtractionPageContext
from crawler_platform.app.core.extractor.validation import attach_page_context
from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
from crawler_platform.app.core.ontology.registry import OntologyRegistry
from crawler_platform.app.core.research.graph_query import SemanticGraphQuery


def make_repo():
    config = load_project_config("configs/perfume_subscription.yaml")
    engine = make_engine("sqlite:///:memory:")
    models.Base.metadata.create_all(engine)
    session = Session(engine)
    repo = KnowledgeRepository(session)
    project = repo.upsert_project(config)
    source = repo.get_source(project.id, "official_brand_site")
    return config, session, repo, project, source


def save_validated_note_claim(repo, project, source, config):
    page = repo.upsert_page(
        project.id,
        source.id,
        "https://example.com/product/cotton",
        "Cotton Example",
        200,
        "Cotton Example\nTop notes: Bergamot",
    )
    bundle = ExtractionBundle(
        entities=[
            ExtractedEntity("Perfume", "Cotton Example", evidence_text="Cotton Example", confidence=0.95),
            ExtractedEntity("Note", "Bergamot", evidence_text="Top notes: Bergamot", confidence=0.95),
        ],
        claims=[
            ExtractedClaim(
                "Cotton Example",
                "Perfume",
                "hasTopNote",
                "Bergamot",
                "Note",
                evidence_text="Top notes: Bergamot",
                confidence=0.96,
                confidence_reason="direct evidence",
            )
        ],
        extractor_name="llm_json_extractor",
        provider="lm_studio",
        raw_output={"extraction_mode": "primary"},
    )
    context = ExtractionPageContext(
        url=page.url,
        final_url=page.url,
        title=page.title,
        page_type="ProductPage",
        clean_text=page.cleaned_text_summary,
        source_zones=[
            {
                "zone_type": "product_description",
                "selector": "main",
                "text": "Top notes: Bergamot",
                "claim_allowed": True,
            }
        ],
    )
    return repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)


def test_project_upsert_seeds_schema_registry():
    _config, session, _repo, project, _source = make_repo()

    registry = OntologyRegistry(session).registry_payload(project.id)

    entity_names = {row["name"] for row in registry["entity_types"]}
    relation_names = {row["name"] for row in registry["relation_types"]}
    assert {"Entity", "Product", "Perfume"}.issubset(entity_names)
    assert {"relatedTo", "hasTopNote", "hasBrand"}.issubset(relation_names)
    session.close()


def test_validated_claim_creates_separate_ontology_triple():
    config, session, repo, project, source = make_repo()

    claims = save_validated_note_claim(repo, project, source, config)

    triples = session.query(models.OntologyTriple).all()
    assert len(claims) == 1
    assert len(triples) == 1
    assert triples[0].predicate == "hasTopNote"
    assert triples[0].status == "merged"
    assert claims[0].metadata_json["ontology_triple_id"] == triples[0].id
    session.close()


def test_schema_proposal_and_knowledge_gap_are_governance_records():
    _config, session, _repo, project, _source = make_repo()
    registry = OntologyRegistry(session)

    proposal = registry.propose_schema_change(
        project.id,
        proposal_type="relation_type",
        name="influencedBy",
        reason="New cross-domain relation observed in evidence.",
        evidence="A research paper influenced a product strategy.",
        confidence=0.73,
    )
    gaps = KnowledgeGapDetector(session).list_open(project.id)

    assert proposal.status == "pending_review"
    assert any(gap["gap_type"] == "schema_governance" and gap["target_name"] == "influencedBy" for gap in gaps)
    session.close()


def test_generic_graph_query_uses_ontology_triples():
    config, session, repo, project, source = make_repo()
    save_validated_note_claim(repo, project, source, config)

    rows = SemanticGraphQuery(session).relation_summary(project.id)

    assert rows[0]["predicate"] == "hasTopNote"
    assert rows[0]["support_count"] == 1
    session.close()


def test_config_driven_academic_relation_without_perfume_adapter():
    config = ProjectConfig(
        project_name="academic_demo",
        domain="academic",
        target_entities=["ResearchPaper", "Person"],
        fields=["title", "author"],
        sources=[SourceConfig(name="research_site", type="paper_index", trust_level=0.9)],
        ontology={
            "entity_types": ["ResearchPaper", "Person"],
            "predicates": ["authoredBy"],
            "relation_types": {
                "authoredBy": {
                    "allowed_subject_types": ["ResearchPaper"],
                    "allowed_object_types": ["Person"],
                    "allowed_page_types": ["ArticlePage"],
                    "allowed_source_zones": ["article_body"],
                    "confidence_rules": {"min_confidence": 0.82},
                }
            },
        },
    )
    engine = make_engine("sqlite:///:memory:")
    models.Base.metadata.create_all(engine)
    session = Session(engine)
    repo = KnowledgeRepository(session)
    project = repo.upsert_project(config)
    source = repo.get_source(project.id, "research_site")
    page = repo.upsert_page(
        project.id,
        source.id,
        "https://example.org/papers/semantic-systems",
        "Semantic Systems",
        200,
        "Semantic Systems was authored by Ada Kim.",
    )
    bundle = ExtractionBundle(
        entities=[
            ExtractedEntity("ResearchPaper", "Semantic Systems", evidence_text="Semantic Systems", confidence=0.95),
            ExtractedEntity("Person", "Ada Kim", evidence_text="authored by Ada Kim", confidence=0.95),
        ],
        claims=[
            ExtractedClaim(
                "Semantic Systems",
                "ResearchPaper",
                "authoredBy",
                "Ada Kim",
                "Person",
                evidence_text="Semantic Systems was authored by Ada Kim.",
                confidence=0.97,
                confidence_reason="directly stated",
            )
        ],
        extractor_name="llm_json_extractor",
        provider="lm_studio",
        raw_output={"extraction_mode": "primary"},
    )
    context = ExtractionPageContext(
        url=page.url,
        final_url=page.url,
        title=page.title,
        page_type="ArticlePage",
        clean_text=page.cleaned_text_summary,
        source_zones=[
            {
                "zone_type": "article_body",
                "selector": "article",
                "text": "Semantic Systems was authored by Ada Kim.",
                "claim_allowed": True,
            }
        ],
    )

    claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)

    triples = session.query(models.OntologyTriple).all()
    assert len(claims) == 1
    assert triples[0].predicate == "authoredBy"
    assert triples[0].status == "merged"
    assert session.query(models.OntologyRelationType).filter_by(name="authoredBy").one().domain == "academic"
    session.close()