From 4158789869cc9b6a878a2a61981ed9b9d461e635 Mon Sep 17 00:00:00 2001 From: "LASTA_DEV01\\lasta" Date: Fri, 8 May 2026 17:41:15 +0900 Subject: [PATCH] [crawler] --- .gitignore | 7 + README.md | 286 +++++++++++ configs/perfume_subscription.yaml | 95 ++++ crawler_platform.db-journal | Bin 0 -> 8720 bytes crawler_platform/__init__.py | 2 + crawler_platform/app/__init__.py | 2 + crawler_platform/app/api/__init__.py | 2 + crawler_platform/app/api/routes.py | 317 ++++++++++++ crawler_platform/app/cli/__init__.py | 2 + crawler_platform/app/cli/main.py | 140 ++++++ crawler_platform/app/config/__init__.py | 2 + crawler_platform/app/config/loader.py | 154 ++++++ crawler_platform/app/core/__init__.py | 2 + crawler_platform/app/core/crawler/__init__.py | 2 + .../app/core/crawler/discovery.py | 52 ++ crawler_platform/app/core/crawler/fetchers.py | 136 ++++++ .../app/core/crawler/html_cleaner.py | 25 + crawler_platform/app/core/crawler/pipeline.py | 55 +++ crawler_platform/app/core/crawler/plugins.py | 48 ++ .../app/core/database/__init__.py | 2 + crawler_platform/app/core/database/models.py | 240 ++++++++++ .../app/core/database/repository.py | 333 +++++++++++++ crawler_platform/app/core/database/session.py | 39 ++ .../app/core/extractor/__init__.py | 2 + .../app/core/extractor/ai_provider.py | 388 +++++++++++++++ crawler_platform/app/core/extractor/base.py | 101 ++++ .../app/core/extractor/factory.py | 19 + .../app/core/extractor/rule_based.py | 85 ++++ .../app/core/ontology/__init__.py | 2 + .../app/core/ontology/definitions.py | 130 +++++ crawler_platform/app/core/ontology/mapper.py | 27 ++ .../app/core/recommendation/__init__.py | 2 + .../app/core/recommendation/scorer.py | 100 ++++ .../app/core/scheduler/__init__.py | 2 + .../app/core/scheduler/update_policy.py | 11 + crawler_platform/app/domains/__init__.py | 2 + .../app/domains/candle/__init__.py | 2 + .../app/domains/coffee/__init__.py | 2 + crawler_platform/app/domains/gift/__init__.py | 2 + .../app/domains/perfume/__init__.py | 2 + .../app/domains/perfume/extractor.py | 350 ++++++++++++++ .../app/domains/supplement/__init__.py | 2 + crawler_platform/app/domains/tea/__init__.py | 2 + crawler_platform/app/main.py | 27 ++ crawler_platform/app/web/__init__.py | 2 + crawler_platform/app/web/static/app.js | 405 ++++++++++++++++ crawler_platform/app/web/static/index.html | 179 +++++++ crawler_platform/app/web/static/styles.css | 452 ++++++++++++++++++ pyproject.toml | 23 + requirements.txt | 10 + tests/fixtures/sample_perfume.html | 17 + tests/test_config_loader.py | 12 + tests/test_perfume_extractor.py | 31 ++ tests/test_smoke_unittest.py | 21 + 54 files changed, 4355 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 configs/perfume_subscription.yaml create mode 100644 crawler_platform.db-journal create mode 100644 crawler_platform/__init__.py create mode 100644 crawler_platform/app/__init__.py create mode 100644 crawler_platform/app/api/__init__.py create mode 100644 crawler_platform/app/api/routes.py create mode 100644 crawler_platform/app/cli/__init__.py create mode 100644 crawler_platform/app/cli/main.py create mode 100644 crawler_platform/app/config/__init__.py create mode 100644 crawler_platform/app/config/loader.py create mode 100644 crawler_platform/app/core/__init__.py create mode 100644 crawler_platform/app/core/crawler/__init__.py create mode 100644 crawler_platform/app/core/crawler/discovery.py create mode 100644 crawler_platform/app/core/crawler/fetchers.py create mode 100644 crawler_platform/app/core/crawler/html_cleaner.py create mode 100644 crawler_platform/app/core/crawler/pipeline.py create mode 100644 crawler_platform/app/core/crawler/plugins.py create mode 100644 crawler_platform/app/core/database/__init__.py create mode 100644 crawler_platform/app/core/database/models.py create mode 100644 crawler_platform/app/core/database/repository.py create mode 100644 crawler_platform/app/core/database/session.py create mode 100644 crawler_platform/app/core/extractor/__init__.py create mode 100644 crawler_platform/app/core/extractor/ai_provider.py create mode 100644 crawler_platform/app/core/extractor/base.py create mode 100644 crawler_platform/app/core/extractor/factory.py create mode 100644 crawler_platform/app/core/extractor/rule_based.py create mode 100644 crawler_platform/app/core/ontology/__init__.py create mode 100644 crawler_platform/app/core/ontology/definitions.py create mode 100644 crawler_platform/app/core/ontology/mapper.py create mode 100644 crawler_platform/app/core/recommendation/__init__.py create mode 100644 crawler_platform/app/core/recommendation/scorer.py create mode 100644 crawler_platform/app/core/scheduler/__init__.py create mode 100644 crawler_platform/app/core/scheduler/update_policy.py create mode 100644 crawler_platform/app/domains/__init__.py create mode 100644 crawler_platform/app/domains/candle/__init__.py create mode 100644 crawler_platform/app/domains/coffee/__init__.py create mode 100644 crawler_platform/app/domains/gift/__init__.py create mode 100644 crawler_platform/app/domains/perfume/__init__.py create mode 100644 crawler_platform/app/domains/perfume/extractor.py create mode 100644 crawler_platform/app/domains/supplement/__init__.py create mode 100644 crawler_platform/app/domains/tea/__init__.py create mode 100644 crawler_platform/app/main.py create mode 100644 crawler_platform/app/web/__init__.py create mode 100644 crawler_platform/app/web/static/app.js create mode 100644 crawler_platform/app/web/static/index.html create mode 100644 crawler_platform/app/web/static/styles.css create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 tests/fixtures/sample_perfume.html create mode 100644 tests/test_config_loader.py create mode 100644 tests/test_perfume_extractor.py create mode 100644 tests/test_smoke_unittest.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e0fe3e4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +crawler_platform.db +*.sqlite +*.sqlite3 +uvicorn.*.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..6229077 --- /dev/null +++ b/README.md @@ -0,0 +1,286 @@ +# Ontology Crawler Platform + +향수 구독 플랫폼을 첫 사용 사례로 삼되, 차, 커피, 캔들, 디퓨저, 영양제, 선물, 패션 소품 같은 개인화 구독 추천 서비스에 재사용할 수 있는 범용 크롤러/온톨로지 기반 지식 DB MVP입니다. + +## 핵심 아이디어 + +이 시스템은 웹에서 가져온 문장을 곧바로 사실로 저장하지 않습니다. 모든 정보는 `Claim`으로 저장됩니다. + +```yaml +subject: Product A +predicate: hasTopNote +object: Bergamot +source: OfficialSite +evidence_text: Top notes: Bergamot, Neroli +confidence: 0.95 +``` + +추천 시스템은 원문 복제가 아니라, 출처, 근거, 신뢰도, 갱신일을 가진 온톨로지 매핑 지식을 사용합니다. + +## 아키텍처 + +```mermaid +flowchart LR + A["URL Discovery"] --> B["Page Fetch"] + B --> C["HTML Clean"] + C --> D["Site Parser Plugin"] + D --> E["Extractor Provider"] + E --> F["Ontology Mapping"] + F --> G["Deduplication"] + G --> H["Confidence Scoring"] + H --> I["Knowledge DB"] + I --> J["Recommendation API"] + I --> K["Update Scheduling"] +``` + +구성 단위: + +- `Project`: 향수 구독, 차 구독, 선물 추천 같은 프로젝트 단위 설정 +- `Source`: 공식몰, 마켓플레이스, 리뷰 사이트 같은 데이터 출처 +- `Page`: 수집된 URL과 정제 텍스트 요약 +- `Entity`: 상품, 브랜드, 노트, 무드, 계절, 상황 등 의미 객체 +- `Claim`: 출처가 주장한 정보 단위 +- `Evidence`: Claim의 근거 문장 +- `Relation`: Entity 간 집계 관계 +- `ExtractionLog`: 추출 방식, Provider, 로그 + +## 폴더 구조 + +```text +crawler_platform/ + app/ + main.py + config/ + core/ + crawler/ + extractor/ + ontology/ + database/ + recommendation/ + scheduler/ + domains/ + perfume/ + tea/ + coffee/ + candle/ + supplement/ + gift/ + api/ + cli/ +configs/ + perfume_subscription.yaml +tests/ +README.md +``` + +## DB 스키마 + +초기 MVP는 SQLAlchemy ORM으로 SQLite와 PostgreSQL을 모두 지원합니다. + +필수 테이블: + +- `projects`: 프로젝트 이름, 도메인, JSON 설정 +- `sources`: 출처 타입, 신뢰도, robots 정책, rate limit +- `pages`: URL, fetch 상태, content hash, 정제 텍스트 요약 +- `entities`: 범용 의미 객체 +- `attributes`: Entity 속성 +- `relations`: Entity 간 집계 관계 +- `claims`: 출처가 주장한 subject-predicate-object 정보 +- `evidence`: Claim 근거 텍스트 +- `extraction_logs`: 추출 Provider와 로그 +- `crawl_jobs`: 예약 수집 작업 +- `user_profiles`: 추천 사용자 +- `user_preferences`: 취향 구조 +- `feedback_logs`: 추천 피드백 + +## 설치 + +```bash +pip install -r requirements.txt +playwright install chromium +``` + +정적 페이지는 `requests + BeautifulSoup`로 처리합니다. 동적 페이지가 필요하면 config에서 `fetcher: playwright`로 바꾸면 됩니다. + +## CLI 사용 + +DB 초기화: + +```bash +python -m crawler_platform.app.cli.main --db sqlite:///crawler_platform.db init-db +``` + +향수 프로젝트 생성: + +```bash +python -m crawler_platform.app.cli.main --db sqlite:///crawler_platform.db create-project --config configs/perfume_subscription.yaml +``` + +온톨로지 조회: + +```bash +python -m crawler_platform.app.cli.main ontology --domain perfume +``` + +단일 URL 수집: + +```bash +python -m crawler_platform.app.cli.main --db sqlite:///crawler_platform.db crawl-url \ + --config configs/perfume_subscription.yaml \ + --source official_brand_site \ + --url https://example.com/perfume/product-page +``` + +네트워크 없이 로컬 샘플 HTML로 파이프라인을 확인할 수도 있습니다. + +```bash +python -m crawler_platform.app.cli.main --db sqlite:///crawler_platform.db crawl-url \ + --config configs/perfume_subscription.yaml \ + --source official_brand_site \ + --url tests/fixtures/sample_perfume.html +``` + +분석기 Provider를 바꿀 수도 있습니다. 기본값은 규칙 기반이며, AI Provider는 모델/API 키 또는 로컬 서버 설정이 필요합니다. + +```bash +python -m crawler_platform.app.cli.main --db sqlite:///crawler_platform.db crawl-url \ + --config configs/perfume_subscription.yaml \ + --source official_brand_site \ + --url tests/fixtures/sample_perfume.html \ + --extractor-provider ollama \ + --extractor-model llama3.1 +``` + +지원 Provider: + +- `rule_based`: 정규식/키워드 기반 기본 분석기 +- `openai`: OpenAI 호환 Chat Completions API, `OPENAI_API_KEY`와 모델 필요 +- `ollama`: 로컬 Ollama, 기본 URL `http://localhost:11434/api/chat` +- `lm_studio`: LM Studio OpenAI 호환 서버, 기본 URL `http://localhost:1234/v1/chat/completions` + +LM Studio 사용 순서: + +1. LM Studio에서 `Developer` 또는 Local Server 화면을 엽니다. +2. OpenAI Compatible Server를 켭니다. +3. 서버 주소가 보통 `http://localhost:1234/v1`인지 확인합니다. +4. 웹 UI의 Analyzer에서 `LM Studio`를 선택합니다. +5. Base URL은 비워두거나 `http://localhost:1234/v1`을 넣습니다. +6. `연결 테스트`로 모델 목록을 불러옵니다. +7. 모델이 자동 입력되면 `수집 실행`을 누릅니다. + +Claim 확인: + +```bash +python -m crawler_platform.app.cli.main --db sqlite:///crawler_platform.db claims --project perfume_subscription +``` + +추천 예시: + +```bash +python -m crawler_platform.app.cli.main --db sqlite:///crawler_platform.db recommend \ + --project perfume_subscription \ + --target-type Perfume \ + --preferences-json "{\"preferred_notes\":[\"Bergamot\",\"Musk\"],\"preferred_moods\":[\"Fresh\"],\"season_context\":\"Summer\"}" +``` + +## FastAPI 실행 + +```bash +uvicorn crawler_platform.app.main:app --reload +``` + +브라우저에서 관리자 UI를 열 수 있습니다. + +```text +http://127.0.0.1:8000/ +``` + +관리자 UI에서 가능한 작업: + +- 프로젝트 config 경로로 프로젝트 생성 +- Source 선택 후 URL 또는 로컬 HTML 샘플 수집 +- 분석기 Provider 선택: Rule-based, OpenAI, Ollama, LM Studio +- 도메인 온톨로지, Entity, Claim 조회 +- Claim 신뢰도와 사유 수동 수정 +- Entity ID 기준 병합 +- 추천용 태그 확인 +- 사용자 취향 입력 후 추천 결과 테스트 + +주요 엔드포인트: + +- `GET /health` +- `GET /` +- `GET /projects` +- `POST /projects` +- `GET /ontology/{domain}` +- `POST /crawl` +- `GET /projects/{project_name}/entities` +- `GET /projects/{project_name}/claims` +- `PATCH /claims/{claim_id}/confidence` +- `POST /entities/merge` +- `GET /projects/{project_name}/recommendation-tags` +- `POST /recommend` + +## 향수 도메인 MVP + +기본 엔티티: + +- `Perfume` +- `Brand` +- `Note` +- `Accord` +- `Mood` +- `Season` +- `Occasion` +- `Review` +- `Price` +- `ProductPage` + +기본 관계: + +- `hasBrand` +- `hasTopNote` +- `hasMiddleNote` +- `hasBaseNote` +- `hasAccord` +- `evokesMood` +- `suitableForSeason` +- `suitableForOccasion` +- `similarTo` +- `soldBy` +- `hasPrice` +- `hasReviewKeyword` + +규칙 기반 추출기는 `Top notes`, `Middle notes`, `Base notes`, 가격, 무드, 계절, 사용 상황, 리뷰 키워드를 우선 추출합니다. + +## 확장 방법 + +새 도메인을 추가할 때는 다음을 추가하면 됩니다. + +1. `configs/{project}.yaml`에 `domain`, `target_entities`, `fields`, `sources`, `ontology` 정의 +2. `crawler_platform/app/core/ontology/definitions.py`에 도메인 온톨로지 추가 +3. 필요하면 `crawler_platform/app/domains/{domain}/extractor.py`에 도메인별 Extractor 구현 +4. 사이트별 HTML 구조가 특수하면 `SiteParser`를 구현하고 `ParserRegistry`에 등록 +5. OpenAI, Ollama, LM Studio 등 AI 추출은 `AIExtractor`를 상속하는 Provider로 추가 + +## 컴플라이언스 설계 + +- `robots.txt` 확인 구조 포함 +- Source별 rate limit과 User-Agent 적용 +- retry, timeout 고려 +- 원문 전체 저장 대신 `evidence_text`와 정제 요약 중심 저장 +- 상품 설명은 복제 저장보다 Claim, 태그, 요약, 근거 중심으로 사용 + +## 테스트 + +```bash +pytest +``` + +외부 테스트 러너가 없을 때는 기본 `unittest` 스모크 테스트를 실행할 수 있습니다. + +```bash +python -m unittest tests.test_smoke_unittest -v +``` + +현재 테스트는 향수 규칙 기반 추출, config 로더, 로컬 HTML fetch 경로를 검증합니다. diff --git a/configs/perfume_subscription.yaml b/configs/perfume_subscription.yaml new file mode 100644 index 0000000..eea1a6b --- /dev/null +++ b/configs/perfume_subscription.yaml @@ -0,0 +1,95 @@ +project_name: perfume_subscription +domain: perfume +target_entities: + - Perfume + - Brand + - Note + - Accord + - Mood + - Season + - Occasion +fields: + - name + - brand + - top_notes + - middle_notes + - base_notes + - accords + - mood_tags + - season_tags + - occasion_tags + - price + - review_keywords +sources: + - name: official_brand_site + type: official + trust_level: 0.95 + parser: generic + fetcher: requests + rate_limit_per_minute: 20 + respect_robots_txt: true + - name: marketplace + type: marketplace + trust_level: 0.8 + parser: generic + fetcher: requests + rate_limit_per_minute: 15 + respect_robots_txt: true + - name: review_site + type: review + trust_level: 0.7 + parser: generic + fetcher: requests + rate_limit_per_minute: 10 + respect_robots_txt: true +ontology: + entity_types: + - Perfume + - Brand + - Note + - Accord + - Mood + - Season + - Occasion + - Review + - Price + - ProductPage + predicates: + - hasBrand + - hasTopNote + - hasMiddleNote + - hasBaseNote + - hasAccord + - evokesMood + - suitableForSeason + - suitableForOccasion + - similarTo + - soldBy + - hasPrice + - hasReviewKeyword + aliases: + top_notes: hasTopNote + middle_notes: hasMiddleNote + heart_notes: hasMiddleNote + base_notes: hasBaseNote + accords: hasAccord + mood_tags: evokesMood + season_tags: suitableForSeason + occasion_tags: suitableForOccasion + review_keywords: hasReviewKeyword +recommendation: + target_entity_type: Perfume + weights: + preferred_note: 2.0 + avoided_note: -3.0 + preferred_mood: 1.5 + season_context: 1.2 + occasion_context: 1.0 +update_policy: + official: + interval_days: 30 + marketplace: + interval_days: 7 + review: + interval_days: 14 + diff --git a/crawler_platform.db-journal b/crawler_platform.db-journal new file mode 100644 index 0000000000000000000000000000000000000000..8a9558b76cd425bfe290982506a2144ee2762d33 GIT binary patch literal 8720 zcmeHNPm3c(6z`cu+!b{cyex|ldM|s}37uqmlF4PrfO}9@2i==4O{M!KQ%-kRZ&h_> zL&!%Eudd*?5d0K-_TXLc;%D%^s`Q_PthgWxN)-cL^;0b@ID8M$9o+uZ!#~#?1C9a5fMdWh;23ZWye0$R4-Rg7;k}k>VDxUKWPc%1tS2St=c+uz(Z^3%X|^p`*M@NtV1!Q?#%& z6*rJX1=ClC3F4(-Qh}shm_rVjB`r7u9bZ|igl8ovZ9qz^JSx6F;$m3_MeENDtH_{HWk`3T;Ne}4I52>g}4S)DT6AxX44cN3E7B5%UJ6L zCaUn1r%XN(l(iCRaS3Rc%PiY<|`-W^oHBuc0e zrG5UEDp_}%gG!{Vr=cUO6$?E(+_dxyoh?=xs|MWNrkAqPs7>>iUad^`0h5g~#sO=ez`-5?K+C3W#hGFpF zaB11R`>jm y7u0?{di@YR{BoZiV2HuTn-JAaLhI7EGM)cvp?vhJI47f^xAz_(q&W2dAAbQzw&j8V literal 0 HcmV?d00001 diff --git a/crawler_platform/__init__.py b/crawler_platform/__init__.py new file mode 100644 index 0000000..22d5b19 --- /dev/null +++ b/crawler_platform/__init__.py @@ -0,0 +1,2 @@ +"""Reusable ontology-centered crawler platform.""" + diff --git a/crawler_platform/app/__init__.py b/crawler_platform/app/__init__.py new file mode 100644 index 0000000..67ce8ed --- /dev/null +++ b/crawler_platform/app/__init__.py @@ -0,0 +1,2 @@ +"""Application package for the crawler platform.""" + diff --git a/crawler_platform/app/api/__init__.py b/crawler_platform/app/api/__init__.py new file mode 100644 index 0000000..2e46bc1 --- /dev/null +++ b/crawler_platform/app/api/__init__.py @@ -0,0 +1,2 @@ +"""FastAPI routes.""" + diff --git a/crawler_platform/app/api/routes.py b/crawler_platform/app/api/routes.py new file mode 100644 index 0000000..8a1dd7c --- /dev/null +++ b/crawler_platform/app/api/routes.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +from dataclasses import asdict +from typing import Any + +from fastapi import HTTPException +from pydantic import BaseModel, Field +from sqlalchemy import select + +from crawler_platform.app.config.loader import load_project_config +from crawler_platform.app.core.crawler.discovery import discover_links +from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher +from crawler_platform.app.core.crawler.pipeline import CrawlPipeline +from crawler_platform.app.core.database import models +from crawler_platform.app.core.database.repository import KnowledgeRepository +from crawler_platform.app.core.database.session import session_scope +from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models +from crawler_platform.app.core.extractor.factory import extractor_for_domain +from crawler_platform.app.core.ontology.definitions import ontology_for_domain +from crawler_platform.app.core.ontology.mapper import ontology_to_dict +from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender + + +class CrawlRequest(BaseModel): + config_path: str + source_name: str + url: str + extractor_provider: str = "rule_based" + extractor_model: str | None = None + extractor_base_url: str | None = None + + +class DiscoverRequest(BaseModel): + config_path: str + source_name: str + url: str + limit: int = 30 + + +class RecommendRequest(BaseModel): + project_name: str + target_entity_type: str = "Perfume" + preferences: dict = Field(default_factory=dict) + limit: int = 10 + + +class CreateProjectRequest(BaseModel): + config_path: str + + +class UpdateClaimConfidenceRequest(BaseModel): + confidence: float + reason: str | None = None + + +class MergeEntitiesRequest(BaseModel): + project_name: str + source_entity_id: int + target_entity_id: int + + +class ExtractorModelsRequest(BaseModel): + provider: str + base_url: str | None = None + + +def register_routes(app, database_url: str) -> None: + @app.get("/health") + def health(): + return {"ok": True} + + @app.get("/projects") + def projects(): + with session_scope(database_url) as session: + rows = session.scalars(select(models.Project).order_by(models.Project.created_at.desc())).all() + return [ + { + "id": project.id, + "name": project.name, + "domain": project.domain, + "created_at": project.created_at.isoformat(), + "updated_at": project.updated_at.isoformat(), + } + for project in rows + ] + + @app.post("/projects") + def create_project(request: CreateProjectRequest): + config = load_project_config(request.config_path) + with session_scope(database_url) as session: + project = KnowledgeRepository(session).upsert_project(config) + return {"id": project.id, "name": project.name, "domain": project.domain} + + @app.get("/projects/{project_name}") + def project_detail(project_name: str): + with session_scope(database_url) as session: + project = KnowledgeRepository(session).get_project(project_name) + sources = session.scalars(select(models.Source).where(models.Source.project_id == project.id)).all() + return { + "id": project.id, + "name": project.name, + "domain": project.domain, + "config": project.config, + "sources": [ + { + "id": source.id, + "name": source.name, + "type": source.type, + "trust_level": source.trust_level, + "respect_robots_txt": source.respect_robots_txt, + "rate_limit_per_minute": source.rate_limit_per_minute, + } + for source in sources + ], + } + + @app.get("/ontology/{domain}") + def ontology(domain: str): + return ontology_to_dict(ontology_for_domain(domain)) + + @app.post("/extractors/models") + def extractor_models(request: ExtractorModelsRequest): + try: + if request.provider == "lm_studio": + models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1") + return {"ok": True, "models": models} + if request.provider == "openai": + import os + + models = list_openai_compatible_models( + request.base_url or "https://api.openai.com/v1", + os.getenv("OPENAI_API_KEY"), + ) + return {"ok": True, "models": models} + if request.provider == "ollama": + return {"ok": False, "error": "Ollama model listing is not implemented yet. Enter the model manually."} + return {"ok": True, "models": [{"id": "rule_based", "owned_by": "local"}]} + except Exception as exc: + return {"ok": False, "error": str(exc), "models": []} + + @app.post("/crawl") + def crawl(request: CrawlRequest): + config = load_project_config(request.config_path) + with session_scope(database_url) as session: + repo = KnowledgeRepository(session) + pipeline = CrawlPipeline( + repo, + extractor_for_domain( + config.domain, + provider=request.extractor_provider, + model=request.extractor_model, + base_url=request.extractor_base_url, + ), + ) + try: + result = pipeline.crawl_url(config, request.source_name, request.url) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return {"page_id": result.page_id, "claim_count": result.claim_count, "entity_count": result.entity_count} + + @app.post("/discover") + def discover(request: DiscoverRequest): + config = load_project_config(request.config_path) + source_config = config.source_by_name(request.source_name) + robots = RobotsPolicy() + if not robots.allowed(request.url, source_config.respect_robots_txt): + return {"ok": False, "error": "robots.txt does not allow discovery for this URL", "links": []} + fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute) + result = fetcher.fetch(request.url) + links = discover_links(result.html, result.final_url or request.url, request.limit) + return { + "ok": True, + "status_code": result.status_code, + "final_url": result.final_url, + "links": [asdict(link) for link in links], + } + + @app.get("/projects/{project_name}/entities") + def project_entities(project_name: str, entity_type: str | None = None, limit: int = 50): + with session_scope(database_url) as session: + project = KnowledgeRepository(session).get_project(project_name) + query = select(models.Entity).where(models.Entity.project_id == project.id) + if entity_type: + query = query.where(models.Entity.entity_type == entity_type) + entities = session.scalars(query.limit(limit)).all() + return [ + { + "id": entity.id, + "type": entity.entity_type, + "name": entity.name, + "metadata": entity.metadata_json, + } + for entity in entities + ] + + @app.get("/projects/{project_name}/claims") + def project_claims(project_name: str, limit: int = 100): + with session_scope(database_url) as session: + project = KnowledgeRepository(session).get_project(project_name) + rows = session.execute( + select(models.Claim, models.Source, models.Page, models.Entity) + .join(models.Source, models.Claim.source_id == models.Source.id) + .join(models.Page, models.Claim.page_id == models.Page.id, isouter=True) + .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id) + .where(models.Claim.project_id == project.id) + .order_by(models.Claim.last_seen_at.desc()) + .limit(limit) + ).all() + results: list[dict[str, Any]] = [] + for claim, source, page, subject in rows: + object_name = None + if claim.object_entity_id: + object_entity = session.get(models.Entity, claim.object_entity_id) + object_name = object_entity.name if object_entity else None + evidence = session.scalar( + select(models.Evidence) + .where(models.Evidence.claim_id == claim.id) + .order_by(models.Evidence.created_at.desc()) + ) + results.append( + { + "id": claim.id, + "subject": subject.name, + "subject_type": subject.entity_type, + "predicate": claim.predicate, + "object": object_name, + "object_value": claim.object_value, + "source": source.name, + "page_url": page.url if page else None, + "confidence": claim.confidence, + "confidence_reason": claim.confidence_reason, + "evidence_text": evidence.evidence_text if evidence else None, + "last_seen_at": claim.last_seen_at.isoformat(), + } + ) + return results + + @app.patch("/claims/{claim_id}/confidence") + def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest): + confidence = min(max(request.confidence, 0.0), 1.0) + with session_scope(database_url) as session: + claim = session.get(models.Claim, claim_id) + if claim is None: + return {"ok": False, "error": "claim not found"} + claim.confidence = confidence + claim.confidence_reason = request.reason or "manual admin update" + claim.last_seen_at = models.utcnow() + return {"ok": True, "claim_id": claim.id, "confidence": claim.confidence} + + @app.post("/entities/merge") + def merge_entities(request: MergeEntitiesRequest): + with session_scope(database_url) as session: + project = KnowledgeRepository(session).get_project(request.project_name) + source = session.get(models.Entity, request.source_entity_id) + target = session.get(models.Entity, request.target_entity_id) + if source is None or target is None or source.project_id != project.id or target.project_id != project.id: + return {"ok": False, "error": "entity not found in project"} + session.query(models.Claim).filter(models.Claim.subject_entity_id == source.id).update( + {models.Claim.subject_entity_id: target.id} + ) + session.query(models.Claim).filter(models.Claim.object_entity_id == source.id).update( + {models.Claim.object_entity_id: target.id} + ) + session.query(models.Relation).filter(models.Relation.subject_entity_id == source.id).update( + {models.Relation.subject_entity_id: target.id} + ) + session.query(models.Relation).filter(models.Relation.object_entity_id == source.id).update( + {models.Relation.object_entity_id: target.id} + ) + source.metadata_json = {**(source.metadata_json or {}), "merged_into": target.id} + source.updated_at = models.utcnow() + return {"ok": True, "source_entity_id": source.id, "target_entity_id": target.id} + + @app.get("/projects/{project_name}/recommendation-tags") + def recommendation_tags(project_name: str): + with session_scope(database_url) as session: + project = KnowledgeRepository(session).get_project(project_name) + tag_predicates = { + "hasTopNote", + "hasMiddleNote", + "hasBaseNote", + "hasScentNote", + "hasFlavorNote", + "evokesMood", + "suitableForSeason", + "suitableForOccasion", + "hasReviewKeyword", + } + rows = session.execute( + select(models.Claim, models.Entity) + .join(models.Entity, models.Claim.object_entity_id == models.Entity.id) + .where(models.Claim.project_id == project.id, models.Claim.predicate.in_(tag_predicates)) + ).all() + grouped: dict[str, dict[str, Any]] = {} + for claim, entity in rows: + key = f"{claim.predicate}:{entity.canonical_name}" + if key not in grouped: + grouped[key] = { + "predicate": claim.predicate, + "name": entity.name, + "type": entity.entity_type, + "support_count": 0, + "max_confidence": 0.0, + } + grouped[key]["support_count"] += 1 + grouped[key]["max_confidence"] = max(grouped[key]["max_confidence"], claim.confidence) + return sorted(grouped.values(), key=lambda item: (item["predicate"], -item["support_count"], item["name"])) + + @app.post("/recommend") + def recommend(request: RecommendRequest): + with session_scope(database_url) as session: + repo = KnowledgeRepository(session) + project = repo.get_project(request.project_name) + recommender = RuleBasedRecommender(session) + pref = PreferenceInput(**request.preferences) + items = recommender.recommend(project.id, request.target_entity_type, pref, request.limit) + return [asdict(item) for item in items] diff --git a/crawler_platform/app/cli/__init__.py b/crawler_platform/app/cli/__init__.py new file mode 100644 index 0000000..8cc6512 --- /dev/null +++ b/crawler_platform/app/cli/__init__.py @@ -0,0 +1,2 @@ +"""Command line admin tools.""" + diff --git a/crawler_platform/app/cli/main.py b/crawler_platform/app/cli/main.py new file mode 100644 index 0000000..2d00ba0 --- /dev/null +++ b/crawler_platform/app/cli/main.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import argparse +from dataclasses import asdict +import json + +from sqlalchemy import select + +from crawler_platform.app.config.loader import load_project_config +from crawler_platform.app.core.crawler.pipeline import CrawlPipeline +from crawler_platform.app.core.database import models +from crawler_platform.app.core.database.repository import KnowledgeRepository +from crawler_platform.app.core.database.session import init_db, session_scope +from crawler_platform.app.core.extractor.factory import extractor_for_domain +from crawler_platform.app.core.ontology.definitions import ontology_for_domain +from crawler_platform.app.core.ontology.mapper import ontology_to_dict +from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Ontology crawler platform admin CLI") + parser.add_argument("--db", default="sqlite:///crawler_platform.db", help="SQLAlchemy database URL") + sub = parser.add_subparsers(dest="command", required=True) + + sub.add_parser("init-db") + + create_project = sub.add_parser("create-project") + create_project.add_argument("--config", required=True) + + ontology = sub.add_parser("ontology") + ontology.add_argument("--domain", required=True) + + crawl = sub.add_parser("crawl-url") + crawl.add_argument("--config", required=True) + crawl.add_argument("--source", required=True) + crawl.add_argument("--url", required=True) + crawl.add_argument("--extractor-provider", default="rule_based", choices=["rule_based", "openai", "ollama", "lm_studio"]) + crawl.add_argument("--extractor-model") + crawl.add_argument("--extractor-base-url") + + claims = sub.add_parser("claims") + claims.add_argument("--project", required=True) + claims.add_argument("--limit", type=int, default=20) + + entities = sub.add_parser("entities") + entities.add_argument("--project", required=True) + entities.add_argument("--type") + entities.add_argument("--limit", type=int, default=20) + + recommend = sub.add_parser("recommend") + recommend.add_argument("--project", required=True) + recommend.add_argument("--target-type", default="Perfume") + recommend.add_argument("--preferences-json", default="{}") + recommend.add_argument("--limit", type=int, default=10) + return parser + + +def main() -> None: + args = build_parser().parse_args() + if args.command == "init-db": + init_db(args.db) + print(f"initialized database: {args.db}") + return + if args.command == "ontology": + print(json.dumps(ontology_to_dict(ontology_for_domain(args.domain)), ensure_ascii=False, indent=2)) + return + if args.command == "create-project": + config = load_project_config(args.config) + with session_scope(args.db) as session: + project = KnowledgeRepository(session).upsert_project(config) + print(json.dumps({"project_id": project.id, "name": project.name, "domain": project.domain}, ensure_ascii=False)) + return + if args.command == "crawl-url": + config = load_project_config(args.config) + with session_scope(args.db) as session: + repo = KnowledgeRepository(session) + pipeline = CrawlPipeline( + repo, + extractor_for_domain( + config.domain, + provider=args.extractor_provider, + model=args.extractor_model, + base_url=args.extractor_base_url, + ), + ) + result = pipeline.crawl_url(config, args.source, args.url) + print(json.dumps(asdict(result), ensure_ascii=False)) + return + if args.command == "claims": + with session_scope(args.db) as session: + project = KnowledgeRepository(session).get_project(args.project) + rows = session.execute( + select(models.Claim, models.Entity) + .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id) + .where(models.Claim.project_id == project.id) + .limit(args.limit) + ).all() + print( + json.dumps( + [ + { + "claim_id": claim.id, + "subject": subject.name, + "predicate": claim.predicate, + "object_entity_id": claim.object_entity_id, + "object_value": claim.object_value, + "confidence": claim.confidence, + } + for claim, subject in rows + ], + ensure_ascii=False, + indent=2, + ) + ) + return + if args.command == "entities": + with session_scope(args.db) as session: + project = KnowledgeRepository(session).get_project(args.project) + query = select(models.Entity).where(models.Entity.project_id == project.id) + if args.type: + query = query.where(models.Entity.entity_type == args.type) + result = session.scalars(query.limit(args.limit)).all() + print( + json.dumps( + [{"id": entity.id, "type": entity.entity_type, "name": entity.name} for entity in result], + ensure_ascii=False, + indent=2, + ) + ) + return + if args.command == "recommend": + preference = PreferenceInput(**json.loads(args.preferences_json)) + with session_scope(args.db) as session: + project = KnowledgeRepository(session).get_project(args.project) + result = RuleBasedRecommender(session).recommend(project.id, args.target_type, preference, args.limit) + print(json.dumps([asdict(item) for item in result], ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/crawler_platform/app/config/__init__.py b/crawler_platform/app/config/__init__.py new file mode 100644 index 0000000..285ab4c --- /dev/null +++ b/crawler_platform/app/config/__init__.py @@ -0,0 +1,2 @@ +"""Project configuration loading.""" + diff --git a/crawler_platform/app/config/loader.py b/crawler_platform/app/config/loader.py new file mode 100644 index 0000000..958d3f4 --- /dev/null +++ b/crawler_platform/app/config/loader.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +import json +from pathlib import Path +from typing import Any + + +@dataclass(slots=True) +class SourceConfig: + name: str + type: str = "unknown" + trust_level: float = 0.5 + base_url: str | None = None + allowed_paths: list[str] = field(default_factory=list) + parser: str = "generic" + fetcher: str = "requests" + rate_limit_per_minute: int = 30 + respect_robots_txt: bool = True + + +@dataclass(slots=True) +class ProjectConfig: + project_name: str + domain: str + target_entities: list[str] + fields: list[str] + sources: list[SourceConfig] + ontology: dict[str, Any] = field(default_factory=dict) + recommendation: dict[str, Any] = field(default_factory=dict) + update_policy: dict[str, Any] = field(default_factory=dict) + + def source_by_name(self, name: str) -> SourceConfig: + for source in self.sources: + if source.name == name: + return source + raise KeyError(f"Unknown source in project config: {name}") + + +def load_project_config(path: str | Path) -> ProjectConfig: + config_path = Path(path) + data = _load_mapping(config_path) + sources = [SourceConfig(**item) for item in data.get("sources", [])] + return ProjectConfig( + project_name=data["project_name"], + domain=data["domain"], + target_entities=list(data.get("target_entities", [])), + fields=list(data.get("fields", [])), + sources=sources, + ontology=dict(data.get("ontology", {})), + recommendation=dict(data.get("recommendation", {})), + update_policy=dict(data.get("update_policy", {})), + ) + + +def _load_mapping(path: Path) -> dict[str, Any]: + text = path.read_text(encoding="utf-8") + if path.suffix.lower() == ".json": + return json.loads(text) + try: + import yaml + except ImportError as exc: + return _load_simple_yaml(text) + return yaml.safe_load(text) + + +def _load_simple_yaml(text: str) -> dict[str, Any]: + """Small YAML fallback for project configs when PyYAML is unavailable. + + It intentionally supports only the subset used by the sample project files: + nested mappings, lists, booleans, ints, floats, and strings. + """ + + lines: list[tuple[int, str]] = [] + for raw_line in text.splitlines(): + if not raw_line.strip() or raw_line.lstrip().startswith("#"): + continue + indent = len(raw_line) - len(raw_line.lstrip(" ")) + lines.append((indent, raw_line.strip())) + value, index = _parse_yaml_block(lines, 0, 0) + if index != len(lines): + raise RuntimeError("Could not parse full YAML config. Install PyYAML for full YAML support.") + if not isinstance(value, dict): + raise RuntimeError("Project config root must be a mapping.") + return value + + +def _parse_yaml_block(lines: list[tuple[int, str]], index: int, indent: int) -> tuple[Any, int]: + if index >= len(lines): + return {}, index + current_indent, content = lines[index] + if current_indent < indent: + return {}, index + if content.startswith("- "): + result: list[Any] = [] + while index < len(lines): + item_indent, item_content = lines[index] + if item_indent != indent or not item_content.startswith("- "): + break + item_raw = item_content[2:].strip() + index += 1 + if not item_raw: + child, index = _parse_yaml_block(lines, index, indent + 2) + result.append(child) + elif ":" in item_raw: + key, raw_value = _split_key_value(item_raw) + item: dict[str, Any] = {key: _parse_scalar(raw_value)} if raw_value else {key: None} + if index < len(lines) and lines[index][0] > indent: + child, index = _parse_yaml_block(lines, index, lines[index][0]) + if isinstance(child, dict): + item.update(child) + result.append(item) + else: + result.append(_parse_scalar(item_raw)) + return result, index + + result: dict[str, Any] = {} + while index < len(lines): + line_indent, line_content = lines[index] + if line_indent != indent or line_content.startswith("- "): + break + key, raw_value = _split_key_value(line_content) + index += 1 + if raw_value: + result[key] = _parse_scalar(raw_value) + elif index < len(lines) and lines[index][0] > indent: + child, index = _parse_yaml_block(lines, index, lines[index][0]) + result[key] = child + else: + result[key] = None + return result, index + + +def _split_key_value(content: str) -> tuple[str, str]: + key, _, raw_value = content.partition(":") + return key.strip(), raw_value.strip() + + +def _parse_scalar(value: str) -> Any: + if value == "": + return None + lowered = value.lower() + if lowered == "true": + return True + if lowered == "false": + return False + if lowered in {"null", "none"}: + return None + try: + if "." in value: + return float(value) + return int(value) + except ValueError: + return value.strip("\"'") diff --git a/crawler_platform/app/core/__init__.py b/crawler_platform/app/core/__init__.py new file mode 100644 index 0000000..c6c6b87 --- /dev/null +++ b/crawler_platform/app/core/__init__.py @@ -0,0 +1,2 @@ +"""Core platform modules.""" + diff --git a/crawler_platform/app/core/crawler/__init__.py b/crawler_platform/app/core/crawler/__init__.py new file mode 100644 index 0000000..52e68f2 --- /dev/null +++ b/crawler_platform/app/core/crawler/__init__.py @@ -0,0 +1,2 @@ +"""Crawler pipeline.""" + diff --git a/crawler_platform/app/core/crawler/discovery.py b/crawler_platform/app/core/crawler/discovery.py new file mode 100644 index 0000000..3129b9c --- /dev/null +++ b/crawler_platform/app/core/crawler/discovery.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dataclasses import dataclass +from urllib.parse import parse_qs, urljoin, urlparse + +from bs4 import BeautifulSoup + + +@dataclass(slots=True) +class DiscoveredUrl: + url: str + label: str + kind: str = "link" + + +def discover_links(html: str, base_url: str, limit: int = 30) -> list[DiscoveredUrl]: + soup = BeautifulSoup(html, "html.parser") + seen: set[str] = set() + results: list[DiscoveredUrl] = [] + for anchor in soup.find_all("a", href=True): + raw_href = anchor.get("href", "") + url = normalize_search_redirect(urljoin(base_url, raw_href)) + if not url or url in seen or not url.startswith(("http://", "https://")): + continue + seen.add(url) + label = anchor.get_text(" ", strip=True)[:160] or urlparse(url).netloc + results.append(DiscoveredUrl(url=url, label=label, kind=classify_url(url))) + if len(results) >= limit: + break + return results + + +def normalize_search_redirect(url: str) -> str: + parsed = urlparse(url) + query = parse_qs(parsed.query) + for key in ("url", "u", "target"): + if key in query and query[key]: + candidate = query[key][0] + if candidate.startswith(("http://", "https://")): + return candidate + return url + + +def classify_url(url: str) -> str: + host = urlparse(url).netloc.lower() + if "smartstore.naver.com" in host or "brand.naver.com" in host: + return "marketplace_product_or_store" + if "shopping.naver.com" in host: + return "shopping" + if any(token in host for token in ("fragrantica", "official", "perfume", "parfum")): + return "product_or_review" + return "link" diff --git a/crawler_platform/app/core/crawler/fetchers.py b/crawler_platform/app/core/crawler/fetchers.py new file mode 100644 index 0000000..955c99c --- /dev/null +++ b/crawler_platform/app/core/crawler/fetchers.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +import time +from urllib.parse import urlparse +from urllib.robotparser import RobotFileParser + + +DEFAULT_USER_AGENT = "OntologyCrawlerBot/0.1 (+contact: admin@example.com)" + + +@dataclass(slots=True) +class FetchResult: + url: str + status_code: int | None + html: str + final_url: str | None = None + headers: dict[str, str] = field(default_factory=dict) + + +class RateLimiter: + def __init__(self, per_minute: int = 30): + self.delay = 60 / max(per_minute, 1) + self._last_called = 0.0 + + def wait(self) -> None: + elapsed = time.monotonic() - self._last_called + if elapsed < self.delay: + time.sleep(self.delay - elapsed) + self._last_called = time.monotonic() + + +class RobotsPolicy: + def __init__(self, user_agent: str = DEFAULT_USER_AGENT): + self.user_agent = user_agent + self._cache: dict[str, RobotFileParser] = {} + + def allowed(self, url: str, respect_robots_txt: bool = True) -> bool: + if not respect_robots_txt: + return True + parsed = urlparse(url) + if parsed.scheme in {"", "file"}: + return True + robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt" + parser = self._cache.get(robots_url) + if parser is None: + parser = RobotFileParser() + parser.set_url(robots_url) + try: + parser.read() + except Exception: + return False + self._cache[robots_url] = parser + return parser.can_fetch(self.user_agent, url) + + +class BaseFetcher: + def fetch(self, url: str) -> FetchResult: + raise NotImplementedError + + +class RequestsFetcher(BaseFetcher): + def __init__( + self, + user_agent: str = DEFAULT_USER_AGENT, + timeout_seconds: int = 15, + retries: int = 2, + rate_limit_per_minute: int = 30, + ): + self.user_agent = user_agent + self.timeout_seconds = timeout_seconds + self.retries = retries + self.rate_limiter = RateLimiter(rate_limit_per_minute) + + def fetch(self, url: str) -> FetchResult: + local_path = _local_path_from_url(url) + if local_path: + return FetchResult(url=url, status_code=200, html=local_path.read_text(encoding="utf-8"), final_url=url) + + import requests + + last_error: Exception | None = None + for _ in range(self.retries + 1): + self.rate_limiter.wait() + try: + response = requests.get( + url, + headers={"User-Agent": self.user_agent}, + timeout=self.timeout_seconds, + ) + return FetchResult( + url=url, + status_code=response.status_code, + html=response.text, + final_url=response.url, + headers=dict(response.headers), + ) + except requests.RequestException as exc: + last_error = exc + raise RuntimeError(f"Failed to fetch {url}: {last_error}") + + +class PlaywrightFetcher(BaseFetcher): + def __init__(self, user_agent: str = DEFAULT_USER_AGENT, timeout_ms: int = 20000): + self.user_agent = user_agent + self.timeout_ms = timeout_ms + + def fetch(self, url: str) -> FetchResult: + from playwright.sync_api import sync_playwright + + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=True) + page = browser.new_page(user_agent=self.user_agent) + response = page.goto(url, wait_until="networkidle", timeout=self.timeout_ms) + html = page.content() + final_url = page.url + status = response.status if response else None + browser.close() + return FetchResult(url=url, status_code=status, html=html, final_url=final_url) + + +def make_fetcher(kind: str, rate_limit_per_minute: int = 30) -> BaseFetcher: + if kind == "playwright": + return PlaywrightFetcher() + return RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute) + + +def _local_path_from_url(url: str) -> Path | None: + if url.startswith("file://"): + path = Path(url.removeprefix("file://")) + return path if path.exists() and path.is_file() else None + path = Path(url) + if path.exists() and path.is_file(): + return path + return None diff --git a/crawler_platform/app/core/crawler/html_cleaner.py b/crawler_platform/app/core/crawler/html_cleaner.py new file mode 100644 index 0000000..1ffe43f --- /dev/null +++ b/crawler_platform/app/core/crawler/html_cleaner.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import re + + +def clean_html(html: str) -> tuple[str | None, str]: + try: + from bs4 import BeautifulSoup + except ImportError: + text = re.sub(r"<[^>]+>", " ", html) + return None, normalize_whitespace(text) + + soup = BeautifulSoup(html, "html.parser") + for tag in soup(["script", "style", "noscript", "svg", "iframe"]): + tag.decompose() + title = soup.title.get_text(" ", strip=True) if soup.title else None + main = soup.find("main") or soup.body or soup + text = main.get_text("\n", strip=True) + return title, normalize_whitespace(text) + + +def normalize_whitespace(text: str) -> str: + lines = [" ".join(line.split()) for line in text.splitlines()] + return "\n".join(line for line in lines if line) + diff --git a/crawler_platform/app/core/crawler/pipeline.py b/crawler_platform/app/core/crawler/pipeline.py new file mode 100644 index 0000000..fc040ea --- /dev/null +++ b/crawler_platform/app/core/crawler/pipeline.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from crawler_platform.app.config.loader import ProjectConfig +from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher +from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry +from crawler_platform.app.core.database.repository import KnowledgeRepository +from crawler_platform.app.core.extractor.base import Extractor + + +@dataclass(slots=True) +class CrawlResult: + page_id: int + claim_count: int + entity_count: int + + +class CrawlPipeline: + def __init__( + self, + repository: KnowledgeRepository, + extractor: Extractor, + parser_registry: ParserRegistry | None = None, + robots_policy: RobotsPolicy | None = None, + ): + self.repository = repository + self.extractor = extractor + self.parser_registry = parser_registry or default_parser_registry() + self.robots_policy = robots_policy or RobotsPolicy() + + def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult: + project = self.repository.upsert_project(project_config) + source_config = project_config.source_by_name(source_name) + source = self.repository.get_source(project.id, source_name) + if not self.robots_policy.allowed(url, source_config.respect_robots_txt): + raise PermissionError(f"robots.txt does not allow crawling: {url}") + + fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute) + fetch_result = fetcher.fetch(url) + parser = self.parser_registry.get(source_config.parser) + parsed = parser.parse(fetch_result.html, fetch_result.final_url or url) + page = self.repository.upsert_page( + project_id=project.id, + source_id=source.id, + url=url, + title=parsed.title, + status_code=fetch_result.status_code, + cleaned_text=parsed.text, + metadata={**parsed.metadata, "final_url": fetch_result.final_url}, + ) + bundle = self.extractor.extract(parsed.text, project_config) + claims = self.repository.save_extraction_bundle(project.id, source, page, bundle) + return CrawlResult(page_id=page.id, claim_count=len(claims), entity_count=len(bundle.entities)) + diff --git a/crawler_platform/app/core/crawler/plugins.py b/crawler_platform/app/core/crawler/plugins.py new file mode 100644 index 0000000..b8504e5 --- /dev/null +++ b/crawler_platform/app/core/crawler/plugins.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + + +@dataclass(slots=True) +class ParsedPage: + title: str | None + text: str + metadata: dict[str, object] + + +class SiteParser(Protocol): + name: str + + def parse(self, html: str, url: str) -> ParsedPage: + raise NotImplementedError + + +class ParserRegistry: + def __init__(self): + self._parsers: dict[str, SiteParser] = {} + + def register(self, parser: SiteParser) -> None: + self._parsers[parser.name] = parser + + def get(self, name: str) -> SiteParser: + if name not in self._parsers: + raise KeyError(f"Parser not registered: {name}") + return self._parsers[name] + + +class GenericProductParser: + name = "generic" + + def parse(self, html: str, url: str) -> ParsedPage: + from crawler_platform.app.core.crawler.html_cleaner import clean_html + + title, text = clean_html(html) + return ParsedPage(title=title, text=text, metadata={"parser": self.name, "url": url}) + + +def default_parser_registry() -> ParserRegistry: + registry = ParserRegistry() + registry.register(GenericProductParser()) + return registry + diff --git a/crawler_platform/app/core/database/__init__.py b/crawler_platform/app/core/database/__init__.py new file mode 100644 index 0000000..293959b --- /dev/null +++ b/crawler_platform/app/core/database/__init__.py @@ -0,0 +1,2 @@ +"""Database models and repositories.""" + diff --git a/crawler_platform/app/core/database/models.py b/crawler_platform/app/core/database/models.py new file mode 100644 index 0000000..a9cb84d --- /dev/null +++ b/crawler_platform/app/core/database/models.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from sqlalchemy import ( + JSON, + Boolean, + Column, + DateTime, + Float, + ForeignKey, + Integer, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.orm import declarative_base, relationship + + +Base = declarative_base() + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +class Project(Base): + __tablename__ = "projects" + + id = Column(Integer, primary_key=True) + name = Column(String(160), unique=True, nullable=False, index=True) + domain = Column(String(80), nullable=False, index=True) + config = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + sources = relationship("Source", back_populates="project", cascade="all, delete-orphan") + + +class Source(Base): + __tablename__ = "sources" + __table_args__ = (UniqueConstraint("project_id", "name", name="uq_source_project_name"),) + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + name = Column(String(160), nullable=False) + type = Column(String(80), nullable=False, default="unknown") + base_url = Column(Text) + trust_level = Column(Float, nullable=False, default=0.5) + respect_robots_txt = Column(Boolean, nullable=False, default=True) + rate_limit_per_minute = Column(Integer, nullable=False, default=30) + update_policy = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + project = relationship("Project", back_populates="sources") + + +class Page(Base): + __tablename__ = "pages" + __table_args__ = (UniqueConstraint("project_id", "url", name="uq_page_project_url"),) + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + source_id = Column(Integer, ForeignKey("sources.id"), nullable=False, index=True) + url = Column(Text, nullable=False) + canonical_url = Column(Text) + title = Column(Text) + content_hash = Column(String(80), index=True) + status_code = Column(Integer) + fetched_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + cleaned_text_summary = Column(Text) + raw_storage_ref = Column(Text) + metadata_json = Column(JSON, nullable=False, default=dict) + + +class Entity(Base): + __tablename__ = "entities" + __table_args__ = ( + UniqueConstraint("project_id", "entity_type", "canonical_name", name="uq_entity_identity"), + ) + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + entity_type = Column(String(120), nullable=False, index=True) + name = Column(String(240), nullable=False) + canonical_name = Column(String(240), nullable=False, index=True) + external_ids = Column(JSON, nullable=False, default=dict) + metadata_json = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + +class Attribute(Base): + __tablename__ = "attributes" + __table_args__ = ( + UniqueConstraint("entity_id", "name", "source_id", name="uq_attribute_entity_source"), + ) + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True) + source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True) + name = Column(String(120), nullable=False, index=True) + value = Column(JSON, nullable=False) + confidence = Column(Float, nullable=False, default=0.5) + updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + +class Relation(Base): + __tablename__ = "relations" + __table_args__ = ( + UniqueConstraint("project_id", "subject_entity_id", "predicate", "object_entity_id", name="uq_relation"), + ) + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True) + predicate = Column(String(160), nullable=False, index=True) + object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True) + confidence = Column(Float, nullable=False, default=0.5) + support_count = Column(Integer, nullable=False, default=1) + updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + +class Claim(Base): + __tablename__ = "claims" + __table_args__ = (UniqueConstraint("project_id", "claim_hash", name="uq_claim_hash"),) + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + source_id = Column(Integer, ForeignKey("sources.id"), nullable=False, index=True) + page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True) + subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True) + predicate = Column(String(160), nullable=False, index=True) + object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True) + object_value = Column(JSON, nullable=True) + value_type = Column(String(80), nullable=False, default="entity") + claim_hash = Column(String(80), nullable=False, index=True) + confidence = Column(Float, nullable=False, default=0.5) + confidence_reason = Column(Text) + extraction_method = Column(String(120), nullable=False, default="rule_based") + status = Column(String(40), nullable=False, default="active") + first_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + last_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + valid_until = Column(DateTime(timezone=True), nullable=True) + metadata_json = Column(JSON, nullable=False, default=dict) + + evidence_items = relationship("Evidence", back_populates="claim", cascade="all, delete-orphan") + + +class Evidence(Base): + __tablename__ = "evidence" + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + claim_id = Column(Integer, ForeignKey("claims.id"), nullable=False, index=True) + page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True) + evidence_text = Column(Text, nullable=False) + evidence_summary = Column(Text) + selector = Column(Text) + start_offset = Column(Integer) + end_offset = Column(Integer) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + claim = relationship("Claim", back_populates="evidence_items") + + +class ExtractionLog(Base): + __tablename__ = "extraction_logs" + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True) + extractor_name = Column(String(160), nullable=False) + provider = Column(String(120), nullable=False, default="rule_based") + input_hash = Column(String(80)) + raw_output = Column(JSON, nullable=False, default=dict) + error = Column(Text) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + +class CrawlJob(Base): + __tablename__ = "crawl_jobs" + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True) + url = Column(Text) + status = Column(String(40), nullable=False, default="pending") + priority = Column(Integer, nullable=False, default=100) + scheduled_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + error = Column(Text) + metadata_json = Column(JSON, nullable=False, default=dict) + + +class UserProfile(Base): + __tablename__ = "user_profiles" + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + external_user_id = Column(String(160), nullable=False, index=True) + display_name = Column(String(160)) + context = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + +class UserPreference(Base): + __tablename__ = "user_preferences" + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + user_profile_id = Column(Integer, ForeignKey("user_profiles.id"), nullable=False, index=True) + likes = Column(JSON, nullable=False, default=list) + dislikes = Column(JSON, nullable=False, default=list) + preferred_moods = Column(JSON, nullable=False, default=list) + preferred_notes = Column(JSON, nullable=False, default=list) + avoided_notes = Column(JSON, nullable=False, default=list) + price_preference = Column(JSON, nullable=False, default=dict) + season_context = Column(String(80)) + occasion_context = Column(String(120)) + feedback_history = Column(JSON, nullable=False, default=list) + updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) + + +class FeedbackLog(Base): + __tablename__ = "feedback_logs" + + id = Column(Integer, primary_key=True) + project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True) + user_profile_id = Column(Integer, ForeignKey("user_profiles.id"), nullable=False, index=True) + entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True) + action = Column(String(80), nullable=False) + score = Column(Float) + reason = Column(Text) + metadata_json = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False) diff --git a/crawler_platform/app/core/database/repository.py b/crawler_platform/app/core/database/repository.py new file mode 100644 index 0000000..ceb822a --- /dev/null +++ b/crawler_platform/app/core/database/repository.py @@ -0,0 +1,333 @@ +from __future__ import annotations + +from dataclasses import asdict +import hashlib +import json +from typing import Any + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from crawler_platform.app.config.loader import ProjectConfig, SourceConfig +from crawler_platform.app.core.database import models +from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle + + +def canonicalize(value: str) -> str: + return " ".join(value.strip().lower().split()) + + +def short_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +class KnowledgeRepository: + def __init__(self, session: Session): + self.session = session + + def upsert_project(self, config: ProjectConfig) -> models.Project: + project = self.session.scalar(select(models.Project).where(models.Project.name == config.project_name)) + config_dict = _project_config_to_dict(config) + if project is None: + project = models.Project(name=config.project_name, domain=config.domain, config=config_dict) + self.session.add(project) + self.session.flush() + else: + project.domain = config.domain + project.config = config_dict + project.updated_at = models.utcnow() + for source_config in config.sources: + self.upsert_source(project, source_config) + return project + + def upsert_source(self, project: models.Project, source_config: SourceConfig) -> models.Source: + source = self.session.scalar( + select(models.Source).where( + models.Source.project_id == project.id, + models.Source.name == source_config.name, + ) + ) + if source is None: + source = models.Source(project_id=project.id, name=source_config.name) + self.session.add(source) + self.session.flush() + source.type = source_config.type + source.base_url = source_config.base_url + source.trust_level = source_config.trust_level + source.respect_robots_txt = source_config.respect_robots_txt + source.rate_limit_per_minute = source_config.rate_limit_per_minute + source.updated_at = models.utcnow() + return source + + def get_project(self, project_name: str) -> models.Project: + project = self.session.scalar(select(models.Project).where(models.Project.name == project_name)) + if project is None: + raise KeyError(f"Project not found: {project_name}") + return project + + def get_source(self, project_id: int, source_name: str) -> models.Source: + source = self.session.scalar( + select(models.Source).where(models.Source.project_id == project_id, models.Source.name == source_name) + ) + if source is None: + raise KeyError(f"Source not found: {source_name}") + return source + + def upsert_page( + self, + project_id: int, + source_id: int, + url: str, + title: str | None, + status_code: int | None, + cleaned_text: str, + metadata: dict[str, Any] | None = None, + ) -> models.Page: + page = self.session.scalar(select(models.Page).where(models.Page.project_id == project_id, models.Page.url == url)) + digest = short_hash(cleaned_text) + summary = cleaned_text[:2000] + if page is None: + page = models.Page(project_id=project_id, source_id=source_id, url=url) + self.session.add(page) + self.session.flush() + page.title = title + page.status_code = status_code + page.content_hash = digest + page.cleaned_text_summary = summary + page.metadata_json = metadata or {} + page.fetched_at = models.utcnow() + return page + + def upsert_entity( + self, + project_id: int, + entity_type: str, + name: str, + metadata: dict[str, Any] | None = None, + ) -> models.Entity: + canonical_name = canonicalize(name) + entity = self.session.scalar( + select(models.Entity).where( + models.Entity.project_id == project_id, + models.Entity.entity_type == entity_type, + models.Entity.canonical_name == canonical_name, + ) + ) + if entity is None: + entity = models.Entity( + project_id=project_id, + entity_type=entity_type, + name=name.strip(), + canonical_name=canonical_name, + metadata_json=metadata or {}, + ) + self.session.add(entity) + self.session.flush() + else: + entity.metadata_json = {**(entity.metadata_json or {}), **(metadata or {})} + entity.updated_at = models.utcnow() + return entity + + def save_extraction_bundle( + self, + project_id: int, + source: models.Source, + page: models.Page, + bundle: ExtractionBundle, + ) -> list[models.Claim]: + entity_index: dict[tuple[str, str], models.Entity] = {} + for extracted_entity in bundle.entities: + entity = self._save_extracted_entity(project_id, source.id, extracted_entity) + entity_index[(extracted_entity.entity_type, canonicalize(extracted_entity.name))] = entity + + claims: list[models.Claim] = [] + for extracted_claim in bundle.claims: + subject = self._entity_for_claim(project_id, extracted_claim.subject_type, extracted_claim.subject_name, entity_index) + object_entity = None + if extracted_claim.object_name and extracted_claim.object_type: + object_entity = self._entity_for_claim( + project_id, + extracted_claim.object_type, + extracted_claim.object_name, + entity_index, + ) + confidence = combine_confidence(extracted_claim.confidence, source.trust_level) + claim_hash = make_claim_hash( + project_id=project_id, + source_id=source.id, + subject_entity_id=subject.id, + predicate=extracted_claim.predicate, + object_entity_id=object_entity.id if object_entity else None, + object_value=extracted_claim.object_value, + ) + claim = self.session.scalar( + select(models.Claim).where( + models.Claim.project_id == project_id, + models.Claim.claim_hash == claim_hash, + ) + ) + if claim is None: + claim = models.Claim( + project_id=project_id, + source_id=source.id, + page_id=page.id, + subject_entity_id=subject.id, + predicate=extracted_claim.predicate, + object_entity_id=object_entity.id if object_entity else None, + object_value=extracted_claim.object_value, + value_type="entity" if object_entity else "literal", + claim_hash=claim_hash, + confidence=confidence, + confidence_reason=extracted_claim.confidence_reason, + extraction_method=bundle.extractor_name, + metadata_json=extracted_claim.metadata, + ) + self.session.add(claim) + self.session.flush() + else: + claim.page_id = page.id + claim.last_seen_at = models.utcnow() + claim.confidence = max(claim.confidence, confidence) + claim.confidence_reason = extracted_claim.confidence_reason or claim.confidence_reason + claim.metadata_json = {**(claim.metadata_json or {}), **extracted_claim.metadata} + if extracted_claim.evidence_text: + self.session.add( + models.Evidence( + project_id=project_id, + claim_id=claim.id, + page_id=page.id, + evidence_text=extracted_claim.evidence_text[:1000], + evidence_summary=extracted_claim.evidence_summary, + ) + ) + if object_entity: + self._upsert_relation(project_id, subject.id, extracted_claim.predicate, object_entity.id, confidence) + claims.append(claim) + + self.session.add( + models.ExtractionLog( + project_id=project_id, + page_id=page.id, + extractor_name=bundle.extractor_name, + provider=bundle.provider, + input_hash=page.content_hash, + raw_output=bundle.raw_output, + ) + ) + return claims + + def _save_extracted_entity( + self, + project_id: int, + source_id: int, + extracted_entity: ExtractedEntity, + ) -> models.Entity: + entity = self.upsert_entity( + project_id, + extracted_entity.entity_type, + extracted_entity.name, + extracted_entity.metadata, + ) + for name, value in extracted_entity.attributes.items(): + attribute = self.session.scalar( + select(models.Attribute).where( + models.Attribute.entity_id == entity.id, + models.Attribute.name == name, + models.Attribute.source_id == source_id, + ) + ) + if attribute is None: + attribute = models.Attribute( + project_id=project_id, + entity_id=entity.id, + source_id=source_id, + name=name, + value=value, + confidence=extracted_entity.confidence, + ) + self.session.add(attribute) + else: + attribute.value = value + attribute.confidence = max(attribute.confidence, extracted_entity.confidence) + attribute.updated_at = models.utcnow() + return entity + + def _entity_for_claim( + self, + project_id: int, + entity_type: str, + name: str, + entity_index: dict[tuple[str, str], models.Entity], + ) -> models.Entity: + key = (entity_type, canonicalize(name)) + if key not in entity_index: + entity_index[key] = self.upsert_entity(project_id, entity_type, name) + return entity_index[key] + + def _upsert_relation( + self, + project_id: int, + subject_id: int, + predicate: str, + object_id: int, + confidence: float, + ) -> models.Relation: + relation = self.session.scalar( + select(models.Relation).where( + models.Relation.project_id == project_id, + models.Relation.subject_entity_id == subject_id, + models.Relation.predicate == predicate, + models.Relation.object_entity_id == object_id, + ) + ) + if relation is None: + relation = models.Relation( + project_id=project_id, + subject_entity_id=subject_id, + predicate=predicate, + object_entity_id=object_id, + confidence=confidence, + ) + self.session.add(relation) + self.session.flush() + else: + relation.support_count += 1 + relation.confidence = max(relation.confidence, confidence) + relation.updated_at = models.utcnow() + return relation + + +def _project_config_to_dict(config: ProjectConfig) -> dict[str, Any]: + return { + "project_name": config.project_name, + "domain": config.domain, + "target_entities": config.target_entities, + "fields": config.fields, + "sources": [asdict(source) for source in config.sources], + "ontology": config.ontology, + "recommendation": config.recommendation, + "update_policy": config.update_policy, + } + + +def combine_confidence(extraction_confidence: float, source_trust: float) -> float: + return round(min(max((extraction_confidence * 0.7) + (source_trust * 0.3), 0.0), 1.0), 4) + + +def make_claim_hash( + project_id: int, + source_id: int, + subject_entity_id: int, + predicate: str, + object_entity_id: int | None, + object_value: Any | None, +) -> str: + payload = { + "project_id": project_id, + "source_id": source_id, + "subject_entity_id": subject_entity_id, + "predicate": predicate, + "object_entity_id": object_entity_id, + "object_value": object_value, + } + return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest() diff --git a/crawler_platform/app/core/database/session.py b/crawler_platform/app/core/database/session.py new file mode 100644 index 0000000..40f26cf --- /dev/null +++ b/crawler_platform/app/core/database/session.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from contextlib import contextmanager +from typing import Iterator + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker + +from crawler_platform.app.core.database.models import Base + + +def make_engine(database_url: str = "sqlite:///crawler_platform.db"): + connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {} + return create_engine(database_url, future=True, connect_args=connect_args) + + +def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None: + engine = make_engine(database_url) + Base.metadata.create_all(engine) + + +def make_session_factory(database_url: str = "sqlite:///crawler_platform.db") -> sessionmaker[Session]: + engine = make_engine(database_url) + return sessionmaker(bind=engine, expire_on_commit=False, class_=Session, future=True) + + +@contextmanager +def session_scope(database_url: str = "sqlite:///crawler_platform.db") -> Iterator[Session]: + factory = make_session_factory(database_url) + session = factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + diff --git a/crawler_platform/app/core/extractor/__init__.py b/crawler_platform/app/core/extractor/__init__.py new file mode 100644 index 0000000..491c6b0 --- /dev/null +++ b/crawler_platform/app/core/extractor/__init__.py @@ -0,0 +1,2 @@ +"""Extractor provider interfaces and implementations.""" + diff --git a/crawler_platform/app/core/extractor/ai_provider.py b/crawler_platform/app/core/extractor/ai_provider.py new file mode 100644 index 0000000..24d497d --- /dev/null +++ b/crawler_platform/app/core/extractor/ai_provider.py @@ -0,0 +1,388 @@ +from __future__ import annotations + +import json +import os +import re +from typing import Any + +import requests + +from crawler_platform.app.config.loader import ProjectConfig +from crawler_platform.app.core.extractor.base import ( + AIExtractor, + ExtractedClaim, + ExtractedEntity, + ExtractionBundle, +) +from crawler_platform.app.core.ontology.mapper import normalize_predicate +from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor + + +class LLMJsonExtractor(AIExtractor): + name = "llm_json_extractor" + + def __init__( + self, + domain: str, + provider: str, + model: str | None = None, + base_url: str | None = None, + timeout_seconds: int = 300, + ): + self.domain = domain + self.provider = provider + self.model = model + self.base_url = base_url + self.timeout_seconds = timeout_seconds + + def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle: + try: + raw = self.complete_json(page_text, project_config) + except Exception as exc: + return self._fallback_bundle(page_text, project_config, str(exc)) + bundle = ExtractionBundle( + entities=parse_entities(raw.get("entities", [])), + claims=parse_claims(raw.get("claims", [])), + extractor_name=self.name, + provider=self.provider, + raw_output={ + "provider": self.provider, + "model": self.model, + "entity_count": len(raw.get("entities", [])), + "claim_count": len(raw.get("claims", [])), + }, + ) + if not bundle.entities or not bundle.claims: + return self._fallback_bundle(page_text, project_config, "AI returned no usable entities or claims") + return self.normalize_to_ontology(bundle, project_config.ontology) + + def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]: + return self.extract(page_text, project_config).entities + + def extract_attributes( + self, + entity: ExtractedEntity, + page_text: str, + project_config: ProjectConfig, + ) -> dict[str, Any]: + return entity.attributes + + def extract_relations( + self, + entities: list[ExtractedEntity], + page_text: str, + project_config: ProjectConfig, + ) -> list[ExtractedClaim]: + return [] + + def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, Any]) -> ExtractionBundle: + for claim in bundle.claims: + claim.predicate = normalize_predicate(claim.predicate, ontology) + return bundle + + def complete_json(self, page_text: str, project_config: ProjectConfig) -> dict[str, Any]: + prompt = build_extraction_prompt(page_text, project_config) + if self.provider == "openai": + return self._complete_openai_compatible(prompt, "OPENAI_API_KEY", "OPENAI_MODEL", self.base_url) + if self.provider == "lm_studio": + return self._complete_openai_compatible( + prompt, + "LM_STUDIO_API_KEY", + "LM_STUDIO_MODEL", + normalize_openai_chat_url(self.base_url or "http://localhost:1234/v1"), + api_key_optional=True, + ) + if self.provider == "ollama": + return self._complete_ollama(prompt) + raise ValueError(f"Unsupported AI extractor provider: {self.provider}") + + def _fallback_bundle(self, page_text: str, project_config: ProjectConfig, error: str) -> ExtractionBundle: + if project_config.domain == "perfume": + bundle = PerfumeRuleBasedExtractor().extract(page_text, project_config) + else: + from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor + + bundle = GenericRuleBasedExtractor().extract(page_text, project_config) + bundle.extractor_name = f"{self.name}_with_rule_fallback" + bundle.provider = f"{self.provider}_fallback" + bundle.raw_output = { + **bundle.raw_output, + "ai_provider": self.provider, + "ai_model": self.model, + "ai_error": error, + "fallback": "rule_based", + } + for entity in bundle.entities: + entity.metadata["ai_fallback_reason"] = error + for claim in bundle.claims: + claim.metadata["ai_fallback_reason"] = error + claim.confidence_reason = f"{claim.confidence_reason}; AI fallback: {error}" if claim.confidence_reason else error + return bundle + + def _complete_openai_compatible( + self, + prompt: str, + api_key_env: str, + model_env: str, + endpoint: str | None, + api_key_optional: bool = False, + ) -> dict[str, Any]: + api_key = os.getenv(api_key_env) + model = self.model or os.getenv(model_env) + if not model and api_key_optional and endpoint: + model = first_openai_compatible_model(endpoint, api_key) + if not model: + raise RuntimeError(f"AI model is required. Set UI model field or {model_env}.") + if not api_key and not api_key_optional: + raise RuntimeError(f"API key is required. Set {api_key_env}.") + url = endpoint or "https://api.openai.com/v1/chat/completions" + headers = {"Content-Type": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + response = requests.post( + url, + headers=headers, + json={ + "model": model, + "messages": [ + {"role": "system", "content": "Extract ontology knowledge as strict JSON only."}, + {"role": "user", "content": prompt}, + ], + "temperature": 0, + }, + timeout=self.timeout_seconds, + ) + if response.status_code >= 400: + raise RuntimeError(f"LLM request failed {response.status_code}: {response.text[:1000]}") + data = response.json() + content = data["choices"][0]["message"]["content"] + return parse_json_content(content, retry=lambda bad: self._repair_json_with_model(bad, endpoint, headers, model)) + + def _complete_ollama(self, prompt: str) -> dict[str, Any]: + model = self.model or os.getenv("OLLAMA_MODEL") + if not model: + raise RuntimeError("Ollama model is required. Set UI model field or OLLAMA_MODEL.") + url = self.base_url or "http://localhost:11434/api/chat" + response = requests.post( + url, + json={ + "model": model, + "messages": [ + {"role": "system", "content": "Extract ontology knowledge as strict JSON only."}, + {"role": "user", "content": prompt}, + ], + "stream": False, + "format": "json", + }, + timeout=self.timeout_seconds, + ) + response.raise_for_status() + content = response.json()["message"]["content"] + return parse_json_content(content) + + def _repair_json_with_model( + self, + bad_content: str, + endpoint: str, + headers: dict[str, str], + model: str, + ) -> dict[str, Any]: + response = requests.post( + endpoint, + headers=headers, + json={ + "model": model, + "messages": [ + { + "role": "system", + "content": "You repair malformed JSON. Return valid JSON only. No markdown.", + }, + { + "role": "user", + "content": ( + "Repair this into valid JSON with top-level keys entities and claims. " + "Drop invalid fragments if needed.\n\n" + f"{bad_content[:12000]}" + ), + }, + ], + "temperature": 0, + }, + timeout=self.timeout_seconds, + ) + if response.status_code >= 400: + raise RuntimeError(f"LLM JSON repair failed {response.status_code}: {response.text[:1000]}") + return parse_json_content(response.json()["choices"][0]["message"]["content"]) + + +def build_extraction_prompt(page_text: str, project_config: ProjectConfig) -> str: + clipped_text = page_text[:6000] + ontology = project_config.ontology or {} + return f""" +Project domain: {project_config.domain} +Target entity types: {project_config.target_entities} +Fields: {project_config.fields} +Allowed predicates: {ontology.get("predicates", [])} + +Return only minified strict JSON. Do not include markdown, analysis, or prose. +Use this shape: +{{ + "entities": [ + {{ + "entity_type": "Perfume", + "name": "Product name", + "attributes": {{"name": "Product name"}}, + "confidence": 0.0, + "evidence_text": "short evidence from page" + }} + ], + "claims": [ + {{ + "subject_name": "Product name", + "subject_type": "Perfume", + "predicate": "hasTopNote", + "object_name": "Bergamot", + "object_type": "Note", + "object_value": null, + "evidence_text": "short evidence from page", + "evidence_summary": "why this claim was extracted", + "confidence": 0.0, + "confidence_reason": "reason" + }} + ] +}} + +Rules: +- Store information as source claims, not absolute facts. +- Keep evidence_text short. Do not copy long descriptions. +- Use only ontology predicates when possible. +- If object is a simple value like price, put it in object_value and leave object_name/object_type null. +- If unsure, lower confidence instead of inventing. +- Extract at most 20 entities and 30 claims. +- For perfume, prioritize name, brand, top/middle/base notes, accords, mood, season, occasion, price, review keywords. + +Page text: +{clipped_text} +""".strip() + + +def parse_json_content(content: str, retry=None) -> dict[str, Any]: + try: + return json.loads(content) + except json.JSONDecodeError: + match = re.search(r"\{.*\}", content, flags=re.DOTALL) + if not match: + if retry: + return retry(content) + raise + try: + return json.loads(match.group(0)) + except json.JSONDecodeError: + if retry: + return retry(content) + repaired = heuristic_repair_json(content) + if repaired is not None: + return repaired + raise + + +def heuristic_repair_json(content: str) -> dict[str, Any] | None: + """Best-effort extraction for chatty local models that emit broken JSON.""" + + entities = [] + claims = [] + for block in re.findall(r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)?\}", content, flags=re.DOTALL): + try: + item = json.loads(block) + except json.JSONDecodeError: + continue + if {"entity_type", "name"}.issubset(item): + entities.append(item) + if {"subject_name", "subject_type", "predicate"}.issubset(item): + claims.append(item) + if entities or claims: + return {"entities": entities, "claims": claims} + return None + + +def normalize_openai_chat_url(base_url: str) -> str: + clean = base_url.rstrip("/") + if clean.endswith("/chat/completions"): + return clean + if clean.endswith("/v1"): + return f"{clean}/chat/completions" + return f"{clean}/v1/chat/completions" + + +def normalize_openai_models_url(base_url: str) -> str: + clean = base_url.rstrip("/") + if clean.endswith("/chat/completions"): + return clean.removesuffix("/chat/completions") + "/models" + if clean.endswith("/models"): + return clean + if clean.endswith("/v1"): + return f"{clean}/models" + return f"{clean}/v1/models" + + +def first_openai_compatible_model(base_url: str, api_key: str | None = None) -> str | None: + models = list_openai_compatible_models(base_url, api_key) + return models[0]["id"] if models else None + + +def list_openai_compatible_models(base_url: str, api_key: str | None = None) -> list[dict[str, Any]]: + url = normalize_openai_models_url(base_url) + headers = {} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + response = requests.get(url, headers=headers, timeout=10) + response.raise_for_status() + data = response.json() + items = data.get("data", []) + return [{"id": item.get("id", ""), "owned_by": item.get("owned_by")} for item in items if item.get("id")] + + +def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]: + entities: list[ExtractedEntity] = [] + for item in items: + name = item.get("name") + entity_type = item.get("entity_type") or item.get("type") + if not name or not entity_type: + continue + entities.append( + ExtractedEntity( + entity_type=str(entity_type), + name=str(name), + attributes=dict(item.get("attributes") or {}), + evidence_text=item.get("evidence_text"), + confidence=float(item.get("confidence") or 0.55), + metadata={"ai_extracted": True}, + ) + ) + return entities + + +def parse_claims(items: list[dict[str, Any]]) -> list[ExtractedClaim]: + claims: list[ExtractedClaim] = [] + for item in items: + subject_name = item.get("subject_name") + subject_type = item.get("subject_type") + predicate = item.get("predicate") + if not subject_name or not subject_type or not predicate: + continue + claims.append( + ExtractedClaim( + subject_name=str(subject_name), + subject_type=str(subject_type), + predicate=str(predicate), + object_name=item.get("object_name"), + object_type=item.get("object_type"), + object_value=item.get("object_value"), + evidence_text=item.get("evidence_text"), + evidence_summary=item.get("evidence_summary"), + confidence=float(item.get("confidence") or 0.55), + confidence_reason=item.get("confidence_reason") or "AI extractor output", + metadata={"ai_extracted": True}, + ) + ) + return claims diff --git a/crawler_platform/app/core/extractor/base.py b/crawler_platform/app/core/extractor/base.py new file mode 100644 index 0000000..7b68854 --- /dev/null +++ b/crawler_platform/app/core/extractor/base.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any + +from crawler_platform.app.config.loader import ProjectConfig + + +@dataclass(slots=True) +class ExtractedEntity: + entity_type: str + name: str + attributes: dict[str, Any] = field(default_factory=dict) + evidence_text: str | None = None + confidence: float = 0.5 + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True) +class ExtractedClaim: + subject_name: str + subject_type: str + predicate: str + object_name: str | None = None + object_type: str | None = None + object_value: Any | None = None + evidence_text: str | None = None + evidence_summary: str | None = None + confidence: float = 0.5 + confidence_reason: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True) +class ExtractionBundle: + entities: list[ExtractedEntity] = field(default_factory=list) + claims: list[ExtractedClaim] = field(default_factory=list) + extractor_name: str = "unknown" + provider: str = "unknown" + raw_output: dict[str, Any] = field(default_factory=dict) + + +class Extractor(ABC): + name = "base" + provider = "base" + + @abstractmethod + def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]: + raise NotImplementedError + + @abstractmethod + def extract_attributes( + self, + entity: ExtractedEntity, + page_text: str, + project_config: ProjectConfig, + ) -> dict[str, Any]: + raise NotImplementedError + + @abstractmethod + def extract_relations( + self, + entities: list[ExtractedEntity], + page_text: str, + project_config: ProjectConfig, + ) -> list[ExtractedClaim]: + raise NotImplementedError + + @abstractmethod + def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, Any]) -> ExtractionBundle: + raise NotImplementedError + + def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle: + entities = self.extract_entities(page_text, project_config) + for entity in entities: + entity.attributes.update(self.extract_attributes(entity, page_text, project_config)) + claims = self.extract_relations(entities, page_text, project_config) + bundle = ExtractionBundle( + entities=entities, + claims=claims, + extractor_name=self.name, + provider=self.provider, + raw_output={"entity_count": len(entities), "claim_count": len(claims)}, + ) + return self.normalize_to_ontology(bundle, project_config.ontology) + + +class AIExtractor(Extractor): + """Provider-neutral AI extractor contract. + + OpenAI, local LLM, Ollama, and LM Studio adapters can subclass this and + implement ``complete_json`` while keeping the rest of the pipeline stable. + """ + + provider = "ai" + + @abstractmethod + def complete_json(self, page_text: str, project_config: ProjectConfig) -> dict[str, Any]: + raise NotImplementedError + diff --git a/crawler_platform/app/core/extractor/factory.py b/crawler_platform/app/core/extractor/factory.py new file mode 100644 index 0000000..e50b62d --- /dev/null +++ b/crawler_platform/app/core/extractor/factory.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor +from crawler_platform.app.core.extractor.base import Extractor +from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor +from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor + + +def extractor_for_domain( + domain: str, + provider: str = "rule_based", + model: str | None = None, + base_url: str | None = None, +) -> Extractor: + if provider in {"openai", "ollama", "lm_studio"}: + return LLMJsonExtractor(domain=domain, provider=provider, model=model, base_url=base_url) + if domain == "perfume": + return PerfumeRuleBasedExtractor() + return GenericRuleBasedExtractor() diff --git a/crawler_platform/app/core/extractor/rule_based.py b/crawler_platform/app/core/extractor/rule_based.py new file mode 100644 index 0000000..6794493 --- /dev/null +++ b/crawler_platform/app/core/extractor/rule_based.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import re + +from crawler_platform.app.config.loader import ProjectConfig +from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle, Extractor +from crawler_platform.app.core.ontology.mapper import normalize_predicate + + +class GenericRuleBasedExtractor(Extractor): + name = "generic_rule_based" + provider = "rule_based" + + def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]: + name = first_non_empty_line(page_text) or "Unknown Product" + primary_type = project_config.target_entities[0] if project_config.target_entities else "Product" + return [ExtractedEntity(entity_type=primary_type, name=name, confidence=0.45)] + + def extract_attributes( + self, + entity: ExtractedEntity, + page_text: str, + project_config: ProjectConfig, + ) -> dict[str, object]: + attrs: dict[str, object] = {"name": entity.name} + price = find_price(page_text) + if price: + attrs["price"] = price + return attrs + + def extract_relations( + self, + entities: list[ExtractedEntity], + page_text: str, + project_config: ProjectConfig, + ) -> list[ExtractedClaim]: + if not entities: + return [] + entity = entities[0] + claims: list[ExtractedClaim] = [] + price = find_price(page_text) + if price: + claims.append( + ExtractedClaim( + subject_name=entity.name, + subject_type=entity.entity_type, + predicate="hasPrice", + object_value=price, + evidence_text=price["evidence"], + confidence=0.6, + confidence_reason="price pattern matched", + ) + ) + return claims + + def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, object]) -> ExtractionBundle: + for claim in bundle.claims: + claim.predicate = normalize_predicate(claim.predicate, ontology) + return bundle + + +def first_non_empty_line(text: str) -> str | None: + for line in text.splitlines(): + clean = line.strip() + if clean: + return clean[:240] + return None + + +def find_price(text: str) -> dict[str, object] | None: + patterns = [ + r"(?P[$€£])\s?(?P\d+(?:[,.]\d{2})?)", + r"(?P\d{1,3}(?:,\d{3})*)\s?(?P원|KRW|USD)", + ] + for pattern in patterns: + match = re.search(pattern, text, flags=re.IGNORECASE) + if match: + amount = match.group("amount").replace(",", "") + return { + "amount": float(amount), + "currency": match.group("currency"), + "evidence": match.group(0), + } + return None + diff --git a/crawler_platform/app/core/ontology/__init__.py b/crawler_platform/app/core/ontology/__init__.py new file mode 100644 index 0000000..f04bbdc --- /dev/null +++ b/crawler_platform/app/core/ontology/__init__.py @@ -0,0 +1,2 @@ +"""Ontology definitions and mapping helpers.""" + diff --git a/crawler_platform/app/core/ontology/definitions.py b/crawler_platform/app/core/ontology/definitions.py new file mode 100644 index 0000000..4103e5f --- /dev/null +++ b/crawler_platform/app/core/ontology/definitions.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(slots=True) +class Ontology: + domain: str + entity_types: list[str] + predicates: list[str] + attributes: list[str] + aliases: dict[str, str] = field(default_factory=dict) + + +COMMON_ONTOLOGY = Ontology( + domain="common", + entity_types=[ + "Source", + "Page", + "Entity", + "Attribute", + "Relation", + "Claim", + "Evidence", + "Extraction", + "Confidence", + "UpdatePolicy", + ], + predicates=[ + "mentions", + "hasAttribute", + "relatedTo", + "sameAs", + "soldBy", + "hasPrice", + ], + attributes=["name", "source_url", "updated_at", "confidence"], +) + + +DOMAIN_ONTOLOGIES: dict[str, Ontology] = { + "perfume": Ontology( + domain="perfume", + entity_types=[ + "Perfume", + "Brand", + "Note", + "Accord", + "Mood", + "Season", + "Occasion", + "Review", + "Price", + "ProductPage", + ], + predicates=[ + "hasBrand", + "hasTopNote", + "hasMiddleNote", + "hasBaseNote", + "hasAccord", + "evokesMood", + "suitableForSeason", + "suitableForOccasion", + "similarTo", + "soldBy", + "hasPrice", + "hasReviewKeyword", + ], + attributes=[ + "name", + "brand", + "gender_bias", + "longevity", + "sillage", + "price_range", + "popularity_score", + "review_count", + "source_url", + "updated_at", + ], + aliases={ + "top_notes": "hasTopNote", + "middle_notes": "hasMiddleNote", + "heart_notes": "hasMiddleNote", + "base_notes": "hasBaseNote", + "accords": "hasAccord", + "mood_tags": "evokesMood", + "season_tags": "suitableForSeason", + "occasion_tags": "suitableForOccasion", + "review_keywords": "hasReviewKeyword", + "price": "hasPrice", + }, + ), + "tea": Ontology( + domain="tea", + entity_types=["Tea", "Ingredient", "Flavor", "Effect", "CaffeineLevel", "MoodState"], + predicates=["hasIngredient", "hasFlavor", "hasEffect", "suitableForCondition"], + attributes=["name", "origin", "caffeine_level", "price_range"], + ), + "coffee": Ontology( + domain="coffee", + entity_types=["CoffeeBean", "Origin", "RoastLevel", "FlavorNote", "BrewMethod"], + predicates=["hasOrigin", "hasRoastLevel", "hasFlavorNote", "recommendedForBrewMethod"], + attributes=["name", "origin", "roast_level", "process", "price_range"], + ), + "candle": Ontology( + domain="candle", + entity_types=["ScentProduct", "ScentNote", "SpaceType", "Mood", "Season"], + predicates=["suitableForSpace", "evokesMood", "hasScentNote"], + attributes=["name", "burn_time", "volume", "price_range"], + ), + "supplement": Ontology( + domain="supplement", + entity_types=["Supplement", "Ingredient", "HealthGoal", "Symptom", "Dosage"], + predicates=["hasIngredient", "supportsGoal", "recommendedForCondition"], + attributes=["name", "dosage", "warnings", "price_range"], + ), + "gift": Ontology( + domain="gift", + entity_types=["GiftProduct", "RecipientType", "Relationship", "Occasion", "PersonalityTag"], + predicates=["suitableForRecipient", "suitableForOccasion", "matchesPersonality"], + attributes=["name", "price_range", "availability", "gift_wrap_available"], + ), +} + + +def ontology_for_domain(domain: str) -> Ontology: + return DOMAIN_ONTOLOGIES.get(domain, COMMON_ONTOLOGY) + diff --git a/crawler_platform/app/core/ontology/mapper.py b/crawler_platform/app/core/ontology/mapper.py new file mode 100644 index 0000000..19c22bc --- /dev/null +++ b/crawler_platform/app/core/ontology/mapper.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Any + + +def normalize_predicate(predicate: str, ontology: dict[str, Any] | None) -> str: + if not ontology: + return predicate + aliases = ontology.get("aliases", {}) + return aliases.get(predicate, predicate) + + +def is_allowed_predicate(predicate: str, ontology: dict[str, Any] | None) -> bool: + if not ontology or not ontology.get("predicates"): + return True + return predicate in ontology["predicates"] + + +def ontology_to_dict(domain_ontology) -> dict[str, Any]: + return { + "domain": domain_ontology.domain, + "entity_types": domain_ontology.entity_types, + "predicates": domain_ontology.predicates, + "attributes": domain_ontology.attributes, + "aliases": domain_ontology.aliases, + } + diff --git a/crawler_platform/app/core/recommendation/__init__.py b/crawler_platform/app/core/recommendation/__init__.py new file mode 100644 index 0000000..5e1ce4e --- /dev/null +++ b/crawler_platform/app/core/recommendation/__init__.py @@ -0,0 +1,2 @@ +"""Recommendation integration helpers.""" + diff --git a/crawler_platform/app/core/recommendation/scorer.py b/crawler_platform/app/core/recommendation/scorer.py new file mode 100644 index 0000000..db03494 --- /dev/null +++ b/crawler_platform/app/core/recommendation/scorer.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from crawler_platform.app.core.database import models +from crawler_platform.app.core.database.repository import canonicalize + + +@dataclass(slots=True) +class PreferenceInput: + likes: list[str] = field(default_factory=list) + dislikes: list[str] = field(default_factory=list) + preferred_moods: list[str] = field(default_factory=list) + preferred_notes: list[str] = field(default_factory=list) + avoided_notes: list[str] = field(default_factory=list) + price_preference: dict[str, Any] = field(default_factory=dict) + season_context: str | None = None + occasion_context: str | None = None + + +@dataclass(slots=True) +class Recommendation: + entity_id: int + name: str + entity_type: str + score: float + reasons: list[str] + + +class RuleBasedRecommender: + def __init__(self, session: Session): + self.session = session + + def recommend( + self, + project_id: int, + target_entity_type: str, + preference: PreferenceInput, + limit: int = 10, + ) -> list[Recommendation]: + entities = self.session.scalars( + select(models.Entity).where( + models.Entity.project_id == project_id, + models.Entity.entity_type == target_entity_type, + ) + ).all() + scored = [self._score_entity(project_id, entity, preference) for entity in entities] + scored = [item for item in scored if item.score > 0] + return sorted(scored, key=lambda item: item.score, reverse=True)[:limit] + + def _score_entity(self, project_id: int, entity: models.Entity, preference: PreferenceInput) -> Recommendation: + claims = self.session.execute( + select(models.Claim, models.Entity) + .join(models.Entity, models.Claim.object_entity_id == models.Entity.id, isouter=True) + .where(models.Claim.project_id == project_id, models.Claim.subject_entity_id == entity.id) + ).all() + score = 0.0 + reasons: list[str] = [] + preferred_notes = {canonicalize(item) for item in preference.preferred_notes} + avoided_notes = {canonicalize(item) for item in preference.avoided_notes} + preferred_moods = {canonicalize(item) for item in preference.preferred_moods} + for claim, object_entity in claims: + object_name = canonicalize(object_entity.name) if object_entity else "" + weight = claim.confidence + if claim.predicate in {"hasTopNote", "hasMiddleNote", "hasBaseNote", "hasScentNote", "hasFlavorNote"}: + if object_name in preferred_notes: + score += 2.0 * weight + reasons.append(f"preferred note matched: {object_entity.name}") + if object_name in avoided_notes: + score -= 3.0 * weight + reasons.append(f"avoided note matched: {object_entity.name}") + if claim.predicate == "evokesMood" and object_name in preferred_moods: + score += 1.5 * weight + reasons.append(f"preferred mood matched: {object_entity.name}") + if preference.season_context and claim.predicate == "suitableForSeason": + if object_name == canonicalize(preference.season_context): + score += 1.2 * weight + reasons.append(f"season context matched: {preference.season_context}") + if preference.occasion_context and claim.predicate == "suitableForOccasion": + if object_name == canonicalize(preference.occasion_context): + score += 1.0 * weight + reasons.append(f"occasion context matched: {preference.occasion_context}") + if canonicalize(entity.name) in {canonicalize(item) for item in preference.dislikes}: + score -= 10 + reasons.append("explicit dislike") + if canonicalize(entity.name) in {canonicalize(item) for item in preference.likes}: + score += 5 + reasons.append("explicit like") + return Recommendation( + entity_id=entity.id, + name=entity.name, + entity_type=entity.entity_type, + score=round(score, 4), + reasons=reasons[:5], + ) + diff --git a/crawler_platform/app/core/scheduler/__init__.py b/crawler_platform/app/core/scheduler/__init__.py new file mode 100644 index 0000000..6a8d676 --- /dev/null +++ b/crawler_platform/app/core/scheduler/__init__.py @@ -0,0 +1,2 @@ +"""Scheduling primitives for recrawls.""" + diff --git a/crawler_platform/app/core/scheduler/update_policy.py b/crawler_platform/app/core/scheduler/update_policy.py new file mode 100644 index 0000000..86758a1 --- /dev/null +++ b/crawler_platform/app/core/scheduler/update_policy.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + + +def next_crawl_at(policy: dict[str, object] | None, now: datetime | None = None) -> datetime: + now = now or datetime.now(timezone.utc) + policy = policy or {} + interval_days = int(policy.get("interval_days", 7)) + return now + timedelta(days=interval_days) + diff --git a/crawler_platform/app/domains/__init__.py b/crawler_platform/app/domains/__init__.py new file mode 100644 index 0000000..41364f8 --- /dev/null +++ b/crawler_platform/app/domains/__init__.py @@ -0,0 +1,2 @@ +"""Domain plugins.""" + diff --git a/crawler_platform/app/domains/candle/__init__.py b/crawler_platform/app/domains/candle/__init__.py new file mode 100644 index 0000000..577d8d6 --- /dev/null +++ b/crawler_platform/app/domains/candle/__init__.py @@ -0,0 +1,2 @@ +"""Candle and diffuser domain extension point.""" + diff --git a/crawler_platform/app/domains/coffee/__init__.py b/crawler_platform/app/domains/coffee/__init__.py new file mode 100644 index 0000000..4adebd4 --- /dev/null +++ b/crawler_platform/app/domains/coffee/__init__.py @@ -0,0 +1,2 @@ +"""Coffee domain extension point.""" + diff --git a/crawler_platform/app/domains/gift/__init__.py b/crawler_platform/app/domains/gift/__init__.py new file mode 100644 index 0000000..85c81ec --- /dev/null +++ b/crawler_platform/app/domains/gift/__init__.py @@ -0,0 +1,2 @@ +"""Gift recommendation domain extension point.""" + diff --git a/crawler_platform/app/domains/perfume/__init__.py b/crawler_platform/app/domains/perfume/__init__.py new file mode 100644 index 0000000..f4f2a26 --- /dev/null +++ b/crawler_platform/app/domains/perfume/__init__.py @@ -0,0 +1,2 @@ +"""Perfume domain plugin.""" + diff --git a/crawler_platform/app/domains/perfume/extractor.py b/crawler_platform/app/domains/perfume/extractor.py new file mode 100644 index 0000000..931e114 --- /dev/null +++ b/crawler_platform/app/domains/perfume/extractor.py @@ -0,0 +1,350 @@ +from __future__ import annotations + +import re + +from crawler_platform.app.config.loader import ProjectConfig +from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle +from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor, find_price, first_non_empty_line +from crawler_platform.app.core.ontology.mapper import normalize_predicate + + +NOTE_LABELS = { + "top_notes": ["top notes", "top note", "opening notes", "탑 노트", "탑노트", "상단 노트"], + "middle_notes": ["middle notes", "heart notes", "미들 노트", "하트 노트", "미들노트"], + "base_notes": ["base notes", "base note", "베이스 노트", "베이스노트", "잔향"], +} + +FIELD_TO_PREDICATE = { + "top_notes": "hasTopNote", + "middle_notes": "hasMiddleNote", + "base_notes": "hasBaseNote", + "accords": "hasAccord", + "mood_tags": "evokesMood", + "season_tags": "suitableForSeason", + "occasion_tags": "suitableForOccasion", + "review_keywords": "hasReviewKeyword", +} + +MOOD_KEYWORDS = { + "Fresh": ["fresh", "clean", "상쾌", "깨끗", "청량"], + "Romantic": ["romantic", "soft", "로맨틱", "부드러운"], + "Elegant": ["elegant", "luxury", "우아", "고급"], + "Cozy": ["cozy", "warm", "포근", "따뜻"], + "Energetic": ["bright", "sparkling", "활기", "발랄"], +} + +SEASON_KEYWORDS = { + "Spring": ["spring", "봄"], + "Summer": ["summer", "여름"], + "Autumn": ["autumn", "fall", "가을"], + "Winter": ["winter", "겨울"], +} + +OCCASION_KEYWORDS = { + "Daily": ["daily", "everyday", "데일리", "매일"], + "Date": ["date", "데이트"], + "Office": ["office", "work", "오피스", "출근"], + "Evening": ["evening", "night", "저녁", "밤"], +} + +ACCORD_KEYWORDS = [ + "citrus", + "floral", + "woody", + "musky", + "amber", + "powdery", + "green", + "spicy", + "sweet", + "fresh", + "시트러스", + "플로럴", + "우디", + "머스크", + "앰버", + "파우더리", +] + +REVIEW_KEYWORDS = [ + "long lasting", + "compliment", + "too strong", + "soft", + "fresh", + "powdery", + "지속력", + "잔향", + "호불호", + "칭찬", + "은은", + "강한", +] + + +class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): + name = "perfume_rule_based" + + def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]: + product_name = extract_product_name(page_text) + attrs = {"name": product_name} + brand = extract_brand(page_text, product_name) + if brand: + attrs["brand"] = brand + price = find_price(page_text) + if price: + attrs["price"] = {k: v for k, v in price.items() if k != "evidence"} + entities = [ExtractedEntity("Perfume", product_name, attrs, confidence=0.68)] + if brand: + entities.append(ExtractedEntity("Brand", brand, confidence=0.62)) + for field, entity_type in [ + ("top_notes", "Note"), + ("middle_notes", "Note"), + ("base_notes", "Note"), + ("accords", "Accord"), + ("mood_tags", "Mood"), + ("season_tags", "Season"), + ("occasion_tags", "Occasion"), + ("review_keywords", "Review"), + ]: + for value, evidence in extract_field_values(field, page_text): + entities.append(ExtractedEntity(entity_type, value, evidence_text=evidence, confidence=0.6)) + return dedupe_entities(entities) + + def extract_attributes( + self, + entity: ExtractedEntity, + page_text: str, + project_config: ProjectConfig, + ) -> dict[str, object]: + if entity.entity_type != "Perfume": + return {} + attrs: dict[str, object] = {} + longevity = find_metric(page_text, ["longevity", "lasting", "지속력"]) + sillage = find_metric(page_text, ["sillage", "projection", "확산력", "발향"]) + if longevity: + attrs["longevity"] = longevity + if sillage: + attrs["sillage"] = sillage + gender_bias = find_gender_bias(page_text) + if gender_bias: + attrs["gender_bias"] = gender_bias + return attrs + + def extract_relations( + self, + entities: list[ExtractedEntity], + page_text: str, + project_config: ProjectConfig, + ) -> list[ExtractedClaim]: + perfume = next((entity for entity in entities if entity.entity_type == "Perfume"), None) + if perfume is None: + return [] + claims: list[ExtractedClaim] = [] + brand = next((entity for entity in entities if entity.entity_type == "Brand"), None) + if brand: + claims.append( + ExtractedClaim( + perfume.name, + "Perfume", + "hasBrand", + brand.name, + "Brand", + evidence_text=brand.evidence_text or brand.name, + confidence=0.78, + confidence_reason="brand pattern matched", + ) + ) + for field, predicate in FIELD_TO_PREDICATE.items(): + entity_type = field_entity_type(field) + for value, evidence in extract_field_values(field, page_text): + claims.append( + ExtractedClaim( + perfume.name, + "Perfume", + predicate, + value, + entity_type, + evidence_text=evidence, + evidence_summary=f"{field} includes {value}", + confidence=field_confidence(field), + confidence_reason=f"{field} rule matched", + ) + ) + price = find_price(page_text) + if price: + claims.append( + ExtractedClaim( + perfume.name, + "Perfume", + "hasPrice", + object_value={k: v for k, v in price.items() if k != "evidence"}, + evidence_text=price["evidence"], + confidence=0.7, + confidence_reason="price pattern matched", + ) + ) + return claims + + def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, object]) -> ExtractionBundle: + for claim in bundle.claims: + claim.predicate = normalize_predicate(claim.predicate, ontology) + return bundle + + +def extract_product_name(page_text: str) -> str: + for line in page_text.splitlines()[:8]: + clean = line.strip() + if clean and not looks_like_navigation(clean): + return clean[:240] + return first_non_empty_line(page_text) or "Unknown Perfume" + + +def extract_brand(page_text: str, product_name: str) -> str | None: + patterns = [ + r"(?:brand|브랜드)\s*[:：]\s*(?P[A-Za-z0-9가-힣 '&.-]{2,80})", + r"by\s+(?P[A-Z][A-Za-z0-9 '&.-]{2,80})", + ] + for pattern in patterns: + match = re.search(pattern, page_text, flags=re.IGNORECASE) + if match: + return cleanup_value(match.group("brand")) + lines = [line.strip() for line in page_text.splitlines() if line.strip()] + if len(lines) >= 2 and lines[1].lower() not in product_name.lower(): + candidate = cleanup_value(lines[1]) + if len(candidate) <= 80 and not looks_like_navigation(candidate): + return candidate + return None + + +def extract_field_values(field: str, page_text: str) -> list[tuple[str, str]]: + if field in NOTE_LABELS: + return extract_labeled_values(page_text, NOTE_LABELS[field]) + if field == "accords": + return keyword_values(page_text, ACCORD_KEYWORDS) + if field == "mood_tags": + return taxonomy_keyword_values(page_text, MOOD_KEYWORDS) + if field == "season_tags": + return taxonomy_keyword_values(page_text, SEASON_KEYWORDS) + if field == "occasion_tags": + return taxonomy_keyword_values(page_text, OCCASION_KEYWORDS) + if field == "review_keywords": + return keyword_values(page_text, REVIEW_KEYWORDS) + return [] + + +def extract_labeled_values(page_text: str, labels: list[str]) -> list[tuple[str, str]]: + values: list[tuple[str, str]] = [] + lines = page_text.splitlines() + for idx, line in enumerate(lines): + lower = line.lower() + if any(label.lower() in lower for label in labels): + evidence = line + raw = line.split(":", 1)[-1] if ":" in line else "" + if not raw and idx + 1 < len(lines): + raw = lines[idx + 1] + evidence = f"{line} {raw}" + for value in split_values(raw): + values.append((value, evidence[:1000])) + return values + + +def split_values(raw: str) -> list[str]: + raw = re.sub(r"\([^)]*\)", "", raw) + parts = re.split(r"[,/|·ㆍ]+|\band\b| 및 | 그리고 ", raw, flags=re.IGNORECASE) + return [cleanup_value(part) for part in parts if 1 < len(cleanup_value(part)) <= 80] + + +def keyword_values(page_text: str, keywords: list[str]) -> list[tuple[str, str]]: + lower = page_text.lower() + found: list[tuple[str, str]] = [] + for keyword in keywords: + if keyword.lower() in lower: + found.append((keyword.title() if keyword.isascii() else keyword, snippet_for(page_text, keyword))) + return found + + +def taxonomy_keyword_values(page_text: str, taxonomy: dict[str, list[str]]) -> list[tuple[str, str]]: + lower = page_text.lower() + found: list[tuple[str, str]] = [] + for label, keywords in taxonomy.items(): + for keyword in keywords: + if keyword.lower() in lower: + found.append((label, snippet_for(page_text, keyword))) + break + return found + + +def snippet_for(text: str, keyword: str, window: int = 160) -> str: + index = text.lower().find(keyword.lower()) + if index < 0: + return keyword + start = max(index - window // 2, 0) + end = min(index + len(keyword) + window // 2, len(text)) + return text[start:end].replace("\n", " ") + + +def cleanup_value(value: str) -> str: + return re.sub(r"\s+", " ", value.strip(" -:：[]()")).strip() + + +def looks_like_navigation(value: str) -> bool: + return value.lower() in {"home", "shop", "menu", "cart", "login", "검색", "장바구니", "홈"} + + +def field_entity_type(field: str) -> str: + return { + "top_notes": "Note", + "middle_notes": "Note", + "base_notes": "Note", + "accords": "Accord", + "mood_tags": "Mood", + "season_tags": "Season", + "occasion_tags": "Occasion", + "review_keywords": "Review", + }[field] + + +def field_confidence(field: str) -> float: + return { + "top_notes": 0.82, + "middle_notes": 0.82, + "base_notes": 0.82, + "accords": 0.66, + "mood_tags": 0.62, + "season_tags": 0.62, + "occasion_tags": 0.6, + "review_keywords": 0.58, + }[field] + + +def find_metric(page_text: str, labels: list[str]) -> str | None: + for label in labels: + match = re.search(rf"{label}\s*[:：]?\s*(?P\d(?:\.\d)?/5|moderate|strong|weak|long|short|좋음|강함|약함)", page_text, re.IGNORECASE) + if match: + return cleanup_value(match.group("value")) + return None + + +def find_gender_bias(page_text: str) -> str | None: + lower = page_text.lower() + if "unisex" in lower or "공용" in lower: + return "unisex" + if "for women" in lower or "여성" in lower: + return "feminine" + if "for men" in lower or "남성" in lower: + return "masculine" + return None + + +def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]: + seen: set[tuple[str, str]] = set() + result: list[ExtractedEntity] = [] + for entity in entities: + key = (entity.entity_type, entity.name.strip().lower()) + if key in seen: + continue + seen.add(key) + result.append(entity) + return result + diff --git a/crawler_platform/app/domains/supplement/__init__.py b/crawler_platform/app/domains/supplement/__init__.py new file mode 100644 index 0000000..8208613 --- /dev/null +++ b/crawler_platform/app/domains/supplement/__init__.py @@ -0,0 +1,2 @@ +"""Supplement domain extension point.""" + diff --git a/crawler_platform/app/domains/tea/__init__.py b/crawler_platform/app/domains/tea/__init__.py new file mode 100644 index 0000000..94d7a60 --- /dev/null +++ b/crawler_platform/app/domains/tea/__init__.py @@ -0,0 +1,2 @@ +"""Tea domain extension point.""" + diff --git a/crawler_platform/app/main.py b/crawler_platform/app/main.py new file mode 100644 index 0000000..fbde1f5 --- /dev/null +++ b/crawler_platform/app/main.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import os +from pathlib import Path + +from fastapi import FastAPI +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles + +from crawler_platform.app.api.routes import register_routes +from crawler_platform.app.core.database.session import init_db + + +DATABASE_URL = os.getenv("CRAWLER_DATABASE_URL", "sqlite:///crawler_platform.db") +STATIC_DIR = Path(__file__).parent / "web" / "static" + +app = FastAPI(title="Ontology Crawler Platform", version="0.1.0") +init_db(DATABASE_URL) +register_routes(app, DATABASE_URL) + +if STATIC_DIR.exists(): + app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") + + +@app.get("/", include_in_schema=False) +def admin_ui(): + return FileResponse(STATIC_DIR / "index.html") diff --git a/crawler_platform/app/web/__init__.py b/crawler_platform/app/web/__init__.py new file mode 100644 index 0000000..9be5afc --- /dev/null +++ b/crawler_platform/app/web/__init__.py @@ -0,0 +1,2 @@ +"""Admin web UI.""" + diff --git a/crawler_platform/app/web/static/app.js b/crawler_platform/app/web/static/app.js new file mode 100644 index 0000000..79b9b57 --- /dev/null +++ b/crawler_platform/app/web/static/app.js @@ -0,0 +1,405 @@ +const state = { + projects: [], + selectedProject: null, + projectDetail: null, + ontology: null, +}; + +const $ = (id) => document.getElementById(id); + +function csv(value) { + return value + .split(",") + .map((item) => item.trim()) + .filter(Boolean); +} + +async function api(path, options = {}) { + const response = await fetch(path, { + headers: { "Content-Type": "application/json" }, + ...options, + }); + if (!response.ok) { + let detail = `${response.status} ${response.statusText}`; + try { + const body = await response.json(); + detail = body.detail || body.error || detail; + } catch { + // Keep the HTTP status text. + } + throw new Error(detail); + } + return response.json(); +} + +function toast(message) { + const node = $("toast"); + node.textContent = message; + node.classList.add("show"); + window.setTimeout(() => node.classList.remove("show"), 2400); +} + +function renderProjects() { + const list = $("projectList"); + list.innerHTML = ""; + state.projects.forEach((project) => { + const button = document.createElement("button"); + button.className = `project-item ${state.selectedProject === project.name ? "active" : ""}`; + button.innerHTML = `${project.name}${project.domain}`; + button.addEventListener("click", () => selectProject(project.name)); + list.appendChild(button); + }); +} + +async function loadProjects() { + state.projects = await api("/projects"); + if (!state.selectedProject && state.projects.length) { + state.selectedProject = state.projects[0].name; + } + renderProjects(); + if (state.selectedProject) { + await selectProject(state.selectedProject); + } +} + +async function selectProject(projectName) { + state.selectedProject = projectName; + state.projectDetail = await api(`/projects/${encodeURIComponent(projectName)}`); + state.ontology = await api(`/ontology/${encodeURIComponent(state.projectDetail.domain)}`); + renderProjects(); + renderOverview(); + renderOntology(); + await Promise.all([loadEntities(), loadClaims(), loadTags()]); +} + +function renderOverview() { + const detail = state.projectDetail; + $("metricProject").textContent = detail?.name ?? "-"; + $("metricDomain").textContent = detail?.domain ?? "-"; + $("metricSources").textContent = detail?.sources?.length ?? 0; + const sourceSelect = $("sourceSelect"); + sourceSelect.innerHTML = ""; + (detail?.sources ?? []).forEach((source) => { + const option = document.createElement("option"); + option.value = source.name; + option.textContent = `${source.name} (${source.type})`; + sourceSelect.appendChild(option); + }); + $("sourceTable").innerHTML = table( + ["Name", "Type", "Trust", "Robots", "Rate"], + (detail?.sources ?? []).map((source) => [ + source.name, + source.type, + source.trust_level, + source.respect_robots_txt ? "on" : "off", + `${source.rate_limit_per_minute}/min`, + ]) + ); +} + +function renderOntology() { + const entityTypes = state.ontology?.entity_types ?? []; + const predicates = state.ontology?.predicates ?? []; + $("ontologyEntities").innerHTML = entityTypes.map(chip).join(""); + $("ontologyPredicates").innerHTML = predicates.map(chip).join(""); + const filter = $("entityTypeFilter"); + filter.innerHTML = `${entityTypes + .map((type) => ``) + .join("")}`; +} + +async function createProject() { + const configPath = $("configPath").value.trim(); + if (!configPath) return; + const result = await api("/projects", { + method: "POST", + body: JSON.stringify({ config_path: configPath }), + }); + toast(`프로젝트 생성: ${result.name}`); + state.selectedProject = result.name; + await loadProjects(); +} + +async function crawl() { + if (!state.selectedProject) return; + const sourceName = $("sourceSelect").value; + const url = $("crawlUrl").value.trim(); + const provider = $("extractorProvider").value; + const model = $("extractorModel").value.trim(); + const baseUrl = $("extractorBaseUrl").value.trim(); + $("crawlResult").textContent = "수집 중..."; + try { + const result = await api("/crawl", { + method: "POST", + body: JSON.stringify({ + config_path: $("configPath").value.trim(), + source_name: sourceName, + url, + extractor_provider: provider, + extractor_model: model || null, + extractor_base_url: baseUrl || null, + }), + }); + $("crawlResult").textContent = `analyzer ${provider}, page ${result.page_id}, claims ${result.claim_count}, entities ${result.entity_count}`; + toast("수집 완료"); + await Promise.all([loadEntities(), loadClaims(), loadTags()]); + } catch (error) { + $("crawlResult").textContent = `수집 실패: ${error.message}`; + toast("수집 실패"); + } +} + +function updateExtractorOptions() { + const provider = $("extractorProvider").value; + $("extractorOptions").classList.toggle("active", provider !== "rule_based"); + if (provider === "ollama" && !$("extractorBaseUrl").value.trim()) { + $("extractorBaseUrl").placeholder = "http://localhost:11434/api/chat"; + } else if (provider === "lm_studio" && !$("extractorBaseUrl").value.trim()) { + $("extractorBaseUrl").placeholder = "http://localhost:1234/v1"; + } else { + $("extractorBaseUrl").placeholder = "optional provider endpoint"; + } +} + +async function testExtractor() { + const provider = $("extractorProvider").value; + const baseUrl = $("extractorBaseUrl").value.trim(); + $("crawlResult").textContent = "분석기 연결 확인 중..."; + const result = await api("/extractors/models", { + method: "POST", + body: JSON.stringify({ + provider, + base_url: baseUrl || null, + }), + }); + if (!result.ok) { + $("crawlResult").textContent = `분석기 연결 실패: ${result.error}`; + toast("분석기 연결 실패"); + return; + } + const models = result.models ?? []; + if (models.length && !$("extractorModel").value.trim()) { + $("extractorModel").value = models[0].id; + } + $("crawlResult").textContent = models.length + ? `연결됨. 모델 ${models.length}개: ${models.map((model) => model.id).join(", ")}` + : "연결됨. 모델 목록은 비어 있습니다."; + toast("분석기 연결 확인 완료"); +} + +async function discover() { + const sourceName = $("sourceSelect").value; + const url = $("crawlUrl").value.trim(); + $("crawlResult").textContent = "주소 발견 중..."; + $("discoveredLinks").innerHTML = ""; + try { + const result = await api("/discover", { + method: "POST", + body: JSON.stringify({ + config_path: $("configPath").value.trim(), + source_name: sourceName, + url, + limit: 30, + }), + }); + if (!result.ok) { + $("crawlResult").textContent = result.error ?? "주소 발견 실패"; + return; + } + $("crawlResult").textContent = `발견된 주소 ${result.links.length}개`; + $("discoveredLinks").innerHTML = result.links.map(renderDiscoveredLink).join(""); + document.querySelectorAll("[data-discovered-url]").forEach((button) => { + button.addEventListener("click", () => { + $("crawlUrl").value = button.dataset.discoveredUrl; + toast("URL 입력칸에 넣었습니다."); + }); + }); + } catch (error) { + $("crawlResult").textContent = `주소 발견 실패: ${error.message}`; + toast("주소 발견 실패"); + } +} + +function renderDiscoveredLink(link) { + return ` + + `; +} + +async function loadEntities() { + if (!state.selectedProject) return; + const type = $("entityTypeFilter").value; + const query = type ? `?entity_type=${encodeURIComponent(type)}&limit=100` : "?limit=100"; + const entities = await api(`/projects/${encodeURIComponent(state.selectedProject)}/entities${query}`); + $("metricEntities").textContent = entities.length; + $("entityTable").innerHTML = table( + ["ID", "Type", "Name", "Metadata"], + entities.map((entity) => [ + entity.id, + entity.type, + entity.name, + `${escapeHtml(JSON.stringify(entity.metadata ?? {}))}`, + ]) + ); +} + +async function mergeEntities() { + if (!state.selectedProject) return; + const sourceId = Number($("mergeSourceId").value); + const targetId = Number($("mergeTargetId").value); + if (!sourceId || !targetId || sourceId === targetId) { + toast("병합할 ID와 남길 ID를 확인하세요."); + return; + } + const result = await api("/entities/merge", { + method: "POST", + body: JSON.stringify({ + project_name: state.selectedProject, + source_entity_id: sourceId, + target_entity_id: targetId, + }), + }); + if (!result.ok) { + toast(result.error ?? "병합 실패"); + return; + } + toast("Entity 병합 완료"); + $("mergeSourceId").value = ""; + $("mergeTargetId").value = ""; + await Promise.all([loadEntities(), loadClaims(), loadTags()]); +} + +async function loadClaims() { + if (!state.selectedProject) return; + const claims = await api(`/projects/${encodeURIComponent(state.selectedProject)}/claims?limit=100`); + $("claimTable").innerHTML = claims.map(renderClaim).join(""); + document.querySelectorAll("[data-save-claim]").forEach((button) => { + button.addEventListener("click", () => updateClaimConfidence(button.dataset.saveClaim)); + }); +} + +function renderClaim(claim) { + const object = claim.object ?? JSON.stringify(claim.object_value ?? ""); + return ` +
+
+ ${escapeHtml(claim.subject)} + ${escapeHtml(claim.predicate)} + ${escapeHtml(object)} + ${Math.round(claim.confidence * 100)}% +
+
${escapeHtml(claim.evidence_text ?? "")}
+
${escapeHtml(claim.source)} · ${escapeHtml(claim.page_url ?? "")}
+
+ + + +
+
+ `; +} + +async function updateClaimConfidence(claimId) { + const confidence = Number($(`confidence-${claimId}`).value); + const reason = $(`reason-${claimId}`).value.trim(); + await api(`/claims/${claimId}/confidence`, { + method: "PATCH", + body: JSON.stringify({ confidence, reason }), + }); + toast("신뢰도 수정 완료"); + await loadClaims(); +} + +async function loadTags() { + if (!state.selectedProject) return; + const tags = await api(`/projects/${encodeURIComponent(state.selectedProject)}/recommendation-tags`); + $("tagTable").innerHTML = table( + ["Predicate", "Type", "Name", "Support", "Confidence"], + tags.map((tag) => [ + tag.predicate, + tag.type, + tag.name, + tag.support_count, + Math.round(tag.max_confidence * 100) + "%", + ]) + ); +} + +async function recommend() { + if (!state.selectedProject) return; + const result = await api("/recommend", { + method: "POST", + body: JSON.stringify({ + project_name: state.selectedProject, + target_entity_type: state.projectDetail?.config?.recommendation?.target_entity_type ?? "Perfume", + preferences: { + preferred_notes: csv($("preferredNotes").value), + avoided_notes: csv($("avoidedNotes").value), + preferred_moods: csv($("preferredMoods").value), + season_context: $("seasonContext").value.trim() || null, + occasion_context: $("occasionContext").value.trim() || null, + }, + limit: 10, + }), + }); + $("recommendTable").innerHTML = table( + ["Name", "Type", "Score", "Reasons"], + result.map((item) => [item.name, item.entity_type, item.score, item.reasons.join(", ")]) + ); +} + +function chip(value) { + return `${escapeHtml(value)}`; +} + +function table(headers, rows) { + if (!rows.length) { + return `
데이터가 없습니다.
`; + } + return ` + + ${headers.map((header) => ``).join("")} + + ${rows + .map((row) => `${row.map((cell) => ``).join("")}`) + .join("")} + +
${escapeHtml(header)}
${String(cell)}
+ `; +} + +function escapeHtml(value) { + return String(value) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +document.querySelectorAll(".tab").forEach((tab) => { + tab.addEventListener("click", () => { + document.querySelectorAll(".tab").forEach((node) => node.classList.remove("active")); + document.querySelectorAll(".tab-panel").forEach((node) => node.classList.remove("active")); + tab.classList.add("active"); + $(tab.dataset.tab).classList.add("active"); + }); +}); + +$("refreshBtn").addEventListener("click", loadProjects); +$("createProjectBtn").addEventListener("click", createProject); +$("discoverBtn").addEventListener("click", discover); +$("crawlBtn").addEventListener("click", crawl); +$("extractorProvider").addEventListener("change", updateExtractorOptions); +$("testExtractorBtn").addEventListener("click", testExtractor); +$("loadEntitiesBtn").addEventListener("click", loadEntities); +$("mergeEntitiesBtn").addEventListener("click", mergeEntities); +$("loadClaimsBtn").addEventListener("click", loadClaims); +$("loadTagsBtn").addEventListener("click", loadTags); +$("recommendBtn").addEventListener("click", recommend); + +updateExtractorOptions(); +loadProjects().catch((error) => toast(error.message)); diff --git a/crawler_platform/app/web/static/index.html b/crawler_platform/app/web/static/index.html new file mode 100644 index 0000000..d444fe7 --- /dev/null +++ b/crawler_platform/app/web/static/index.html @@ -0,0 +1,179 @@ + + + + + + Ontology Crawler Platform + + + +
+
+

Ontology Crawler

+

프로젝트별 수집, Claim 검수, 온톨로지 매핑, 추천 태그 확인

+
+ +
+ +
+ + +
+ + +
+
+
+ Project + - +
+
+ Domain + - +
+
+ Sources + 0 +
+
+ Entities + 0 +
+
+
+

Sources

+
+
+
+ +
+
+
+

Entity Types

+
+
+
+

Predicates

+
+
+
+
+ +
+
+ + +
+
+ + + +
+
+
+ +
+
+ +
+
+
+ +
+
+ +
+
+
+ +
+
+ + + + + +
+ +
+
+
+
+ +
+ + + diff --git a/crawler_platform/app/web/static/styles.css b/crawler_platform/app/web/static/styles.css new file mode 100644 index 0000000..79a426d --- /dev/null +++ b/crawler_platform/app/web/static/styles.css @@ -0,0 +1,452 @@ +:root { + color-scheme: light; + --bg: #f6f7f4; + --surface: #ffffff; + --surface-2: #eef3ee; + --text: #202420; + --muted: #667063; + --line: #d8ded6; + --accent: #236b5b; + --accent-2: #9c4f30; + --danger: #a33434; + --shadow: 0 16px 40px rgba(24, 35, 28, 0.08); +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + min-width: 320px; + background: var(--bg); + color: var(--text); + font-family: Inter, "Segoe UI", Arial, sans-serif; + letter-spacing: 0; +} + +button, +input, +select { + font: inherit; +} + +button { + border: 1px solid var(--line); + background: var(--surface); + color: var(--text); + border-radius: 6px; + min-height: 36px; + padding: 0 12px; + cursor: pointer; +} + +button:hover { + border-color: var(--accent); +} + +button.primary { + background: var(--accent); + color: white; + border-color: var(--accent); +} + +input, +select { + width: 100%; + min-height: 36px; + border: 1px solid var(--line); + border-radius: 6px; + background: white; + padding: 0 10px; + color: var(--text); +} + +label { + display: grid; + gap: 6px; + color: var(--muted); + font-size: 12px; + font-weight: 700; +} + +.topbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + padding: 18px 24px; + border-bottom: 1px solid var(--line); + background: rgba(255, 255, 255, 0.86); + position: sticky; + top: 0; + z-index: 5; + backdrop-filter: blur(12px); +} + +.topbar h1 { + margin: 0; + font-size: 22px; +} + +.topbar p { + margin: 4px 0 0; + color: var(--muted); + font-size: 13px; +} + +.icon-button { + width: 40px; + padding: 0; + font-size: 20px; +} + +.layout { + display: grid; + grid-template-columns: 320px minmax(0, 1fr); + gap: 18px; + padding: 18px; +} + +.sidebar, +.workspace { + min-width: 0; +} + +.sidebar { + display: grid; + align-content: start; + gap: 14px; +} + +.panel, +.wide-panel, +.metric { + background: var(--surface); + border: 1px solid var(--line); + border-radius: 8px; + box-shadow: var(--shadow); +} + +.panel, +.wide-panel { + padding: 14px; +} + +.panel-head { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 10px; +} + +h2 { + margin: 0; + font-size: 15px; +} + +.field-row { + display: grid; + grid-template-columns: minmax(0, 1fr) 42px; + gap: 8px; +} + +.list { + display: grid; + gap: 8px; + margin-top: 12px; +} + +.project-item { + display: grid; + gap: 2px; + text-align: left; + height: auto; + min-height: 52px; + padding: 8px 10px; +} + +.project-item.active { + border-color: var(--accent); + background: var(--surface-2); +} + +.project-item strong { + font-size: 14px; +} + +.project-item span { + color: var(--muted); + font-size: 12px; +} + +.mini-log { + min-height: 38px; + margin-top: 10px; + color: var(--muted); + font-size: 12px; + line-height: 1.45; +} + +.button-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; +} + +.extractor-options { + display: none; + gap: 8px; +} + +.extractor-options.active { + display: grid; +} + +.extractor-options button { + width: 100%; +} + +.discovered-links { + display: grid; + gap: 6px; + margin-top: 10px; + max-height: 280px; + overflow: auto; +} + +.discovered-link { + display: grid; + gap: 2px; + min-height: 44px; + padding: 7px 8px; + text-align: left; + overflow-wrap: anywhere; +} + +.discovered-link span { + color: var(--muted); + font-size: 11px; +} + +.tabs { + display: flex; + gap: 6px; + overflow-x: auto; + padding-bottom: 10px; +} + +.tab { + white-space: nowrap; +} + +.tab.active { + background: var(--text); + color: white; + border-color: var(--text); +} + +.tab-panel { + display: none; +} + +.tab-panel.active { + display: grid; + gap: 14px; +} + +.summary-grid { + display: grid; + grid-template-columns: repeat(4, minmax(120px, 1fr)); + gap: 12px; +} + +.metric { + padding: 14px; +} + +.metric span { + display: block; + color: var(--muted); + font-size: 12px; + font-weight: 700; +} + +.metric strong { + display: block; + margin-top: 8px; + font-size: 20px; + overflow-wrap: anywhere; +} + +.split { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 14px; +} + +.chips { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-top: 12px; +} + +.chip { + display: inline-flex; + align-items: center; + min-height: 28px; + padding: 0 10px; + border: 1px solid var(--line); + border-radius: 999px; + background: var(--surface-2); + font-size: 12px; +} + +.toolbar { + display: flex; + gap: 8px; + align-items: center; +} + +.toolbar.wrap { + flex-wrap: wrap; +} + +.toolbar select { + max-width: 240px; +} + +.merge-bar { + display: grid; + grid-template-columns: minmax(120px, 180px) minmax(120px, 180px) 80px; + gap: 8px; +} + +.table { + overflow-x: auto; + background: var(--surface); + border: 1px solid var(--line); + border-radius: 8px; +} + +table { + width: 100%; + border-collapse: collapse; + font-size: 13px; +} + +th, +td { + padding: 10px; + border-bottom: 1px solid var(--line); + text-align: left; + vertical-align: top; +} + +th { + color: var(--muted); + background: #fafbf8; + font-size: 12px; +} + +.claim-list { + display: grid; + gap: 10px; +} + +.claim-card { + display: grid; + gap: 8px; + padding: 12px; + background: var(--surface); + border: 1px solid var(--line); + border-radius: 8px; +} + +.claim-main { + display: flex; + flex-wrap: wrap; + gap: 8px; + align-items: center; +} + +.predicate { + color: var(--accent); + font-weight: 800; +} + +.confidence { + margin-left: auto; + color: var(--accent-2); + font-weight: 800; +} + +.evidence { + color: var(--muted); + font-size: 12px; + line-height: 1.45; +} + +.claim-actions { + display: grid; + grid-template-columns: 120px minmax(0, 1fr) 90px; + gap: 8px; +} + +.recommend-grid { + display: grid; + grid-template-columns: repeat(5, minmax(120px, 1fr)); + gap: 10px; +} + +#toast { + position: fixed; + right: 18px; + bottom: 18px; + max-width: min(420px, calc(100vw - 36px)); + padding: 12px 14px; + border-radius: 8px; + background: var(--text); + color: white; + opacity: 0; + transform: translateY(10px); + transition: 180ms ease; + pointer-events: none; + font-size: 13px; +} + +#toast.show { + opacity: 1; + transform: translateY(0); +} + +@media (max-width: 920px) { + .layout { + grid-template-columns: 1fr; + } + + .summary-grid, + .split, + .recommend-grid { + grid-template-columns: 1fr 1fr; + } +} + +@media (max-width: 560px) { + .topbar { + padding: 14px; + } + + .layout { + padding: 12px; + } + + .summary-grid, + .split, + .recommend-grid, + .claim-actions, + .merge-bar { + grid-template-columns: 1fr; + } +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..da15c59 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "crawler-platform" +version = "0.1.0" +description = "Ontology-centered crawler and knowledge DB for reusable subscription recommendation platforms." +requires-python = ">=3.11" +dependencies = [ + "beautifulsoup4>=4.12", + "fastapi>=0.110", + "playwright>=1.44", + "pydantic>=2", + "PyYAML>=6", + "requests>=2.31", + "SQLAlchemy>=2", + "uvicorn[standard]>=0.29", +] + +[project.optional-dependencies] +test = ["pytest>=8"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] +addopts = "-p no:cacheprovider" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0536cd5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +beautifulsoup4>=4.12 +fastapi>=0.110 +playwright>=1.44 +pydantic>=2 +pytest>=8 +PyYAML>=6 +requests>=2.31 +SQLAlchemy>=2 +uvicorn[standard]>=0.29 + diff --git a/tests/fixtures/sample_perfume.html b/tests/fixtures/sample_perfume.html new file mode 100644 index 0000000..010ec94 --- /dev/null +++ b/tests/fixtures/sample_perfume.html @@ -0,0 +1,17 @@ + + + + Neroli Summer Eau de Parfum + + +
+

Neroli Summer Eau de Parfum

+

Brand: Example Maison

+

Top notes: Bergamot, Neroli

+

Middle notes: Jasmine

+

Base notes: Musk

+

A fresh daily perfume for summer office wear.

+

Price $89

+
+ + diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py new file mode 100644 index 0000000..69a4815 --- /dev/null +++ b/tests/test_config_loader.py @@ -0,0 +1,12 @@ +from pathlib import Path + +from crawler_platform.app.config.loader import load_project_config + + +def test_load_perfume_config(): + config = load_project_config(Path("configs/perfume_subscription.yaml")) + + assert config.project_name == "perfume_subscription" + assert config.domain == "perfume" + assert config.source_by_name("official_brand_site").trust_level == 0.95 + diff --git a/tests/test_perfume_extractor.py b/tests/test_perfume_extractor.py new file mode 100644 index 0000000..c3f260c --- /dev/null +++ b/tests/test_perfume_extractor.py @@ -0,0 +1,31 @@ +from crawler_platform.app.config.loader import ProjectConfig, SourceConfig +from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor + + +def test_perfume_rule_based_extractor_maps_claims(): + config = ProjectConfig( + project_name="perfume_subscription", + domain="perfume", + target_entities=["Perfume", "Brand", "Note"], + fields=["name", "brand", "top_notes", "middle_notes", "base_notes"], + sources=[SourceConfig(name="official_brand_site", trust_level=0.95)], + ontology={"aliases": {"top_notes": "hasTopNote"}}, + ) + text = """ + Neroli Summer Eau de Parfum + Brand: Example Maison + Top notes: Bergamot, Neroli + Middle notes: Jasmine + Base notes: Musk + A fresh daily perfume for summer office wear. Price $89. + """ + bundle = PerfumeRuleBasedExtractor().extract(text, config) + predicates = {claim.predicate for claim in bundle.claims} + entity_names = {entity.name for entity in bundle.entities} + + assert "Neroli Summer Eau de Parfum" in entity_names + assert "Bergamot" in entity_names + assert "hasTopNote" in predicates + assert "suitableForSeason" in predicates + assert "hasPrice" in predicates + diff --git a/tests/test_smoke_unittest.py b/tests/test_smoke_unittest.py new file mode 100644 index 0000000..157f807 --- /dev/null +++ b/tests/test_smoke_unittest.py @@ -0,0 +1,21 @@ +import unittest + +from crawler_platform.app.config.loader import load_project_config +from crawler_platform.app.core.crawler.fetchers import RequestsFetcher +from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor + + +class SmokeTest(unittest.TestCase): + def test_config_fetch_and_extract_without_network(self): + config = load_project_config("configs/perfume_subscription.yaml") + fetch_result = RequestsFetcher().fetch("tests/fixtures/sample_perfume.html") + bundle = PerfumeRuleBasedExtractor().extract(fetch_result.html, config) + + self.assertEqual(config.domain, "perfume") + self.assertEqual(fetch_result.status_code, 200) + self.assertIn("hasTopNote", {claim.predicate for claim in bundle.claims}) + self.assertIn("hasPrice", {claim.predicate for claim in bundle.claims}) + + +if __name__ == "__main__": + unittest.main()