[crawler]

crawler_platform/app/domains/__init__.py

@@ -0,0 +1,2 @@
+"""Domain plugins."""
+

crawler_platform/app/domains/candle/__init__.py

@@ -0,0 +1,2 @@
+"""Candle and diffuser domain extension point."""
+

crawler_platform/app/domains/coffee/__init__.py

@@ -0,0 +1,2 @@
+"""Coffee domain extension point."""
+

crawler_platform/app/domains/gift/__init__.py

@@ -0,0 +1,2 @@
+"""Gift recommendation domain extension point."""
+

crawler_platform/app/domains/perfume/__init__.py

@@ -0,0 +1,2 @@
+"""Perfume domain plugin."""
+

crawler_platform/app/domains/perfume/extractor.py

@@ -0,0 +1,350 @@
+from __future__ import annotations
+
+import re
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle
+from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor, find_price, first_non_empty_line
+from crawler_platform.app.core.ontology.mapper import normalize_predicate
+
+
+NOTE_LABELS = {
+    "top_notes": ["top notes", "top note", "opening notes", "탑 노트", "탑노트", "상단 노트"],
+    "middle_notes": ["middle notes", "heart notes", "미들 노트", "하트 노트", "미들노트"],
+    "base_notes": ["base notes", "base note", "베이스 노트", "베이스노트", "잔향"],
+}
+
+FIELD_TO_PREDICATE = {
+    "top_notes": "hasTopNote",
+    "middle_notes": "hasMiddleNote",
+    "base_notes": "hasBaseNote",
+    "accords": "hasAccord",
+    "mood_tags": "evokesMood",
+    "season_tags": "suitableForSeason",
+    "occasion_tags": "suitableForOccasion",
+    "review_keywords": "hasReviewKeyword",
+}
+
+MOOD_KEYWORDS = {
+    "Fresh": ["fresh", "clean", "상쾌", "깨끗", "청량"],
+    "Romantic": ["romantic", "soft", "로맨틱", "부드러운"],
+    "Elegant": ["elegant", "luxury", "우아", "고급"],
+    "Cozy": ["cozy", "warm", "포근", "따뜻"],
+    "Energetic": ["bright", "sparkling", "활기", "발랄"],
+}
+
+SEASON_KEYWORDS = {
+    "Spring": ["spring", "봄"],
+    "Summer": ["summer", "여름"],
+    "Autumn": ["autumn", "fall", "가을"],
+    "Winter": ["winter", "겨울"],
+}
+
+OCCASION_KEYWORDS = {
+    "Daily": ["daily", "everyday", "데일리", "매일"],
+    "Date": ["date", "데이트"],
+    "Office": ["office", "work", "오피스", "출근"],
+    "Evening": ["evening", "night", "저녁", "밤"],
+}
+
+ACCORD_KEYWORDS = [
+    "citrus",
+    "floral",
+    "woody",
+    "musky",
+    "amber",
+    "powdery",
+    "green",
+    "spicy",
+    "sweet",
+    "fresh",
+    "시트러스",
+    "플로럴",
+    "우디",
+    "머스크",
+    "앰버",
+    "파우더리",
+]
+
+REVIEW_KEYWORDS = [
+    "long lasting",
+    "compliment",
+    "too strong",
+    "soft",
+    "fresh",
+    "powdery",
+    "지속력",
+    "잔향",
+    "호불호",
+    "칭찬",
+    "은은",
+    "강한",
+]
+
+
+class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
+    name = "perfume_rule_based"
+
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        product_name = extract_product_name(page_text)
+        attrs = {"name": product_name}
+        brand = extract_brand(page_text, product_name)
+        if brand:
+            attrs["brand"] = brand
+        price = find_price(page_text)
+        if price:
+            attrs["price"] = {k: v for k, v in price.items() if k != "evidence"}
+        entities = [ExtractedEntity("Perfume", product_name, attrs, confidence=0.68)]
+        if brand:
+            entities.append(ExtractedEntity("Brand", brand, confidence=0.62))
+        for field, entity_type in [
+            ("top_notes", "Note"),
+            ("middle_notes", "Note"),
+            ("base_notes", "Note"),
+            ("accords", "Accord"),
+            ("mood_tags", "Mood"),
+            ("season_tags", "Season"),
+            ("occasion_tags", "Occasion"),
+            ("review_keywords", "Review"),
+        ]:
+            for value, evidence in extract_field_values(field, page_text):
+                entities.append(ExtractedEntity(entity_type, value, evidence_text=evidence, confidence=0.6))
+        return dedupe_entities(entities)
+
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, object]:
+        if entity.entity_type != "Perfume":
+            return {}
+        attrs: dict[str, object] = {}
+        longevity = find_metric(page_text, ["longevity", "lasting", "지속력"])
+        sillage = find_metric(page_text, ["sillage", "projection", "확산력", "발향"])
+        if longevity:
+            attrs["longevity"] = longevity
+        if sillage:
+            attrs["sillage"] = sillage
+        gender_bias = find_gender_bias(page_text)
+        if gender_bias:
+            attrs["gender_bias"] = gender_bias
+        return attrs
+
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        perfume = next((entity for entity in entities if entity.entity_type == "Perfume"), None)
+        if perfume is None:
+            return []
+        claims: list[ExtractedClaim] = []
+        brand = next((entity for entity in entities if entity.entity_type == "Brand"), None)
+        if brand:
+            claims.append(
+                ExtractedClaim(
+                    perfume.name,
+                    "Perfume",
+                    "hasBrand",
+                    brand.name,
+                    "Brand",
+                    evidence_text=brand.evidence_text or brand.name,
+                    confidence=0.78,
+                    confidence_reason="brand pattern matched",
+                )
+            )
+        for field, predicate in FIELD_TO_PREDICATE.items():
+            entity_type = field_entity_type(field)
+            for value, evidence in extract_field_values(field, page_text):
+                claims.append(
+                    ExtractedClaim(
+                        perfume.name,
+                        "Perfume",
+                        predicate,
+                        value,
+                        entity_type,
+                        evidence_text=evidence,
+                        evidence_summary=f"{field} includes {value}",
+                        confidence=field_confidence(field),
+                        confidence_reason=f"{field} rule matched",
+                    )
+                )
+        price = find_price(page_text)
+        if price:
+            claims.append(
+                ExtractedClaim(
+                    perfume.name,
+                    "Perfume",
+                    "hasPrice",
+                    object_value={k: v for k, v in price.items() if k != "evidence"},
+                    evidence_text=price["evidence"],
+                    confidence=0.7,
+                    confidence_reason="price pattern matched",
+                )
+            )
+        return claims
+
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, object]) -> ExtractionBundle:
+        for claim in bundle.claims:
+            claim.predicate = normalize_predicate(claim.predicate, ontology)
+        return bundle
+
+
+def extract_product_name(page_text: str) -> str:
+    for line in page_text.splitlines()[:8]:
+        clean = line.strip()
+        if clean and not looks_like_navigation(clean):
+            return clean[:240]
+    return first_non_empty_line(page_text) or "Unknown Perfume"
+
+
+def extract_brand(page_text: str, product_name: str) -> str | None:
+    patterns = [
+        r"(?:brand|브랜드)\s*[:：]\s*(?P<brand>[A-Za-z0-9가-힣 '&.-]{2,80})",
+        r"by\s+(?P<brand>[A-Z][A-Za-z0-9 '&.-]{2,80})",
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, page_text, flags=re.IGNORECASE)
+        if match:
+            return cleanup_value(match.group("brand"))
+    lines = [line.strip() for line in page_text.splitlines() if line.strip()]
+    if len(lines) >= 2 and lines[1].lower() not in product_name.lower():
+        candidate = cleanup_value(lines[1])
+        if len(candidate) <= 80 and not looks_like_navigation(candidate):
+            return candidate
+    return None
+
+
+def extract_field_values(field: str, page_text: str) -> list[tuple[str, str]]:
+    if field in NOTE_LABELS:
+        return extract_labeled_values(page_text, NOTE_LABELS[field])
+    if field == "accords":
+        return keyword_values(page_text, ACCORD_KEYWORDS)
+    if field == "mood_tags":
+        return taxonomy_keyword_values(page_text, MOOD_KEYWORDS)
+    if field == "season_tags":
+        return taxonomy_keyword_values(page_text, SEASON_KEYWORDS)
+    if field == "occasion_tags":
+        return taxonomy_keyword_values(page_text, OCCASION_KEYWORDS)
+    if field == "review_keywords":
+        return keyword_values(page_text, REVIEW_KEYWORDS)
+    return []
+
+
+def extract_labeled_values(page_text: str, labels: list[str]) -> list[tuple[str, str]]:
+    values: list[tuple[str, str]] = []
+    lines = page_text.splitlines()
+    for idx, line in enumerate(lines):
+        lower = line.lower()
+        if any(label.lower() in lower for label in labels):
+            evidence = line
+            raw = line.split(":", 1)[-1] if ":" in line else ""
+            if not raw and idx + 1 < len(lines):
+                raw = lines[idx + 1]
+                evidence = f"{line} {raw}"
+            for value in split_values(raw):
+                values.append((value, evidence[:1000]))
+    return values
+
+
+def split_values(raw: str) -> list[str]:
+    raw = re.sub(r"\([^)]*\)", "", raw)
+    parts = re.split(r"[,/|·ㆍ]+|\band\b| 및 | 그리고 ", raw, flags=re.IGNORECASE)
+    return [cleanup_value(part) for part in parts if 1 < len(cleanup_value(part)) <= 80]
+
+
+def keyword_values(page_text: str, keywords: list[str]) -> list[tuple[str, str]]:
+    lower = page_text.lower()
+    found: list[tuple[str, str]] = []
+    for keyword in keywords:
+        if keyword.lower() in lower:
+            found.append((keyword.title() if keyword.isascii() else keyword, snippet_for(page_text, keyword)))
+    return found
+
+
+def taxonomy_keyword_values(page_text: str, taxonomy: dict[str, list[str]]) -> list[tuple[str, str]]:
+    lower = page_text.lower()
+    found: list[tuple[str, str]] = []
+    for label, keywords in taxonomy.items():
+        for keyword in keywords:
+            if keyword.lower() in lower:
+                found.append((label, snippet_for(page_text, keyword)))
+                break
+    return found
+
+
+def snippet_for(text: str, keyword: str, window: int = 160) -> str:
+    index = text.lower().find(keyword.lower())
+    if index < 0:
+        return keyword
+    start = max(index - window // 2, 0)
+    end = min(index + len(keyword) + window // 2, len(text))
+    return text[start:end].replace("\n", " ")
+
+
+def cleanup_value(value: str) -> str:
+    return re.sub(r"\s+", " ", value.strip(" -:：[]()")).strip()
+
+
+def looks_like_navigation(value: str) -> bool:
+    return value.lower() in {"home", "shop", "menu", "cart", "login", "검색", "장바구니", "홈"}
+
+
+def field_entity_type(field: str) -> str:
+    return {
+        "top_notes": "Note",
+        "middle_notes": "Note",
+        "base_notes": "Note",
+        "accords": "Accord",
+        "mood_tags": "Mood",
+        "season_tags": "Season",
+        "occasion_tags": "Occasion",
+        "review_keywords": "Review",
+    }[field]
+
+
+def field_confidence(field: str) -> float:
+    return {
+        "top_notes": 0.82,
+        "middle_notes": 0.82,
+        "base_notes": 0.82,
+        "accords": 0.66,
+        "mood_tags": 0.62,
+        "season_tags": 0.62,
+        "occasion_tags": 0.6,
+        "review_keywords": 0.58,
+    }[field]
+
+
+def find_metric(page_text: str, labels: list[str]) -> str | None:
+    for label in labels:
+        match = re.search(rf"{label}\s*[:：]?\s*(?P<value>\d(?:\.\d)?/5|moderate|strong|weak|long|short|좋음|강함|약함)", page_text, re.IGNORECASE)
+        if match:
+            return cleanup_value(match.group("value"))
+    return None
+
+
+def find_gender_bias(page_text: str) -> str | None:
+    lower = page_text.lower()
+    if "unisex" in lower or "공용" in lower:
+        return "unisex"
+    if "for women" in lower or "여성" in lower:
+        return "feminine"
+    if "for men" in lower or "남성" in lower:
+        return "masculine"
+    return None
+
+
+def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]:
+    seen: set[tuple[str, str]] = set()
+    result: list[ExtractedEntity] = []
+    for entity in entities:
+        key = (entity.entity_type, entity.name.strip().lower())
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(entity)
+    return result
+

crawler_platform/app/domains/supplement/__init__.py

@@ -0,0 +1,2 @@
+"""Supplement domain extension point."""
+

crawler_platform/app/domains/tea/__init__.py

@@ -0,0 +1,2 @@
+"""Tea domain extension point."""
+