from __future__ import annotations import re from crawler_platform.app.config.loader import ProjectConfig from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor, find_price, first_non_empty_line from crawler_platform.app.core.ontology.mapper import normalize_predicate NOTE_LABELS = { "top_notes": ["top notes", "top note", "opening notes", "탑 노트", "탑노트", "상단 노트"], "middle_notes": ["middle notes", "heart notes", "미들 노트", "하트 노트", "미들노트"], "base_notes": ["base notes", "base note", "베이스 노트", "베이스노트", "잔향"], } FIELD_TO_PREDICATE = { "top_notes": "hasTopNote", "middle_notes": "hasMiddleNote", "base_notes": "hasBaseNote", "accords": "hasAccord", "mood_tags": "evokesMood", "season_tags": "suitableForSeason", "occasion_tags": "suitableForOccasion", "review_keywords": "hasReviewKeyword", } MOOD_KEYWORDS = { "Fresh": ["fresh", "clean", "상쾌", "깨끗", "청량"], "Romantic": ["romantic", "soft", "로맨틱", "부드러운"], "Elegant": ["elegant", "luxury", "우아", "고급"], "Cozy": ["cozy", "warm", "포근", "따뜻"], "Energetic": ["bright", "sparkling", "활기", "발랄"], } SEASON_KEYWORDS = { "Spring": ["spring", "봄"], "Summer": ["summer", "여름"], "Autumn": ["autumn", "fall", "가을"], "Winter": ["winter", "겨울"], } OCCASION_KEYWORDS = { "Daily": ["daily", "everyday", "데일리", "매일"], "Date": ["date", "데이트"], "Office": ["office", "work", "오피스", "출근"], "Evening": ["evening", "night", "저녁", "밤"], } ACCORD_KEYWORDS = [ "citrus", "floral", "woody", "musky", "amber", "powdery", "green", "spicy", "sweet", "fresh", "시트러스", "플로럴", "우디", "머스크", "앰버", "파우더리", ] REVIEW_KEYWORDS = [ "long lasting", "compliment", "too strong", "soft", "fresh", "powdery", "지속력", "잔향", "호불호", "칭찬", "은은", "강한", ] class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): name = "perfume_rule_based" def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]: product_name = extract_product_name(page_text) attrs = {"name": product_name} brand = extract_brand(page_text, product_name) if brand: attrs["brand"] = brand price = find_price(page_text) if price: attrs["price"] = {k: v for k, v in price.items() if k != "evidence"} entities = [ExtractedEntity("Perfume", product_name, attrs, confidence=0.68)] if brand: entities.append(ExtractedEntity("Brand", brand, confidence=0.62)) for field, entity_type in [ ("top_notes", "Note"), ("middle_notes", "Note"), ("base_notes", "Note"), ("accords", "Accord"), ("mood_tags", "Mood"), ("season_tags", "Season"), ("occasion_tags", "Occasion"), ("review_keywords", "Review"), ]: for value, evidence in extract_field_values(field, page_text): entities.append(ExtractedEntity(entity_type, value, evidence_text=evidence, confidence=0.6)) return dedupe_entities(entities) def extract_attributes( self, entity: ExtractedEntity, page_text: str, project_config: ProjectConfig, ) -> dict[str, object]: if entity.entity_type != "Perfume": return {} attrs: dict[str, object] = {} longevity = find_metric(page_text, ["longevity", "lasting", "지속력"]) sillage = find_metric(page_text, ["sillage", "projection", "확산력", "발향"]) if longevity: attrs["longevity"] = longevity if sillage: attrs["sillage"] = sillage gender_bias = find_gender_bias(page_text) if gender_bias: attrs["gender_bias"] = gender_bias return attrs def extract_relations( self, entities: list[ExtractedEntity], page_text: str, project_config: ProjectConfig, ) -> list[ExtractedClaim]: perfume = next((entity for entity in entities if entity.entity_type == "Perfume"), None) if perfume is None: return [] claims: list[ExtractedClaim] = [] brand = next((entity for entity in entities if entity.entity_type == "Brand"), None) if brand: claims.append( ExtractedClaim( perfume.name, "Perfume", "hasBrand", brand.name, "Brand", evidence_text=brand.evidence_text or brand.name, confidence=0.78, confidence_reason="brand pattern matched", ) ) for field, predicate in FIELD_TO_PREDICATE.items(): entity_type = field_entity_type(field) for value, evidence in extract_field_values(field, page_text): claims.append( ExtractedClaim( perfume.name, "Perfume", predicate, value, entity_type, evidence_text=evidence, evidence_summary=f"{field} includes {value}", confidence=field_confidence(field), confidence_reason=f"{field} rule matched", ) ) price = find_price(page_text) if price: claims.append( ExtractedClaim( perfume.name, "Perfume", "hasPrice", object_value={k: v for k, v in price.items() if k != "evidence"}, evidence_text=price["evidence"], confidence=0.7, confidence_reason="price pattern matched", ) ) return claims def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, object]) -> ExtractionBundle: for claim in bundle.claims: claim.predicate = normalize_predicate(claim.predicate, ontology) return bundle def extract_product_name(page_text: str) -> str: for line in page_text.splitlines()[:8]: clean = line.strip() if clean and not looks_like_navigation(clean): return clean[:240] return first_non_empty_line(page_text) or "Unknown Perfume" def extract_brand(page_text: str, product_name: str) -> str | None: patterns = [ r"(?:brand|브랜드)\s*[::]\s*(?P[A-Za-z0-9가-힣 '&.-]{2,80})", r"by\s+(?P[A-Z][A-Za-z0-9 '&.-]{2,80})", ] for pattern in patterns: match = re.search(pattern, page_text, flags=re.IGNORECASE) if match: return cleanup_value(match.group("brand")) lines = [line.strip() for line in page_text.splitlines() if line.strip()] if len(lines) >= 2 and lines[1].lower() not in product_name.lower(): candidate = cleanup_value(lines[1]) if len(candidate) <= 80 and not looks_like_navigation(candidate): return candidate return None def extract_field_values(field: str, page_text: str) -> list[tuple[str, str]]: if field in NOTE_LABELS: return extract_labeled_values(page_text, NOTE_LABELS[field]) if field == "accords": return keyword_values(page_text, ACCORD_KEYWORDS) if field == "mood_tags": return taxonomy_keyword_values(page_text, MOOD_KEYWORDS) if field == "season_tags": return taxonomy_keyword_values(page_text, SEASON_KEYWORDS) if field == "occasion_tags": return taxonomy_keyword_values(page_text, OCCASION_KEYWORDS) if field == "review_keywords": return keyword_values(page_text, REVIEW_KEYWORDS) return [] def extract_labeled_values(page_text: str, labels: list[str]) -> list[tuple[str, str]]: values: list[tuple[str, str]] = [] lines = page_text.splitlines() for idx, line in enumerate(lines): lower = line.lower() if any(label.lower() in lower for label in labels): evidence = line raw = line.split(":", 1)[-1] if ":" in line else "" if not raw and idx + 1 < len(lines): raw = lines[idx + 1] evidence = f"{line} {raw}" for value in split_values(raw): values.append((value, evidence[:1000])) return values def split_values(raw: str) -> list[str]: raw = re.sub(r"\([^)]*\)", "", raw) parts = re.split(r"[,/|·ㆍ]+|\band\b| 및 | 그리고 ", raw, flags=re.IGNORECASE) return [cleanup_value(part) for part in parts if 1 < len(cleanup_value(part)) <= 80] def keyword_values(page_text: str, keywords: list[str]) -> list[tuple[str, str]]: lower = page_text.lower() found: list[tuple[str, str]] = [] for keyword in keywords: if keyword.lower() in lower: found.append((keyword.title() if keyword.isascii() else keyword, snippet_for(page_text, keyword))) return found def taxonomy_keyword_values(page_text: str, taxonomy: dict[str, list[str]]) -> list[tuple[str, str]]: lower = page_text.lower() found: list[tuple[str, str]] = [] for label, keywords in taxonomy.items(): for keyword in keywords: if keyword.lower() in lower: found.append((label, snippet_for(page_text, keyword))) break return found def snippet_for(text: str, keyword: str, window: int = 160) -> str: index = text.lower().find(keyword.lower()) if index < 0: return keyword start = max(index - window // 2, 0) end = min(index + len(keyword) + window // 2, len(text)) return text[start:end].replace("\n", " ") def cleanup_value(value: str) -> str: return re.sub(r"\s+", " ", value.strip(" -::[]()")).strip() def looks_like_navigation(value: str) -> bool: return value.lower() in {"home", "shop", "menu", "cart", "login", "검색", "장바구니", "홈"} def field_entity_type(field: str) -> str: return { "top_notes": "Note", "middle_notes": "Note", "base_notes": "Note", "accords": "Accord", "mood_tags": "Mood", "season_tags": "Season", "occasion_tags": "Occasion", "review_keywords": "Review", }[field] def field_confidence(field: str) -> float: return { "top_notes": 0.82, "middle_notes": 0.82, "base_notes": 0.82, "accords": 0.66, "mood_tags": 0.62, "season_tags": 0.62, "occasion_tags": 0.6, "review_keywords": 0.58, }[field] def find_metric(page_text: str, labels: list[str]) -> str | None: for label in labels: match = re.search(rf"{label}\s*[::]?\s*(?P\d(?:\.\d)?/5|moderate|strong|weak|long|short|좋음|강함|약함)", page_text, re.IGNORECASE) if match: return cleanup_value(match.group("value")) return None def find_gender_bias(page_text: str) -> str | None: lower = page_text.lower() if "unisex" in lower or "공용" in lower: return "unisex" if "for women" in lower or "여성" in lower: return "feminine" if "for men" in lower or "남성" in lower: return "masculine" return None def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]: seen: set[tuple[str, str]] = set() result: list[ExtractedEntity] = [] for entity in entities: key = (entity.entity_type, entity.name.strip().lower()) if key in seen: continue seen.add(key) result.append(entity) return result