@@ -0,0 +1,538 @@
from __future__ import annotations
import re
from crawler_platform . app . config . loader import ProjectConfig
from crawler_platform . app . core . extractor . base import ExtractedClaim , ExtractedEntity , ExtractionBundle
from crawler_platform . app . core . extractor . rule_based import GenericRuleBasedExtractor , find_price , first_non_empty_line
from crawler_platform . app . core . ontology . mapper import normalize_predicate
NOTE_LABELS = {
" top_notes " : [ " top notes " , " top note " , " opening notes " , " 탑 노트 " , " 탑노트 " , " 상단 노트 " ] ,
" middle_notes " : [ " middle notes " , " heart notes " , " 미들 노트 " , " 하트 노트 " , " 미들노트 " ] ,
" base_notes " : [ " base notes " , " base note " , " 베이스 노트 " , " 베이스노트 " , " 잔향 " ] ,
}
FIELD_TO_PREDICATE = {
" top_notes " : " hasTopNote " ,
" middle_notes " : " hasMiddleNote " ,
" base_notes " : " hasBaseNote " ,
" accords " : " hasAccord " ,
" mood_tags " : " evokesMood " ,
" season_tags " : " suitableForSeason " ,
" occasion_tags " : " suitableForOccasion " ,
" review_keywords " : " hasReviewKeyword " ,
}
MOOD_KEYWORDS = {
" Fresh " : [ " fresh " , " clean " , " 상쾌 " , " 깨끗 " , " 청량 " ] ,
" Romantic " : [ " romantic " , " soft " , " 로맨틱 " , " 부드러운 " ] ,
" Elegant " : [ " elegant " , " luxury " , " 우아 " , " 고급 " ] ,
" Cozy " : [ " cozy " , " warm " , " 포근 " , " 따뜻 " ] ,
" Energetic " : [ " bright " , " sparkling " , " 활기 " , " 발랄 " ] ,
}
SEASON_KEYWORDS = {
" Spring " : [ " spring " , " 봄 " ] ,
" Summer " : [ " summer " , " 여름 " ] ,
" Autumn " : [ " autumn " , " fall " , " 가을 " ] ,
" Winter " : [ " winter " , " 겨울 " ] ,
}
OCCASION_KEYWORDS = {
" Daily " : [ " daily " , " everyday " , " 데일리 " , " 매일 " ] ,
" Date " : [ " date " , " 데이트 " ] ,
" Office " : [ " office " , " work " , " 오피스 " , " 출근 " ] ,
" Evening " : [ " evening " , " night " , " 저녁 " , " 밤 " ] ,
}
ACCORD_KEYWORDS = [
" citrus " ,
" floral " ,
" woody " ,
" musky " ,
" amber " ,
" powdery " ,
" green " ,
" spicy " ,
" sweet " ,
" fresh " ,
" 시트러스 " ,
" 플로럴 " ,
" 우디 " ,
" 머스크 " ,
" 앰버 " ,
" 파우더리 " ,
]
REVIEW_KEYWORDS = [
" long lasting " ,
" compliment " ,
" too strong " ,
" soft " ,
" fresh " ,
" powdery " ,
" 지속력 " ,
" 잔향 " ,
" 호불호 " ,
" 칭찬 " ,
" 은은 " ,
" 강한 " ,
]
class PerfumeRuleBasedExtractor ( GenericRuleBasedExtractor ) :
name = " perfume_rule_based "
def extract_entities ( self , page_text : str , project_config : ProjectConfig ) - > list [ ExtractedEntity ] :
product_cards = extract_product_cards ( page_text )
if product_cards :
brand = valid_brand ( extract_brand ( page_text , " " ) ) or infer_site_brand ( page_text )
entities : list [ ExtractedEntity ] = [ ]
if brand :
entities . append ( ExtractedEntity ( " Brand " , brand , confidence = 0.62 ) )
for card in product_cards :
attrs : dict [ str , object ] = { " name " : card [ " name " ] }
if brand :
attrs [ " brand " ] = brand
if card . get ( " price " ) :
attrs [ " price " ] = card [ " price " ]
entities . append (
ExtractedEntity (
" Perfume " ,
str ( card [ " name " ] ) ,
attrs ,
evidence_text = str ( card . get ( " evidence " ) or card [ " name " ] ) ,
confidence = 0.74 ,
metadata = { " page_pattern " : " product_listing " } ,
)
)
return dedupe_entities ( entities )
product_name = extract_product_name ( page_text )
attrs = { " name " : product_name }
brand = extract_brand ( page_text , product_name )
if brand :
attrs [ " brand " ] = brand
price = find_price ( page_text )
if price :
attrs [ " price " ] = { k : v for k , v in price . items ( ) if k != " evidence " }
entities = [ ExtractedEntity ( " Perfume " , product_name , attrs , confidence = 0.74 ) ]
if brand :
entities . append ( ExtractedEntity ( " Brand " , brand , confidence = 0.62 ) )
for field , entity_type in [
( " top_notes " , " Note " ) ,
( " middle_notes " , " Note " ) ,
( " base_notes " , " Note " ) ,
( " accords " , " Accord " ) ,
( " mood_tags " , " Mood " ) ,
( " season_tags " , " Season " ) ,
( " occasion_tags " , " Occasion " ) ,
( " review_keywords " , " Review " ) ,
] :
for value , evidence in extract_field_values ( field , page_text ) :
entities . append ( ExtractedEntity ( entity_type , value , evidence_text = evidence , confidence = 0.6 ) )
return dedupe_entities ( entities )
def extract_attributes (
self ,
entity : ExtractedEntity ,
page_text : str ,
project_config : ProjectConfig ,
) - > dict [ str , object ] :
if entity . entity_type != " Perfume " :
return { }
attrs : dict [ str , object ] = { }
longevity = find_metric ( page_text , [ " longevity " , " lasting " , " 지속력 " ] )
sillage = find_metric ( page_text , [ " sillage " , " projection " , " 확산력 " , " 발향 " ] )
if longevity :
attrs [ " longevity " ] = longevity
if sillage :
attrs [ " sillage " ] = sillage
gender_bias = find_gender_bias ( page_text )
if gender_bias :
attrs [ " gender_bias " ] = gender_bias
return attrs
def extract_relations (
self ,
entities : list [ ExtractedEntity ] ,
page_text : str ,
project_config : ProjectConfig ,
) - > list [ ExtractedClaim ] :
product_cards = extract_product_cards ( page_text )
if product_cards :
claims : list [ ExtractedClaim ] = [ ]
brand = next ( ( entity for entity in entities if entity . entity_type == " Brand " ) , None )
for card in product_cards :
if brand :
claims . append (
ExtractedClaim (
str ( card [ " name " ] ) ,
" Perfume " ,
" hasBrand " ,
brand . name ,
" Brand " ,
evidence_text = brand . evidence_text or brand . name ,
confidence = 0.72 ,
confidence_reason = " site brand inferred from listing page " ,
)
)
if card . get ( " price " ) :
claims . append (
ExtractedClaim (
str ( card [ " name " ] ) ,
" Perfume " ,
" hasPrice " ,
object_value = card [ " price " ] ,
evidence_text = str ( card . get ( " evidence " ) or card [ " name " ] ) ,
confidence = 0.84 ,
confidence_reason = " Korean product listing price pattern matched " ,
)
)
return claims
perfume = next ( ( entity for entity in entities if entity . entity_type == " Perfume " ) , None )
if perfume is None :
return [ ]
claims : list [ ExtractedClaim ] = [ ]
brand = next ( ( entity for entity in entities if entity . entity_type == " Brand " ) , None )
if brand :
claims . append (
ExtractedClaim (
perfume . name ,
" Perfume " ,
" hasBrand " ,
brand . name ,
" Brand " ,
evidence_text = brand . evidence_text or brand . name ,
confidence = 0.78 ,
confidence_reason = " brand pattern matched " ,
)
)
for field , predicate in FIELD_TO_PREDICATE . items ( ) :
entity_type = field_entity_type ( field )
for value , evidence in extract_field_values ( field , page_text ) :
claims . append (
ExtractedClaim (
perfume . name ,
" Perfume " ,
predicate ,
value ,
entity_type ,
evidence_text = evidence ,
evidence_summary = f " { field } includes { value } " ,
confidence = field_confidence ( field ) ,
confidence_reason = f " { field } rule matched " ,
)
)
price = find_price ( page_text )
if price :
claims . append (
ExtractedClaim (
perfume . name ,
" Perfume " ,
" hasPrice " ,
object_value = { k : v for k , v in price . items ( ) if k != " evidence " } ,
evidence_text = price [ " evidence " ] ,
confidence = 0.86 ,
confidence_reason = " price pattern matched " ,
)
)
return claims
def normalize_to_ontology ( self , bundle : ExtractionBundle , ontology : dict [ str , object ] ) - > ExtractionBundle :
for claim in bundle . claims :
claim . predicate = normalize_predicate ( claim . predicate , ontology )
return bundle
def extract_product_name ( page_text : str ) - > str :
candidates : list [ tuple [ int , str ] ] = [ ]
for line in page_text . splitlines ( ) [ : 12 ] :
clean = line . strip ( )
if not clean or looks_like_navigation ( clean ) or is_template_placeholder ( clean ) :
continue
if looks_like_metric_or_price ( clean ) :
continue
candidates . append ( ( product_line_score ( clean ) , clean [ : 240 ] ) )
strong = [ candidate for candidate in candidates if candidate [ 0 ] > 0 ]
if strong :
return max ( strong , key = lambda item : item [ 0 ] ) [ 1 ]
if candidates :
return candidates [ 0 ] [ 1 ]
return first_non_empty_line ( page_text ) or " Unknown Perfume "
def extract_brand ( page_text : str , product_name : str ) - > str | None :
patterns = [
r " (?:brand|브랜드) \ s*[:： \ s*(?P<brand>[A-Za-z0-9가-힣 ' &.-] { 2,80}) " ,
r " by \ s+(?P<brand>[A-Z][A-Za-z0-9 ' &.-] { 2,80}) " ,
]
for pattern in patterns :
match = re . search ( pattern , page_text , flags = re . IGNORECASE )
if match :
return cleanup_value ( match . group ( " brand " ) )
inferred = infer_site_brand ( page_text )
if inferred :
return inferred
lines = [ line . strip ( ) for line in page_text . splitlines ( ) if line . strip ( ) ]
if len ( lines ) > = 2 and lines [ 1 ] . lower ( ) not in product_name . lower ( ) :
candidate = cleanup_value ( lines [ 1 ] )
if len ( candidate ) < = 80 and not looks_like_navigation ( candidate ) and product_line_score ( candidate ) < = 0 :
return candidate
return None
def product_line_score ( value : str ) - > int :
lower = value . lower ( )
score = 0
if re . search ( r " \ d+ \ s*(?:ml|g|개입) " , lower ) :
score + = 4
if any ( keyword in value for keyword in [ " 향수 " , " 디퓨저 " , " 스프레이 " , " 핸드크림 " , " 미스트 " , " 샤쉐 " , " 퍼퓸 " ] ) :
score + = 3
if value . startswith ( " [ " ) or any ( keyword in value for keyword in [ " 기획 " , " 추가할인 " , " 모음 " ] ) :
score + = 2
if " 912 " in value :
score + = 2
if " 시작 " in value and not re . search ( r " \ d+ \ s*(?:ml|g|개입) " , lower ) :
score - = 4
return score
def looks_like_metric_or_price ( value : str ) - > bool :
clean = value . replace ( " , " , " " ) . strip ( )
if re . fullmatch ( r " \ d+(?: \ . \ d+)? " , clean ) :
return True
return bool ( re . fullmatch ( r " \ d+(?: \ . \ d+)? \ s*(?:원|krw|usd)? " , clean , flags = re . IGNORECASE ) )
def extract_product_cards ( page_text : str ) - > list [ dict [ str , object ] ] :
lines = [ line . strip ( ) for line in page_text . splitlines ( ) if line . strip ( ) ]
cards : list [ dict [ str , object ] ] = [ ]
idx = 0
while idx < len ( lines ) :
if lines [ idx ] != " 상품명 " :
idx + = 1
continue
name , name_idx = next_value_after_label ( lines , idx )
if not name or is_template_placeholder ( name ) or name in { " : " , " 상품명 " } :
idx + = 1
continue
card : dict [ str , object ] = { " name " : cleanup_value ( name ) , " evidence " : f " 상품명: { name } " }
scan_end = next_label_index ( lines , " 상품명 " , name_idx + 1 ) or min ( len ( lines ) , name_idx + 12 )
for price_label in ( " 할인판매가 " , " 판매가 " , " price " , " Price " ) :
label_idx = find_label_index ( lines , price_label , name_idx + 1 , scan_end )
if label_idx is None :
continue
raw_price , _price_idx = next_value_after_label ( lines , label_idx )
parsed = parse_price_value ( raw_price )
if parsed :
card [ " price " ] = parsed
card [ " evidence " ] = f " { card [ ' evidence ' ] } / { price_label } : { raw_price } "
break
cards . append ( card )
idx = scan_end
return dedupe_product_cards ( cards )
def next_value_after_label ( lines : list [ str ] , label_idx : int ) - > tuple [ str | None , int ] :
for idx in range ( label_idx + 1 , min ( len ( lines ) , label_idx + 5 ) ) :
value = cleanup_value ( lines [ idx ] )
if not value or value == " : " :
continue
return value , idx
return None , label_idx
def next_label_index ( lines : list [ str ] , label : str , start : int ) - > int | None :
for idx in range ( start , len ( lines ) ) :
if lines [ idx ] == label :
return idx
return None
def find_label_index ( lines : list [ str ] , label : str , start : int , end : int ) - > int | None :
lower_label = label . lower ( )
for idx in range ( start , min ( end , len ( lines ) ) ) :
if lines [ idx ] . lower ( ) == lower_label :
return idx
return None
def parse_price_value ( raw_price : str | None ) - > dict [ str , object ] | None :
if not raw_price :
return None
match = re . search ( r " (?P<amount> \ d { 1,3}(?:, \ d {3} )*| \ d+) \ s*(?P<currency>원|KRW|₩|USD| \ $)? " , raw_price )
if not match :
return None
currency = match . group ( " currency " ) or " KRW "
if currency in { " 원 " , " ₩ " } :
currency = " KRW "
return { " amount " : float ( match . group ( " amount " ) . replace ( " , " , " " ) ) , " currency " : currency }
def dedupe_product_cards ( cards : list [ dict [ str , object ] ] ) - > list [ dict [ str , object ] ] :
seen : set [ str ] = set ( )
result : list [ dict [ str , object ] ] = [ ]
for card in cards :
key = str ( card [ " name " ] ) . strip ( ) . lower ( )
if key in seen :
continue
seen . add ( key )
result . append ( card )
return result
def infer_site_brand ( page_text : str ) - > str | None :
if " 912 공식 홈페이지 " in page_text or " 912 " in page_text [ : 500 ] :
return " 912 "
return None
def valid_brand ( value : str | None ) - > str | None :
if not value or is_template_placeholder ( value ) :
return None
return value
def extract_field_values ( field : str , page_text : str ) - > list [ tuple [ str , str ] ] :
if field in NOTE_LABELS :
return extract_labeled_values ( page_text , NOTE_LABELS [ field ] )
if field == " accords " :
return keyword_values ( page_text , ACCORD_KEYWORDS )
if field == " mood_tags " :
return taxonomy_keyword_values ( page_text , MOOD_KEYWORDS )
if field == " season_tags " :
return taxonomy_keyword_values ( page_text , SEASON_KEYWORDS )
if field == " occasion_tags " :
return taxonomy_keyword_values ( page_text , OCCASION_KEYWORDS )
if field == " review_keywords " :
return keyword_values ( page_text , REVIEW_KEYWORDS )
return [ ]
def extract_labeled_values ( page_text : str , labels : list [ str ] ) - > list [ tuple [ str , str ] ] :
values : list [ tuple [ str , str ] ] = [ ]
lines = page_text . splitlines ( )
for idx , line in enumerate ( lines ) :
lower = line . lower ( )
if any ( label . lower ( ) in lower for label in labels ) :
evidence = line
raw = line . split ( " : " , 1 ) [ - 1 ] if " : " in line else " "
if not raw and idx + 1 < len ( lines ) :
raw = lines [ idx + 1 ]
evidence = f " { line } { raw } "
for value in split_values ( raw ) :
values . append ( ( value , evidence [ : 1000 ] ) )
return values
def split_values ( raw : str ) - > list [ str ] :
raw = re . sub ( r " \ ([^)]* \ ) " , " " , raw )
parts = re . split ( r " [,/|·ㆍ]+| \ band \ b| 및 | 그리고 " , raw , flags = re . IGNORECASE )
return [ cleanup_value ( part ) for part in parts if 1 < len ( cleanup_value ( part ) ) < = 80 ]
def keyword_values ( page_text : str , keywords : list [ str ] ) - > list [ tuple [ str , str ] ] :
lower = page_text . lower ( )
found : list [ tuple [ str , str ] ] = [ ]
for keyword in keywords :
if keyword . lower ( ) in lower :
found . append ( ( keyword . title ( ) if keyword . isascii ( ) else keyword , snippet_for ( page_text , keyword ) ) )
return found
def taxonomy_keyword_values ( page_text : str , taxonomy : dict [ str , list [ str ] ] ) - > list [ tuple [ str , str ] ] :
lower = page_text . lower ( )
found : list [ tuple [ str , str ] ] = [ ]
for label , keywords in taxonomy . items ( ) :
for keyword in keywords :
if keyword . lower ( ) in lower :
found . append ( ( label , snippet_for ( page_text , keyword ) ) )
break
return found
def snippet_for ( text : str , keyword : str , window : int = 160 ) - > str :
index = text . lower ( ) . find ( keyword . lower ( ) )
if index < 0 :
return keyword
start = max ( index - window / / 2 , 0 )
end = min ( index + len ( keyword ) + window / / 2 , len ( text ) )
return text [ start : end ] . replace ( " \n " , " " )
def cleanup_value ( value : str ) - > str :
return re . sub ( r " \ s+ " , " " , value . strip ( " -:： " ) ) . strip ( )
def looks_like_navigation ( value : str ) - > bool :
return value . lower ( ) in { " home " , " shop " , " menu " , " cart " , " login " , " 검색 " , " 장바구니 " , " 홈 " }
def is_template_placeholder ( value : str ) - > bool :
clean = value . strip ( )
return clean . startswith ( " { # " ) or clean . endswith ( " } " ) or clean in {
" CLONE FRAGRANCE " ,
" NICHE FRAGRANCE " ,
" HOME FRAGRANCE " ,
}
def field_entity_type ( field : str ) - > str :
return {
" top_notes " : " Note " ,
" middle_notes " : " Note " ,
" base_notes " : " Note " ,
" accords " : " Accord " ,
" mood_tags " : " Mood " ,
" season_tags " : " Season " ,
" occasion_tags " : " Occasion " ,
" review_keywords " : " Review " ,
} [ field ]
def field_confidence ( field : str ) - > float :
return {
" top_notes " : 0.82 ,
" middle_notes " : 0.82 ,
" base_notes " : 0.82 ,
" accords " : 0.66 ,
" mood_tags " : 0.62 ,
" season_tags " : 0.62 ,
" occasion_tags " : 0.6 ,
" review_keywords " : 0.58 ,
} [ field ]
def find_metric ( page_text : str , labels : list [ str ] ) - > str | None :
for label in labels :
match = re . search ( rf " { label } \ s*[:： \ s*(?P<value> \ d(?: \ . \ d)?/5|moderate|strong|weak|long|short|좋음|강함|약함) " , page_text , re . IGNORECASE )
if match :
return cleanup_value ( match . group ( " value " ) )
return None
def find_gender_bias ( page_text : str ) - > str | None :
lower = page_text . lower ( )
if " unisex " in lower or " 공용 " in lower :
return " unisex "
if " for women " in lower or " 여성 " in lower :
return " feminine "
if " for men " in lower or " 남성 " in lower :
return " masculine "
return None
def dedupe_entities ( entities : list [ ExtractedEntity ] ) - > list [ ExtractedEntity ] :
seen : set [ tuple [ str , str ] ] = set ( )
result : list [ ExtractedEntity ] = [ ]
for entity in entities :
key = ( entity . entity_type , entity . name . strip ( ) . lower ( ) )
if key in seen :
continue
seen . add ( key )
result . append ( entity )
return result
LASTA_DEV01\lasta