[crawler]

2026-05-08 17:41:15 +09:00
commit 4158789869
54 changed files with 4355 additions and 0 deletions
--- a/tests/fixtures/sample_perfume.html
+++ b/tests/fixtures/sample_perfume.html
@@ -0,0 +1,17 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <title>Neroli Summer Eau de Parfum</title>
+  </head>
+  <body>
+    <main>
+      <h1>Neroli Summer Eau de Parfum</h1>
+      <p>Brand: Example Maison</p>
+      <p>Top notes: Bergamot, Neroli</p>
+      <p>Middle notes: Jasmine</p>
+      <p>Base notes: Musk</p>
+      <p>A fresh daily perfume for summer office wear.</p>
+      <p>Price $89</p>
+    </main>
+  </body>
+</html>
--- a/tests/test_config_loader.py
+++ b/tests/test_config_loader.py
@@ -0,0 +1,12 @@
+from pathlib import Path
+
+from crawler_platform.app.config.loader import load_project_config
+
+
+def test_load_perfume_config():
+    config = load_project_config(Path("configs/perfume_subscription.yaml"))
+
+    assert config.project_name == "perfume_subscription"
+    assert config.domain == "perfume"
+    assert config.source_by_name("official_brand_site").trust_level == 0.95
+
--- a/tests/test_perfume_extractor.py
+++ b/tests/test_perfume_extractor.py
@@ -0,0 +1,31 @@
+from crawler_platform.app.config.loader import ProjectConfig, SourceConfig
+from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
+
+
+def test_perfume_rule_based_extractor_maps_claims():
+    config = ProjectConfig(
+        project_name="perfume_subscription",
+        domain="perfume",
+        target_entities=["Perfume", "Brand", "Note"],
+        fields=["name", "brand", "top_notes", "middle_notes", "base_notes"],
+        sources=[SourceConfig(name="official_brand_site", trust_level=0.95)],
+        ontology={"aliases": {"top_notes": "hasTopNote"}},
+    )
+    text = """
+    Neroli Summer Eau de Parfum
+    Brand: Example Maison
+    Top notes: Bergamot, Neroli
+    Middle notes: Jasmine
+    Base notes: Musk
+    A fresh daily perfume for summer office wear. Price $89.
+    """
+    bundle = PerfumeRuleBasedExtractor().extract(text, config)
+    predicates = {claim.predicate for claim in bundle.claims}
+    entity_names = {entity.name for entity in bundle.entities}
+
+    assert "Neroli Summer Eau de Parfum" in entity_names
+    assert "Bergamot" in entity_names
+    assert "hasTopNote" in predicates
+    assert "suitableForSeason" in predicates
+    assert "hasPrice" in predicates
+
--- a/tests/test_smoke_unittest.py
+++ b/tests/test_smoke_unittest.py
@@ -0,0 +1,21 @@
+import unittest
+
+from crawler_platform.app.config.loader import load_project_config
+from crawler_platform.app.core.crawler.fetchers import RequestsFetcher
+from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
+
+
+class SmokeTest(unittest.TestCase):
+    def test_config_fetch_and_extract_without_network(self):
+        config = load_project_config("configs/perfume_subscription.yaml")
+        fetch_result = RequestsFetcher().fetch("tests/fixtures/sample_perfume.html")
+        bundle = PerfumeRuleBasedExtractor().extract(fetch_result.html, config)
+
+        self.assertEqual(config.domain, "perfume")
+        self.assertEqual(fetch_result.status_code, 200)
+        self.assertIn("hasTopNote", {claim.predicate for claim in bundle.claims})
+        self.assertIn("hasPrice", {claim.predicate for claim in bundle.claims})
+
+
+if __name__ == "__main__":
+    unittest.main()