lasta/AI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test

Browse Source
This commit is contained in:
lasta
2026-05-11 22:55:01 +09:00
parent 3663734781
commit 0f34a451fc
10 changed files with 634 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
crawler_platform/app/core/database/repository.py
View File

@@ -82,6 +82,30 @@ class KnowledgeRepository:
raise KeyError(f"Source not found: {source_name}")
return source
def reset_project_runtime_data(self, project_id: int) -> dict[str, int]:
delete_order = [
models.FeedbackLog,
models.UserPreference,
models.UserProfile,
models.CrawlJob,
models.ExtractionLog,
models.Evidence,
models.Relation,
models.Claim,
models.Attribute,
models.Page,
models.Entity,
]
deleted: dict[str, int] = {}
for table_model in delete_order:
count = (
self.session.query(table_model)
.filter(table_model.project_id == project_id)
.delete(synchronize_session=False)
)
deleted[table_model.__tablename__] = int(count or 0)
return deleted
def upsert_page(
self,
project_id: int,

132
crawler_platform/app/core/extractor/ai_provider.py
View File

@@ -29,6 +29,10 @@ class LLMJsonExtractor(AIExtractor):
base_url: str | None = None,
timeout_seconds: int = 300,
):
# Local LM Studio runs on consumer hardware; keep a shorter timeout so
# we can quickly fallback instead of stalling a crawl worker for 5+ min.
if provider == "lm_studio" and timeout_seconds == 300:
timeout_seconds = 120
self.domain = domain
self.provider = provider
self.model = model
@@ -36,25 +40,18 @@ class LLMJsonExtractor(AIExtractor):
self.timeout_seconds = timeout_seconds
def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
try:
raw = self.complete_json(page_text, project_config)
except Exception as exc:
return self._fallback_bundle(page_text, project_config, str(exc))
bundle = ExtractionBundle(
entities=parse_entities(raw.get("entities", [])),
claims=parse_claims(raw.get("claims", [])),
extractor_name=self.name,
provider=self.provider,
raw_output={
"provider": self.provider,
"model": self.model,
"entity_count": len(raw.get("entities", [])),
"claim_count": len(raw.get("claims", [])),
},
)
if not bundle.entities or not bundle.claims:
return self._fallback_bundle(page_text, project_config, "AI returned no usable entities or claims")
return self.normalize_to_ontology(bundle, project_config.ontology)
errors: list[str] = []
for compact_mode in (False, True):
mode_name = "compact_retry" if compact_mode else "primary"
try:
raw = self.complete_json(page_text, project_config, compact=compact_mode)
bundle = self._bundle_from_raw(raw, mode_name)
if bundle.entities and bundle.claims:
return self.normalize_to_ontology(bundle, project_config.ontology)
errors.append(f"{mode_name}: AI returned no usable entities or claims")
except Exception as exc:
errors.append(f"{mode_name}: {exc}")
return self._fallback_bundle(page_text, project_config, " | ".join(errors))
def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
return self.extract(page_text, project_config).entities
@@ -80,10 +77,22 @@ class LLMJsonExtractor(AIExtractor):
claim.predicate = normalize_predicate(claim.predicate, ontology)
return bundle
def complete_json(self, page_text: str, project_config: ProjectConfig) -> dict[str, Any]:
prompt = build_extraction_prompt(page_text, project_config)
def complete_json(self, page_text: str, project_config: ProjectConfig, compact: bool = False) -> dict[str, Any]:
if self.provider == "lm_studio":
char_limit = 1200 if compact else 2200
max_tokens = 220 if compact else 420
else:
char_limit = 2200 if compact else 4000
max_tokens = 400 if compact else 800
prompt = build_extraction_prompt(page_text, project_config, char_limit=char_limit)
if self.provider == "openai":
return self._complete_openai_compatible(prompt, "OPENAI_API_KEY", "OPENAI_MODEL", self.base_url)
return self._complete_openai_compatible(
prompt,
"OPENAI_API_KEY",
"OPENAI_MODEL",
self.base_url,
max_tokens=max_tokens,
)
if self.provider == "lm_studio":
return self._complete_openai_compatible(
prompt,
@@ -91,9 +100,10 @@ class LLMJsonExtractor(AIExtractor):
"LM_STUDIO_MODEL",
normalize_openai_chat_url(self.base_url or "http://localhost:1234/v1"),
api_key_optional=True,
max_tokens=max_tokens,
)
if self.provider == "ollama":
return self._complete_ollama(prompt)
return self._complete_ollama(prompt, max_tokens=max_tokens)
raise ValueError(f"Unsupported AI extractor provider: {self.provider}")
def _fallback_bundle(self, page_text: str, project_config: ProjectConfig, error: str) -> ExtractionBundle:
@@ -104,13 +114,14 @@ class LLMJsonExtractor(AIExtractor):
bundle = GenericRuleBasedExtractor().extract(page_text, project_config)
bundle.extractor_name = f"{self.name}_with_rule_fallback"
bundle.provider = f"{self.provider}_fallback"
bundle.provider = self.provider
bundle.raw_output = {
**bundle.raw_output,
"ai_provider": self.provider,
"ai_model": self.model,
"ai_error": error,
"ai_warning": error,
"fallback": "rule_based",
"extraction_mode": "fallback",
}
for entity in bundle.entities:
entity.metadata["ai_fallback_reason"] = error
@@ -119,6 +130,21 @@ class LLMJsonExtractor(AIExtractor):
claim.confidence_reason = f"{claim.confidence_reason}; AI fallback: {error}" if claim.confidence_reason else error
return bundle
def _bundle_from_raw(self, raw: dict[str, Any], mode: str) -> ExtractionBundle:
return ExtractionBundle(
entities=parse_entities(raw.get("entities", [])),
claims=parse_claims(raw.get("claims", [])),
extractor_name=self.name,
provider=self.provider,
raw_output={
"provider": self.provider,
"model": self.model,
"entity_count": len(raw.get("entities", [])),
"claim_count": len(raw.get("claims", [])),
"extraction_mode": mode,
},
)
def _complete_openai_compatible(
self,
prompt: str,
@@ -126,6 +152,7 @@ class LLMJsonExtractor(AIExtractor):
model_env: str,
endpoint: str | None,
api_key_optional: bool = False,
max_tokens: int | None = None,
) -> dict[str, Any]:
api_key = os.getenv(api_key_env)
model = self.model or os.getenv(model_env)
@@ -150,6 +177,7 @@ class LLMJsonExtractor(AIExtractor):
],
"temperature": 0,
"response_format": extraction_response_format(),
"max_tokens": max_tokens,
},
timeout=self.timeout_seconds,
)
@@ -159,7 +187,7 @@ class LLMJsonExtractor(AIExtractor):
content = data["choices"][0]["message"]["content"]
return parse_json_content(content, retry=lambda bad: self._repair_json_with_model(bad, endpoint, headers, model))
def _complete_ollama(self, prompt: str) -> dict[str, Any]:
def _complete_ollama(self, prompt: str, max_tokens: int | None = None) -> dict[str, Any]:
model = self.model or os.getenv("OLLAMA_MODEL")
if not model:
raise RuntimeError("Ollama model is required. Set UI model field or OLLAMA_MODEL.")
@@ -174,6 +202,7 @@ class LLMJsonExtractor(AIExtractor):
],
"stream": False,
"format": "json",
"options": {"num_predict": max_tokens} if max_tokens else {},
},
timeout=self.timeout_seconds,
)
@@ -208,6 +237,7 @@ class LLMJsonExtractor(AIExtractor):
},
],
"temperature": 0,
"max_tokens": 300,
},
timeout=self.timeout_seconds,
)
@@ -216,8 +246,8 @@ class LLMJsonExtractor(AIExtractor):
return parse_json_content(response.json()["choices"][0]["message"]["content"])
def build_extraction_prompt(page_text: str, project_config: ProjectConfig) -> str:
clipped_text = page_text[:6000]
def build_extraction_prompt(page_text: str, project_config: ProjectConfig, char_limit: int = 4000) -> str:
clipped_text = prepare_page_text_for_prompt(page_text, char_limit)
ontology = project_config.ontology or {}
return f"""
Project domain: {project_config.domain}
@@ -259,14 +289,58 @@ Rules:
- Use only ontology predicates when possible.
- If object is a simple value like price, put it in object_value and leave object_name/object_type null.
- If unsure, lower confidence instead of inventing.
- Extract at most 20 entities and 30 claims.
- Extract at most 10 entities and 15 claims.
- For perfume, prioritize name, brand, top/middle/base notes, accords, mood, season, occasion, price, review keywords.
- Skip navigation, cart, coupon, pagination, login, and policy boilerplate unless it contains product facts.
Page text:
{clipped_text}
""".strip()
def prepare_page_text_for_prompt(page_text: str, char_limit: int) -> str:
noisy_terms = {
"first page",
"previous page",
"next page",
"last page",
"add to cart",
"cart",
"checkout",
"coupon",
"login",
"sign in",
"privacy policy",
"terms",
"review write",
"all reviews",
"first",
"previous",
"next",
"last",
}
lines = []
for raw in page_text.splitlines():
line = raw.strip()
if not line:
continue
lowered = line.lower()
if lowered in noisy_terms:
continue
if line.isdigit():
continue
if len(line) <= 2:
continue
lines.append(line)
compact = "\n".join(lines) if lines else page_text
if len(compact) <= char_limit:
return compact
head_len = int(char_limit * 0.7)
tail_len = char_limit - head_len
return f"{compact[:head_len]}\n...\n{compact[-tail_len:]}"
def extraction_response_format() -> dict[str, Any]:
return {
"type": "json_schema",