diff --git a/crawler_platform.db-journal b/crawler_platform.db-journal deleted file mode 100644 index 8a9558b..0000000 Binary files a/crawler_platform.db-journal and /dev/null differ diff --git a/crawler_platform/app/core/extractor/ai_provider.py b/crawler_platform/app/core/extractor/ai_provider.py index 24d497d..35f6faf 100644 --- a/crawler_platform/app/core/extractor/ai_provider.py +++ b/crawler_platform/app/core/extractor/ai_provider.py @@ -149,6 +149,7 @@ class LLMJsonExtractor(AIExtractor): {"role": "user", "content": prompt}, ], "temperature": 0, + "response_format": extraction_response_format(), }, timeout=self.timeout_seconds, ) @@ -266,6 +267,55 @@ Page text: """.strip() +def extraction_response_format() -> dict[str, Any]: + return { + "type": "json_schema", + "json_schema": { + "name": "ontology_extraction", + "schema": { + "type": "object", + "properties": { + "entities": { + "type": "array", + "items": { + "type": "object", + "properties": { + "entity_type": {"type": "string"}, + "type": {"type": "string"}, + "name": {"type": "string"}, + "attributes": {"type": "object"}, + "confidence": {"type": "number"}, + "evidence_text": {"type": "string"}, + }, + "required": ["name"], + }, + }, + "claims": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subject_name": {"type": "string"}, + "subject_type": {"type": "string"}, + "predicate": {"type": "string"}, + "object_name": {"type": ["string", "null"]}, + "object_type": {"type": ["string", "null"]}, + "object_value": {}, + "evidence_text": {"type": ["string", "null"]}, + "evidence_summary": {"type": ["string", "null"]}, + "confidence": {"type": "number"}, + "confidence_reason": {"type": ["string", "null"]}, + }, + "required": ["subject_name", "subject_type", "predicate"], + }, + }, + }, + "required": ["entities", "claims"], + }, + }, + } + + def parse_json_content(content: str, retry=None) -> dict[str, Any]: try: return json.loads(content)